From 3d99a1ca66060ebbee02072ed17353e9c5695d9b Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 25 Jun 2018 15:01:59 +0200 Subject: [PATCH 01/51] VmaJsonWriter::ContinueString: Fixed bug with incorrect JSON generated when pUserData string contains '\' character. --- src/vk_mem_alloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 8981fdc..03a0e27 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -5103,7 +5103,7 @@ void VmaJsonWriter::ContinueString(const char* pStr) for(size_t i = 0; i < strLen; ++i) { char ch = pStr[i]; - if(ch == '\'') + if(ch == '\\') { m_SB.Add("\\\\"); } From 0d80dab2c09cb0bbeeaa075fa79b6c1356d1d37a Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Wed, 1 Aug 2018 16:20:24 +0200 Subject: [PATCH 02/51] Fixed minor documentation mistakes - issue #32 - thanks @dgough ! --- src/vk_mem_alloc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 03a0e27..149445e 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -298,7 +298,7 @@ The library provides following functions for mapping of a specific #VmaAllocatio They are safer and more convenient to use than standard Vulkan functions. You can map an allocation multiple times simultaneously - mapping is reference-counted internally. You can also map different allocations simultaneously regardless of whether they use the same `VkDeviceMemory` block. -They way it's implemented is that the library always maps entire memory block, not just region of the allocation. +The way it's implemented is that the library always maps entire memory block, not just region of the allocation. For further details, see description of vmaMapMemory() function. Example: @@ -312,7 +312,7 @@ struct ConstantBuffer ConstantBuffer constantBufferData; VmaAllocator allocator; -VmaBuffer constantBuffer; +VkBuffer constantBuffer; VmaAllocation constantBufferAllocation; // You can map and fill your buffer using following code: From 1d536111cc9f32d95cd6894574a234772003bc16 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 12:23:55 +0200 Subject: [PATCH 03/51] Removed path to MathFu from premake5.lua, which is not longer included in the project. --- premake/premake5.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/premake/premake5.lua b/premake/premake5.lua index 8893601..9a5b723 100644 --- a/premake/premake5.lua +++ b/premake/premake5.lua @@ -12,7 +12,7 @@ startproject "VulkanSample" filter "platforms:x64" system "Windows" architecture "x64" -includedirs { "../third_party/mathfu-1.1.0/include", "$(VULKAN_SDK)/Include" } +includedirs { "$(VULKAN_SDK)/Include" } libdirs { "$(VULKAN_SDK)/Lib" } filter "platforms:Linux-x64" From 6d2e2e0cac9d0a1775683647e9b730aff57eed5b Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 13:44:20 +0200 Subject: [PATCH 04/51] Created project VmaReplay, started coding it. --- premake/premake5.lua | 39 + src/VmaReplay/Common.cpp | 154 +++ src/VmaReplay/Common.h | 96 ++ src/VmaReplay/VmaReplay.cpp | 2018 +++++++++++++++++++++++++++++++++++ src/VmaReplay/VmaUsage.cpp | 2 + src/VmaReplay/VmaUsage.h | 27 + src/vk_mem_alloc.h | 184 +++- 7 files changed, 2519 insertions(+), 1 deletion(-) create mode 100644 src/VmaReplay/Common.cpp create mode 100644 src/VmaReplay/Common.h create mode 100644 src/VmaReplay/VmaReplay.cpp create mode 100644 src/VmaReplay/VmaUsage.cpp create mode 100644 src/VmaReplay/VmaUsage.h diff --git a/premake/premake5.lua b/premake/premake5.lua index 9a5b723..c6d2b93 100644 --- a/premake/premake5.lua +++ b/premake/premake5.lua @@ -21,6 +21,7 @@ architecture "x64" includedirs { "$(VULKAN_SDK)/include" } libdirs { "$(VULKAN_SDK)/lib" } + project "VulkanSample" kind "ConsoleApp" language "C++" @@ -57,3 +58,41 @@ buildoptions { "/MDd" } filter { "configurations:Release", "platforms:Windows-x64" } buildoptions { "/MD" } + + +project "VmaReplay" +kind "ConsoleApp" +language "C++" +location "../build" +filename ("VmaReplay_" .. _SUFFIX) +targetdir "../bin" +objdir "../build/Desktop_%{_SUFFIX}/%{cfg.platform}/%{cfg.buildcfg}" +floatingpoint "Fast" +files { "../src/VmaReplay/*.h", "../src/VmaReplay/*.cpp" } +flags { "NoPCH", "FatalWarnings" } +characterset "Default" + +filter "configurations:Debug" +defines { "_DEBUG", "DEBUG" } +flags { } +targetsuffix ("_Debug_" .. _SUFFIX) + +filter "configurations:Release" +defines { "NDEBUG" } +optimize "On" +flags { "LinkTimeOptimization" } +targetsuffix ("_Release_" .. _SUFFIX) + +filter { "platforms:x64" } +defines { "WIN32", "_CONSOLE", "PROFILE", "_WINDOWS", "_WIN32_WINNT=0x0601" } +links { "vulkan-1" } + +filter { "platforms:Linux-x64" } +buildoptions { "-std=c++0x" } +links { "vulkan" } + +filter { "configurations:Debug", "platforms:x64" } +buildoptions { "/MDd" } + +filter { "configurations:Release", "platforms:Windows-x64" } +buildoptions { "/MD" } diff --git a/src/VmaReplay/Common.cpp b/src/VmaReplay/Common.cpp new file mode 100644 index 0000000..8f2baf3 --- /dev/null +++ b/src/VmaReplay/Common.cpp @@ -0,0 +1,154 @@ +#include "Common.h" + +void ReadFile(std::vector& out, const char* fileName) +{ + std::ifstream file(fileName, std::ios::ate | std::ios::binary); + assert(file.is_open()); + size_t fileSize = (size_t)file.tellg(); + if(fileSize > 0) + { + out.resize(fileSize); + file.seekg(0); + file.read(out.data(), fileSize); + } + else + out.clear(); +} + +void SetConsoleColor(CONSOLE_COLOR color) +{ + WORD attr = 0; + switch(color) + { + case CONSOLE_COLOR::INFO: + attr = FOREGROUND_INTENSITY;; + break; + case CONSOLE_COLOR::NORMAL: + attr = FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE; + break; + case CONSOLE_COLOR::WARNING: + attr = FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_INTENSITY; + break; + case CONSOLE_COLOR::ERROR_: + attr = FOREGROUND_RED | FOREGROUND_INTENSITY; + break; + default: + assert(0); + } + + HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); + SetConsoleTextAttribute(out, attr); +} + +void PrintMessage(CONSOLE_COLOR color, const char* msg) +{ + if(color != CONSOLE_COLOR::NORMAL) + SetConsoleColor(color); + + printf("%s\n", msg); + + if (color != CONSOLE_COLOR::NORMAL) + SetConsoleColor(CONSOLE_COLOR::NORMAL); +} + +void PrintMessage(CONSOLE_COLOR color, const wchar_t* msg) +{ + if(color != CONSOLE_COLOR::NORMAL) + SetConsoleColor(color); + + wprintf(L"%s\n", msg); + + if (color != CONSOLE_COLOR::NORMAL) + SetConsoleColor(CONSOLE_COLOR::NORMAL); +} + +static const size_t CONSOLE_SMALL_BUF_SIZE = 256; + +void PrintMessageV(CONSOLE_COLOR color, const char* format, va_list argList) +{ + size_t dstLen = (size_t)::_vscprintf(format, argList); + if(dstLen) + { + bool useSmallBuf = dstLen < CONSOLE_SMALL_BUF_SIZE; + char smallBuf[CONSOLE_SMALL_BUF_SIZE]; + std::vector bigBuf(useSmallBuf ? 0 : dstLen + 1); + char* bufPtr = useSmallBuf ? smallBuf : bigBuf.data(); + ::vsprintf_s(bufPtr, dstLen + 1, format, argList); + PrintMessage(color, bufPtr); + } +} + +void PrintMessageV(CONSOLE_COLOR color, const wchar_t* format, va_list argList) +{ + size_t dstLen = (size_t)::_vcwprintf(format, argList); + if(dstLen) + { + bool useSmallBuf = dstLen < CONSOLE_SMALL_BUF_SIZE; + wchar_t smallBuf[CONSOLE_SMALL_BUF_SIZE]; + std::vector bigBuf(useSmallBuf ? 0 : dstLen + 1); + wchar_t* bufPtr = useSmallBuf ? smallBuf : bigBuf.data(); + ::vswprintf_s(bufPtr, dstLen + 1, format, argList); + PrintMessage(color, bufPtr); + } +} + +void PrintMessageF(CONSOLE_COLOR color, const char* format, ...) +{ + va_list argList; + va_start(argList, format); + PrintMessageV(color, format, argList); + va_end(argList); +} + +void PrintMessageF(CONSOLE_COLOR color, const wchar_t* format, ...) +{ + va_list argList; + va_start(argList, format); + PrintMessageV(color, format, argList); + va_end(argList); +} + +void PrintWarningF(const char* format, ...) +{ + va_list argList; + va_start(argList, format); + PrintMessageV(CONSOLE_COLOR::WARNING, format, argList); + va_end(argList); +} + +void PrintWarningF(const wchar_t* format, ...) +{ + va_list argList; + va_start(argList, format); + PrintMessageV(CONSOLE_COLOR::WARNING, format, argList); + va_end(argList); +} + +void PrintErrorF(const char* format, ...) +{ + va_list argList; + va_start(argList, format); + PrintMessageV(CONSOLE_COLOR::WARNING, format, argList); + va_end(argList); +} + +void PrintErrorF(const wchar_t* format, ...) +{ + va_list argList; + va_start(argList, format); + PrintMessageV(CONSOLE_COLOR::WARNING, format, argList); + va_end(argList); +} + +void SaveFile(const wchar_t* filePath, const void* data, size_t dataSize) +{ + FILE* f = nullptr; + _wfopen_s(&f, filePath, L"wb"); + if(f) + { + fwrite(data, 1, dataSize, f); + fclose(f); + } + else + assert(0); +} diff --git a/src/VmaReplay/Common.h b/src/VmaReplay/Common.h new file mode 100644 index 0000000..c4934b2 --- /dev/null +++ b/src/VmaReplay/Common.h @@ -0,0 +1,96 @@ +#pragma once + +#include "VmaUsage.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +typedef std::chrono::high_resolution_clock::time_point time_point; +typedef std::chrono::high_resolution_clock::duration duration; + +#define ERR_GUARD_VULKAN(Expr) do { VkResult res__ = (Expr); if (res__ < 0) assert(0); } while(0) + +extern VkPhysicalDevice g_hPhysicalDevice; +extern VkDevice g_hDevice; +extern VmaAllocator g_hAllocator; +extern bool g_MemoryAliasingWarningEnabled; + +inline float ToFloatSeconds(duration d) +{ + return std::chrono::duration_cast>(d).count(); +} + +template +inline T ceil_div(T x, T y) +{ + return (x+y-1) / y; +} + +template +static inline T align_up(T val, T align) +{ + return (val + align - 1) / align * align; +} + +static const float PI = 3.14159265358979323846264338327950288419716939937510582f; + +class RandomNumberGenerator +{ +public: + RandomNumberGenerator() : m_Value{GetTickCount()} {} + RandomNumberGenerator(uint32_t seed) : m_Value{seed} { } + void Seed(uint32_t seed) { m_Value = seed; } + uint32_t Generate() { return GenerateFast() ^ (GenerateFast() >> 7); } + +private: + uint32_t m_Value; + uint32_t GenerateFast() { return m_Value = (m_Value * 196314165 + 907633515); } +}; + +void ReadFile(std::vector& out, const char* fileName); + +enum class CONSOLE_COLOR +{ + INFO, + NORMAL, + WARNING, + ERROR_, + COUNT +}; + +void SetConsoleColor(CONSOLE_COLOR color); + +void PrintMessage(CONSOLE_COLOR color, const char* msg); +void PrintMessage(CONSOLE_COLOR color, const wchar_t* msg); + +inline void Print(const char* msg) { PrintMessage(CONSOLE_COLOR::NORMAL, msg); } +inline void Print(const wchar_t* msg) { PrintMessage(CONSOLE_COLOR::NORMAL, msg); } +inline void PrintWarning(const char* msg) { PrintMessage(CONSOLE_COLOR::WARNING, msg); } +inline void PrintWarning(const wchar_t* msg) { PrintMessage(CONSOLE_COLOR::WARNING, msg); } +inline void PrintError(const char* msg) { PrintMessage(CONSOLE_COLOR::ERROR_, msg); } +inline void PrintError(const wchar_t* msg) { PrintMessage(CONSOLE_COLOR::ERROR_, msg); } + +void PrintMessageV(CONSOLE_COLOR color, const char* format, va_list argList); +void PrintMessageV(CONSOLE_COLOR color, const wchar_t* format, va_list argList); +void PrintMessageF(CONSOLE_COLOR color, const char* format, ...); +void PrintMessageF(CONSOLE_COLOR color, const wchar_t* format, ...); +void PrintWarningF(const char* format, ...); +void PrintWarningF(const wchar_t* format, ...); +void PrintErrorF(const char* format, ...); +void PrintErrorF(const wchar_t* format, ...); + +void SaveFile(const wchar_t* filePath, const void* data, size_t dataSize); diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp new file mode 100644 index 0000000..3fd9b4b --- /dev/null +++ b/src/VmaReplay/VmaReplay.cpp @@ -0,0 +1,2018 @@ +// +// Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// + +#include "VmaUsage.h" +#include "Common.h" + +#if 0 +static const char* const SHADER_PATH1 = "./"; +static const char* const SHADER_PATH2 = "../bin/"; +static const wchar_t* const WINDOW_CLASS_NAME = L"VULKAN_MEMORY_ALLOCATOR_SAMPLE"; +static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_LUNARG_standard_validation"; +static const char* const APP_TITLE_A = "Vulkan Memory Allocator Sample 2.0"; +static const wchar_t* const APP_TITLE_W = L"Vulkan Memory Allocator Sample 2.0"; + +static const bool VSYNC = true; +static const uint32_t COMMAND_BUFFER_COUNT = 2; +static void* const CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA = (void*)(intptr_t)43564544; +static const bool USE_CUSTOM_CPU_ALLOCATION_CALLBACKS = false; + +VkPhysicalDevice g_hPhysicalDevice; +VkDevice g_hDevice; +VmaAllocator g_hAllocator; +bool g_MemoryAliasingWarningEnabled = true; + +static bool g_EnableValidationLayer = true; +static bool VK_KHR_get_memory_requirements2_enabled = false; +static bool VK_KHR_dedicated_allocation_enabled = false; + +static HINSTANCE g_hAppInstance; +static HWND g_hWnd; +static LONG g_SizeX = 1280, g_SizeY = 720; +static VkInstance g_hVulkanInstance; +static VkSurfaceKHR g_hSurface; +static VkQueue g_hPresentQueue; +static VkSurfaceFormatKHR g_SurfaceFormat; +static VkExtent2D g_Extent; +static VkSwapchainKHR g_hSwapchain; +static std::vector g_SwapchainImages; +static std::vector g_SwapchainImageViews; +static std::vector g_Framebuffers; +static VkCommandPool g_hCommandPool; +static VkCommandBuffer g_MainCommandBuffers[COMMAND_BUFFER_COUNT]; +static VkFence g_MainCommandBufferExecutedFances[COMMAND_BUFFER_COUNT]; +static uint32_t g_NextCommandBufferIndex; +static VkSemaphore g_hImageAvailableSemaphore; +static VkSemaphore g_hRenderFinishedSemaphore; +static uint32_t g_GraphicsQueueFamilyIndex = UINT_MAX; +static uint32_t g_PresentQueueFamilyIndex = UINT_MAX; +static VkDescriptorSetLayout g_hDescriptorSetLayout; +static VkDescriptorPool g_hDescriptorPool; +static VkDescriptorSet g_hDescriptorSet; // Automatically destroyed with m_DescriptorPool. +static VkSampler g_hSampler; +static VkFormat g_DepthFormat; +static VkImage g_hDepthImage; +static VmaAllocation g_hDepthImageAlloc; +static VkImageView g_hDepthImageView; + +static VkSurfaceCapabilitiesKHR g_SurfaceCapabilities; +static std::vector g_SurfaceFormats; +static std::vector g_PresentModes; + +static PFN_vkCreateDebugReportCallbackEXT g_pvkCreateDebugReportCallbackEXT; +static PFN_vkDebugReportMessageEXT g_pvkDebugReportMessageEXT; +static PFN_vkDestroyDebugReportCallbackEXT g_pvkDestroyDebugReportCallbackEXT; +static VkDebugReportCallbackEXT g_hCallback; + +static VkQueue g_hGraphicsQueue; +static VkCommandBuffer g_hTemporaryCommandBuffer; + +static VkPipelineLayout g_hPipelineLayout; +static VkRenderPass g_hRenderPass; +static VkPipeline g_hPipeline; + +static VkBuffer g_hVertexBuffer; +static VmaAllocation g_hVertexBufferAlloc; +static VkBuffer g_hIndexBuffer; +static VmaAllocation g_hIndexBufferAlloc; +static uint32_t g_VertexCount; +static uint32_t g_IndexCount; + +static VkImage g_hTextureImage; +static VmaAllocation g_hTextureImageAlloc; +static VkImageView g_hTextureImageView; + +static void* CustomCpuAllocation( + void* pUserData, size_t size, size_t alignment, + VkSystemAllocationScope allocationScope) +{ + assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); + return _aligned_malloc(size, alignment); +} + +static void* CustomCpuReallocation( + void* pUserData, void* pOriginal, size_t size, size_t alignment, + VkSystemAllocationScope allocationScope) +{ + assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); + return _aligned_realloc(pOriginal, size, alignment); +} + +static void CustomCpuFree(void* pUserData, void* pMemory) +{ + assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); + _aligned_free(pMemory); +} + +static void BeginSingleTimeCommands() +{ + VkCommandBufferBeginInfo cmdBufBeginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + cmdBufBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + ERR_GUARD_VULKAN( vkBeginCommandBuffer(g_hTemporaryCommandBuffer, &cmdBufBeginInfo) ); +} + +static void EndSingleTimeCommands() +{ + ERR_GUARD_VULKAN( vkEndCommandBuffer(g_hTemporaryCommandBuffer) ); + + VkSubmitInfo submitInfo = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &g_hTemporaryCommandBuffer; + + ERR_GUARD_VULKAN( vkQueueSubmit(g_hGraphicsQueue, 1, &submitInfo, VK_NULL_HANDLE) ); + ERR_GUARD_VULKAN( vkQueueWaitIdle(g_hGraphicsQueue) ); +} + +static void LoadShader(std::vector& out, const char* fileName) +{ + std::ifstream file(std::string(SHADER_PATH1) + fileName, std::ios::ate | std::ios::binary); + if(file.is_open() == false) + file.open(std::string(SHADER_PATH2) + fileName, std::ios::ate | std::ios::binary); + assert(file.is_open()); + size_t fileSize = (size_t)file.tellg(); + if(fileSize > 0) + { + out.resize(fileSize); + file.seekg(0); + file.read(out.data(), fileSize); + file.close(); + } + else + out.clear(); +} + +VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( + VkDebugReportFlagsEXT flags, + VkDebugReportObjectTypeEXT objectType, + uint64_t object, + size_t location, + int32_t messageCode, + const char* pLayerPrefix, + const char* pMessage, + void* pUserData) +{ + // "Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug." + if(!g_MemoryAliasingWarningEnabled && flags == VK_DEBUG_REPORT_WARNING_BIT_EXT && + (strstr(pMessage, " is aliased with non-linear ") || strstr(pMessage, " is aliased with linear "))) + { + return VK_FALSE; + } + + // Ignoring because when VK_KHR_dedicated_allocation extension is enabled, + // vkGetBufferMemoryRequirements2KHR function is used instead, while Validation + // Layer seems to be unaware of it. + if (strstr(pMessage, "but vkGetBufferMemoryRequirements() has not been called on that buffer") != nullptr) + { + return VK_FALSE; + } + if (strstr(pMessage, "but vkGetImageMemoryRequirements() has not been called on that image") != nullptr) + { + return VK_FALSE; + } + + switch(flags) + { + case VK_DEBUG_REPORT_WARNING_BIT_EXT: + SetConsoleColor(CONSOLE_COLOR::WARNING); + break; + case VK_DEBUG_REPORT_ERROR_BIT_EXT: + SetConsoleColor(CONSOLE_COLOR::ERROR_); + break; + default: + SetConsoleColor(CONSOLE_COLOR::INFO); + } + + printf("%s \xBA %s\n", pLayerPrefix, pMessage); + + SetConsoleColor(CONSOLE_COLOR::NORMAL); + + if(flags == VK_DEBUG_REPORT_WARNING_BIT_EXT || + flags == VK_DEBUG_REPORT_ERROR_BIT_EXT) + { + OutputDebugStringA(pMessage); + OutputDebugStringA("\n"); + } + + return VK_FALSE; +} + +static VkSurfaceFormatKHR ChooseSurfaceFormat() +{ + assert(!g_SurfaceFormats.empty()); + + if((g_SurfaceFormats.size() == 1) && (g_SurfaceFormats[0].format == VK_FORMAT_UNDEFINED)) + { + VkSurfaceFormatKHR result = { VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR }; + return result; + } + + for(const auto& format : g_SurfaceFormats) + { + if((format.format == VK_FORMAT_B8G8R8A8_UNORM) && + (format.colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR)) + { + return format; + } + } + + return g_SurfaceFormats[0]; +} + +VkPresentModeKHR ChooseSwapPresentMode() +{ + VkPresentModeKHR preferredMode = VSYNC ? VK_PRESENT_MODE_MAILBOX_KHR : VK_PRESENT_MODE_IMMEDIATE_KHR; + + if(std::find(g_PresentModes.begin(), g_PresentModes.end(), preferredMode) != + g_PresentModes.end()) + { + return preferredMode; + } + + return VK_PRESENT_MODE_FIFO_KHR; +} + +static VkExtent2D ChooseSwapExtent() +{ + if(g_SurfaceCapabilities.currentExtent.width != UINT_MAX) + return g_SurfaceCapabilities.currentExtent; + + VkExtent2D result = { + std::max(g_SurfaceCapabilities.minImageExtent.width, + std::min(g_SurfaceCapabilities.maxImageExtent.width, (uint32_t)g_SizeX)), + std::max(g_SurfaceCapabilities.minImageExtent.height, + std::min(g_SurfaceCapabilities.maxImageExtent.height, (uint32_t)g_SizeY)) }; + return result; +} + +struct Vertex +{ + float pos[3]; + float color[3]; + float texCoord[2]; +}; + +static void CreateMesh() +{ + assert(g_hAllocator); + + static Vertex vertices[] = { + // -X + { { -1.f, -1.f, -1.f}, {1.0f, 0.0f, 0.0f}, {0.f, 0.f} }, + { { -1.f, -1.f, 1.f}, {1.0f, 0.0f, 0.0f}, {1.f, 0.f} }, + { { -1.f, 1.f, -1.f}, {1.0f, 0.0f, 0.0f}, {0.f, 1.f} }, + { { -1.f, 1.f, 1.f}, {1.0f, 0.0f, 0.0f}, {1.f, 1.f} }, + // +X + { { 1.f, -1.f, 1.f}, {0.0f, 1.0f, 0.0f}, {0.f, 0.f} }, + { { 1.f, -1.f, -1.f}, {0.0f, 1.0f, 0.0f}, {1.f, 0.f} }, + { { 1.f, 1.f, 1.f}, {0.0f, 1.0f, 0.0f}, {0.f, 1.f} }, + { { 1.f, 1.f, -1.f}, {0.0f, 1.0f, 0.0f}, {1.f, 1.f} }, + // -Z + { { 1.f, -1.f, -1.f}, {0.0f, 0.0f, 1.0f}, {0.f, 0.f} }, + { {-1.f, -1.f, -1.f}, {0.0f, 0.0f, 1.0f}, {1.f, 0.f} }, + { { 1.f, 1.f, -1.f}, {0.0f, 0.0f, 1.0f}, {0.f, 1.f} }, + { {-1.f, 1.f, -1.f}, {0.0f, 0.0f, 1.0f}, {1.f, 1.f} }, + // +Z + { {-1.f, -1.f, 1.f}, {1.0f, 1.0f, 0.0f}, {0.f, 0.f} }, + { { 1.f, -1.f, 1.f}, {1.0f, 1.0f, 0.0f}, {1.f, 0.f} }, + { {-1.f, 1.f, 1.f}, {1.0f, 1.0f, 0.0f}, {0.f, 1.f} }, + { { 1.f, 1.f, 1.f}, {1.0f, 1.0f, 0.0f}, {1.f, 1.f} }, + // -Y + { {-1.f, -1.f, -1.f}, {0.0f, 1.0f, 1.0f}, {0.f, 0.f} }, + { { 1.f, -1.f, -1.f}, {0.0f, 1.0f, 1.0f}, {1.f, 0.f} }, + { {-1.f, -1.f, 1.f}, {0.0f, 1.0f, 1.0f}, {0.f, 1.f} }, + { { 1.f, -1.f, 1.f}, {0.0f, 1.0f, 1.0f}, {1.f, 1.f} }, + // +Y + { { 1.f, 1.f, -1.f}, {1.0f, 0.0f, 1.0f}, {0.f, 0.f} }, + { {-1.f, 1.f, -1.f}, {1.0f, 0.0f, 1.0f}, {1.f, 0.f} }, + { { 1.f, 1.f, 1.f}, {1.0f, 0.0f, 1.0f}, {0.f, 1.f} }, + { {-1.f, 1.f, 1.f}, {1.0f, 0.0f, 1.0f}, {1.f, 1.f} }, + }; + static uint16_t indices[] = { + 0, 1, 2, 3, USHRT_MAX, + 4, 5, 6, 7, USHRT_MAX, + 8, 9, 10, 11, USHRT_MAX, + 12, 13, 14, 15, USHRT_MAX, + 16, 17, 18, 19, USHRT_MAX, + 20, 21, 22, 23, USHRT_MAX, + }; + + size_t vertexBufferSize = sizeof(Vertex) * _countof(vertices); + size_t indexBufferSize = sizeof(uint16_t) * _countof(indices); + g_IndexCount = (uint32_t)_countof(indices); + + // Create vertex buffer + + VkBufferCreateInfo vbInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + vbInfo.size = vertexBufferSize; + vbInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + vbInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + + VmaAllocationCreateInfo vbAllocCreateInfo = {}; + vbAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + vbAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + + VkBuffer stagingVertexBuffer = VK_NULL_HANDLE; + VmaAllocation stagingVertexBufferAlloc = VK_NULL_HANDLE; + VmaAllocationInfo stagingVertexBufferAllocInfo = {}; + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &vbInfo, &vbAllocCreateInfo, &stagingVertexBuffer, &stagingVertexBufferAlloc, &stagingVertexBufferAllocInfo) ); + + memcpy(stagingVertexBufferAllocInfo.pMappedData, vertices, vertexBufferSize); + + // No need to flush stagingVertexBuffer memory because CPU_ONLY memory is always HOST_COHERENT. + + vbInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; + vbAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + vbAllocCreateInfo.flags = 0; + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &vbInfo, &vbAllocCreateInfo, &g_hVertexBuffer, &g_hVertexBufferAlloc, nullptr) ); + + // Create index buffer + + VkBufferCreateInfo ibInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + ibInfo.size = indexBufferSize; + ibInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + ibInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + + VmaAllocationCreateInfo ibAllocCreateInfo = {}; + ibAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + ibAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + + VkBuffer stagingIndexBuffer = VK_NULL_HANDLE; + VmaAllocation stagingIndexBufferAlloc = VK_NULL_HANDLE; + VmaAllocationInfo stagingIndexBufferAllocInfo = {}; + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &ibInfo, &ibAllocCreateInfo, &stagingIndexBuffer, &stagingIndexBufferAlloc, &stagingIndexBufferAllocInfo) ); + + memcpy(stagingIndexBufferAllocInfo.pMappedData, indices, indexBufferSize); + + // No need to flush stagingIndexBuffer memory because CPU_ONLY memory is always HOST_COHERENT. + + ibInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT; + ibAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + ibAllocCreateInfo.flags = 0; + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &ibInfo, &ibAllocCreateInfo, &g_hIndexBuffer, &g_hIndexBufferAlloc, nullptr) ); + + // Copy buffers + + BeginSingleTimeCommands(); + + VkBufferCopy vbCopyRegion = {}; + vbCopyRegion.srcOffset = 0; + vbCopyRegion.dstOffset = 0; + vbCopyRegion.size = vbInfo.size; + vkCmdCopyBuffer(g_hTemporaryCommandBuffer, stagingVertexBuffer, g_hVertexBuffer, 1, &vbCopyRegion); + + VkBufferCopy ibCopyRegion = {}; + ibCopyRegion.srcOffset = 0; + ibCopyRegion.dstOffset = 0; + ibCopyRegion.size = ibInfo.size; + vkCmdCopyBuffer(g_hTemporaryCommandBuffer, stagingIndexBuffer, g_hIndexBuffer, 1, &ibCopyRegion); + + EndSingleTimeCommands(); + + vmaDestroyBuffer(g_hAllocator, stagingIndexBuffer, stagingIndexBufferAlloc); + vmaDestroyBuffer(g_hAllocator, stagingVertexBuffer, stagingVertexBufferAlloc); +} + +static void CreateTexture(uint32_t sizeX, uint32_t sizeY) +{ + // Create Image + + const VkDeviceSize imageSize = sizeX * sizeY * 4; + + VkImageCreateInfo stagingImageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + stagingImageInfo.imageType = VK_IMAGE_TYPE_2D; + stagingImageInfo.extent.width = sizeX; + stagingImageInfo.extent.height = sizeY; + stagingImageInfo.extent.depth = 1; + stagingImageInfo.mipLevels = 1; + stagingImageInfo.arrayLayers = 1; + stagingImageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + stagingImageInfo.tiling = VK_IMAGE_TILING_LINEAR; + stagingImageInfo.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; + stagingImageInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + stagingImageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + stagingImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + stagingImageInfo.flags = 0; + + VmaAllocationCreateInfo stagingImageAllocCreateInfo = {}; + stagingImageAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + stagingImageAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; + + VkImage stagingImage = VK_NULL_HANDLE; + VmaAllocation stagingImageAlloc = VK_NULL_HANDLE; + VmaAllocationInfo stagingImageAllocInfo = {}; + ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &stagingImageInfo, &stagingImageAllocCreateInfo, &stagingImage, &stagingImageAlloc, &stagingImageAllocInfo) ); + + VkImageSubresource imageSubresource = {}; + imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imageSubresource.mipLevel = 0; + imageSubresource.arrayLayer = 0; + + VkSubresourceLayout imageLayout = {}; + vkGetImageSubresourceLayout(g_hDevice, stagingImage, &imageSubresource, &imageLayout); + + char* const pMipLevelData = (char*)stagingImageAllocInfo.pMappedData + imageLayout.offset; + uint8_t* pRowData = (uint8_t*)pMipLevelData; + for(uint32_t y = 0; y < sizeY; ++y) + { + uint32_t* pPixelData = (uint32_t*)pRowData; + for(uint32_t x = 0; x < sizeY; ++x) + { + *pPixelData = + ((x & 0x18) == 0x08 ? 0x000000FF : 0x00000000) | + ((x & 0x18) == 0x10 ? 0x0000FFFF : 0x00000000) | + ((y & 0x18) == 0x08 ? 0x0000FF00 : 0x00000000) | + ((y & 0x18) == 0x10 ? 0x00FF0000 : 0x00000000); + ++pPixelData; + } + pRowData += imageLayout.rowPitch; + } + + // No need to flush stagingImage memory because CPU_ONLY memory is always HOST_COHERENT. + + // Create g_hTextureImage in GPU memory. + + VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.extent.width = sizeX; + imageInfo.extent.height = sizeY; + imageInfo.extent.depth = 1; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.flags = 0; + + VmaAllocationCreateInfo imageAllocCreateInfo = {}; + imageAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &imageInfo, &imageAllocCreateInfo, &g_hTextureImage, &g_hTextureImageAlloc, nullptr) ); + + // Transition image layouts, copy image. + + BeginSingleTimeCommands(); + + VkImageMemoryBarrier imgMemBarrier = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; + imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; + imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + imgMemBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + imgMemBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + imgMemBarrier.image = stagingImage; + imgMemBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imgMemBarrier.subresourceRange.baseMipLevel = 0; + imgMemBarrier.subresourceRange.levelCount = 1; + imgMemBarrier.subresourceRange.baseArrayLayer = 0; + imgMemBarrier.subresourceRange.layerCount = 1; + imgMemBarrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; + imgMemBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + + vkCmdPipelineBarrier( + g_hTemporaryCommandBuffer, + VK_PIPELINE_STAGE_HOST_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, + 0, nullptr, + 0, nullptr, + 1, &imgMemBarrier); + + imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + imgMemBarrier.image = g_hTextureImage; + imgMemBarrier.srcAccessMask = 0; + imgMemBarrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + + vkCmdPipelineBarrier( + g_hTemporaryCommandBuffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, + 0, nullptr, + 0, nullptr, + 1, &imgMemBarrier); + + VkImageCopy imageCopy = {}; + imageCopy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imageCopy.srcSubresource.baseArrayLayer = 0; + imageCopy.srcSubresource.mipLevel = 0; + imageCopy.srcSubresource.layerCount = 1; + imageCopy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imageCopy.dstSubresource.baseArrayLayer = 0; + imageCopy.dstSubresource.mipLevel = 0; + imageCopy.dstSubresource.layerCount = 1; + imageCopy.srcOffset.x = 0; + imageCopy.srcOffset.y = 0; + imageCopy.srcOffset.z = 0; + imageCopy.dstOffset.x = 0; + imageCopy.dstOffset.y = 0; + imageCopy.dstOffset.z = 0; + imageCopy.extent.width = sizeX; + imageCopy.extent.height = sizeY; + imageCopy.extent.depth = 1; + vkCmdCopyImage( + g_hTemporaryCommandBuffer, + stagingImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + g_hTextureImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, &imageCopy); + + imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + imgMemBarrier.image = g_hTextureImage; + imgMemBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + imgMemBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + + vkCmdPipelineBarrier( + g_hTemporaryCommandBuffer, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + 0, + 0, nullptr, + 0, nullptr, + 1, &imgMemBarrier); + + EndSingleTimeCommands(); + + vmaDestroyImage(g_hAllocator, stagingImage, stagingImageAlloc); + + // Create ImageView + + VkImageViewCreateInfo textureImageViewInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; + textureImageViewInfo.image = g_hTextureImage; + textureImageViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + textureImageViewInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + textureImageViewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + textureImageViewInfo.subresourceRange.baseMipLevel = 0; + textureImageViewInfo.subresourceRange.levelCount = 1; + textureImageViewInfo.subresourceRange.baseArrayLayer = 0; + textureImageViewInfo.subresourceRange.layerCount = 1; + ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &textureImageViewInfo, nullptr, &g_hTextureImageView) ); +} + +struct UniformBufferObject +{ + mat4 ModelViewProj; +}; + +static void RegisterDebugCallbacks() +{ + g_pvkCreateDebugReportCallbackEXT = + reinterpret_cast + (vkGetInstanceProcAddr(g_hVulkanInstance, "vkCreateDebugReportCallbackEXT")); + g_pvkDebugReportMessageEXT = + reinterpret_cast + (vkGetInstanceProcAddr(g_hVulkanInstance, "vkDebugReportMessageEXT")); + g_pvkDestroyDebugReportCallbackEXT = + reinterpret_cast + (vkGetInstanceProcAddr(g_hVulkanInstance, "vkDestroyDebugReportCallbackEXT")); + assert(g_pvkCreateDebugReportCallbackEXT); + assert(g_pvkDebugReportMessageEXT); + assert(g_pvkDestroyDebugReportCallbackEXT); + + VkDebugReportCallbackCreateInfoEXT callbackCreateInfo; + callbackCreateInfo.sType = VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT; + callbackCreateInfo.pNext = nullptr; + callbackCreateInfo.flags = //VK_DEBUG_REPORT_INFORMATION_BIT_EXT | + VK_DEBUG_REPORT_ERROR_BIT_EXT | + VK_DEBUG_REPORT_WARNING_BIT_EXT | + VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT /*| + VK_DEBUG_REPORT_DEBUG_BIT_EXT*/; + callbackCreateInfo.pfnCallback = &MyDebugReportCallback; + callbackCreateInfo.pUserData = nullptr; + + ERR_GUARD_VULKAN( g_pvkCreateDebugReportCallbackEXT(g_hVulkanInstance, &callbackCreateInfo, nullptr, &g_hCallback) ); +} + +static bool IsLayerSupported(const VkLayerProperties* pProps, size_t propCount, const char* pLayerName) +{ + const VkLayerProperties* propsEnd = pProps + propCount; + return std::find_if( + pProps, + propsEnd, + [pLayerName](const VkLayerProperties& prop) -> bool { + return strcmp(pLayerName, prop.layerName) == 0; + }) != propsEnd; +} + +static VkFormat FindSupportedFormat( + const std::vector& candidates, + VkImageTiling tiling, + VkFormatFeatureFlags features) +{ + for (VkFormat format : candidates) + { + VkFormatProperties props; + vkGetPhysicalDeviceFormatProperties(g_hPhysicalDevice, format, &props); + + if ((tiling == VK_IMAGE_TILING_LINEAR) && + ((props.linearTilingFeatures & features) == features)) + { + return format; + } + else if ((tiling == VK_IMAGE_TILING_OPTIMAL) && + ((props.optimalTilingFeatures & features) == features)) + { + return format; + } + } + return VK_FORMAT_UNDEFINED; +} + +static VkFormat FindDepthFormat() +{ + std::vector formats; + formats.push_back(VK_FORMAT_D32_SFLOAT); + formats.push_back(VK_FORMAT_D32_SFLOAT_S8_UINT); + formats.push_back(VK_FORMAT_D24_UNORM_S8_UINT); + + return FindSupportedFormat( + formats, + VK_IMAGE_TILING_OPTIMAL, + VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT); +} + +static void CreateSwapchain() +{ + // Query surface formats. + + ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfaceCapabilitiesKHR(g_hPhysicalDevice, g_hSurface, &g_SurfaceCapabilities) ); + + uint32_t formatCount = 0; + ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfaceFormatsKHR(g_hPhysicalDevice, g_hSurface, &formatCount, nullptr) ); + g_SurfaceFormats.resize(formatCount); + ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfaceFormatsKHR(g_hPhysicalDevice, g_hSurface, &formatCount, g_SurfaceFormats.data()) ); + + uint32_t presentModeCount = 0; + ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfacePresentModesKHR(g_hPhysicalDevice, g_hSurface, &presentModeCount, nullptr) ); + g_PresentModes.resize(presentModeCount); + ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfacePresentModesKHR(g_hPhysicalDevice, g_hSurface, &presentModeCount, g_PresentModes.data()) ); + + // Create swap chain + + g_SurfaceFormat = ChooseSurfaceFormat(); + VkPresentModeKHR presentMode = ChooseSwapPresentMode(); + g_Extent = ChooseSwapExtent(); + + uint32_t imageCount = g_SurfaceCapabilities.minImageCount + 1; + if((g_SurfaceCapabilities.maxImageCount > 0) && + (imageCount > g_SurfaceCapabilities.maxImageCount)) + { + imageCount = g_SurfaceCapabilities.maxImageCount; + } + + VkSwapchainCreateInfoKHR swapChainInfo = { VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR }; + swapChainInfo.surface = g_hSurface; + swapChainInfo.minImageCount = imageCount; + swapChainInfo.imageFormat = g_SurfaceFormat.format; + swapChainInfo.imageColorSpace = g_SurfaceFormat.colorSpace; + swapChainInfo.imageExtent = g_Extent; + swapChainInfo.imageArrayLayers = 1; + swapChainInfo.imageUsage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + swapChainInfo.preTransform = g_SurfaceCapabilities.currentTransform; + swapChainInfo.compositeAlpha = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR; + swapChainInfo.presentMode = presentMode; + swapChainInfo.clipped = VK_TRUE; + swapChainInfo.oldSwapchain = g_hSwapchain; + + uint32_t queueFamilyIndices[] = { g_GraphicsQueueFamilyIndex, g_PresentQueueFamilyIndex }; + if(g_PresentQueueFamilyIndex != g_GraphicsQueueFamilyIndex) + { + swapChainInfo.imageSharingMode = VK_SHARING_MODE_CONCURRENT; + swapChainInfo.queueFamilyIndexCount = 2; + swapChainInfo.pQueueFamilyIndices = queueFamilyIndices; + } + else + { + swapChainInfo.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE; + } + + VkSwapchainKHR hNewSwapchain = VK_NULL_HANDLE; + ERR_GUARD_VULKAN( vkCreateSwapchainKHR(g_hDevice, &swapChainInfo, nullptr, &hNewSwapchain) ); + if(g_hSwapchain != VK_NULL_HANDLE) + vkDestroySwapchainKHR(g_hDevice, g_hSwapchain, nullptr); + g_hSwapchain = hNewSwapchain; + + // Retrieve swapchain images. + + uint32_t swapchainImageCount = 0; + ERR_GUARD_VULKAN( vkGetSwapchainImagesKHR(g_hDevice, g_hSwapchain, &swapchainImageCount, nullptr) ); + g_SwapchainImages.resize(swapchainImageCount); + ERR_GUARD_VULKAN( vkGetSwapchainImagesKHR(g_hDevice, g_hSwapchain, &swapchainImageCount, g_SwapchainImages.data()) ); + + // Create swapchain image views. + + for(size_t i = g_SwapchainImageViews.size(); i--; ) + vkDestroyImageView(g_hDevice, g_SwapchainImageViews[i], nullptr); + g_SwapchainImageViews.clear(); + + VkImageViewCreateInfo swapchainImageViewInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; + g_SwapchainImageViews.resize(swapchainImageCount); + for(uint32_t i = 0; i < swapchainImageCount; ++i) + { + swapchainImageViewInfo.image = g_SwapchainImages[i]; + swapchainImageViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + swapchainImageViewInfo.format = g_SurfaceFormat.format; + swapchainImageViewInfo.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; + swapchainImageViewInfo.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; + swapchainImageViewInfo.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; + swapchainImageViewInfo.components.a = VK_COMPONENT_SWIZZLE_IDENTITY; + swapchainImageViewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + swapchainImageViewInfo.subresourceRange.baseMipLevel = 0; + swapchainImageViewInfo.subresourceRange.levelCount = 1; + swapchainImageViewInfo.subresourceRange.baseArrayLayer = 0; + swapchainImageViewInfo.subresourceRange.layerCount = 1; + ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &swapchainImageViewInfo, nullptr, &g_SwapchainImageViews[i]) ); + } + + // Create depth buffer + + g_DepthFormat = FindDepthFormat(); + assert(g_DepthFormat != VK_FORMAT_UNDEFINED); + + VkImageCreateInfo depthImageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + depthImageInfo.imageType = VK_IMAGE_TYPE_2D; + depthImageInfo.extent.width = g_Extent.width; + depthImageInfo.extent.height = g_Extent.height; + depthImageInfo.extent.depth = 1; + depthImageInfo.mipLevels = 1; + depthImageInfo.arrayLayers = 1; + depthImageInfo.format = g_DepthFormat; + depthImageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + depthImageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + depthImageInfo.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT; + depthImageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + depthImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + depthImageInfo.flags = 0; + + VmaAllocationCreateInfo depthImageAllocCreateInfo = {}; + depthImageAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + + ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &depthImageInfo, &depthImageAllocCreateInfo, &g_hDepthImage, &g_hDepthImageAlloc, nullptr) ); + + VkImageViewCreateInfo depthImageViewInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; + depthImageViewInfo.image = g_hDepthImage; + depthImageViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + depthImageViewInfo.format = g_DepthFormat; + depthImageViewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT; + depthImageViewInfo.subresourceRange.baseMipLevel = 0; + depthImageViewInfo.subresourceRange.levelCount = 1; + depthImageViewInfo.subresourceRange.baseArrayLayer = 0; + depthImageViewInfo.subresourceRange.layerCount = 1; + + ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &depthImageViewInfo, nullptr, &g_hDepthImageView) ); + + // Create pipeline layout + { + if(g_hPipelineLayout != VK_NULL_HANDLE) + { + vkDestroyPipelineLayout(g_hDevice, g_hPipelineLayout, nullptr); + g_hPipelineLayout = VK_NULL_HANDLE; + } + + VkPushConstantRange pushConstantRanges[1]; + ZeroMemory(&pushConstantRanges, sizeof pushConstantRanges); + pushConstantRanges[0].offset = 0; + pushConstantRanges[0].size = sizeof(UniformBufferObject); + pushConstantRanges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT; + + VkDescriptorSetLayout descriptorSetLayouts[] = { g_hDescriptorSetLayout }; + VkPipelineLayoutCreateInfo pipelineLayoutInfo = { VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO }; + pipelineLayoutInfo.setLayoutCount = 1; + pipelineLayoutInfo.pSetLayouts = descriptorSetLayouts; + pipelineLayoutInfo.pushConstantRangeCount = 1; + pipelineLayoutInfo.pPushConstantRanges = pushConstantRanges; + ERR_GUARD_VULKAN( vkCreatePipelineLayout(g_hDevice, &pipelineLayoutInfo, nullptr, &g_hPipelineLayout) ); + } + + // Create render pass + { + if(g_hRenderPass != VK_NULL_HANDLE) + { + vkDestroyRenderPass(g_hDevice, g_hRenderPass, nullptr); + g_hRenderPass = VK_NULL_HANDLE; + } + + VkAttachmentDescription attachments[2]; + ZeroMemory(attachments, sizeof(attachments)); + + attachments[0].format = g_SurfaceFormat.format; + attachments[0].samples = VK_SAMPLE_COUNT_1_BIT; + attachments[0].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; + attachments[0].storeOp = VK_ATTACHMENT_STORE_OP_STORE; + attachments[0].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachments[0].stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachments[0].initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + attachments[0].finalLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + + attachments[1].format = g_DepthFormat; + attachments[1].samples = VK_SAMPLE_COUNT_1_BIT; + attachments[1].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; + attachments[1].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachments[1].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachments[1].stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachments[1].initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + attachments[1].finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + + VkAttachmentReference colorAttachmentRef = {}; + colorAttachmentRef.attachment = 0; + colorAttachmentRef.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + + VkAttachmentReference depthStencilAttachmentRef = {}; + depthStencilAttachmentRef.attachment = 1; + depthStencilAttachmentRef.layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + + VkSubpassDescription subpassDesc = {}; + subpassDesc.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpassDesc.colorAttachmentCount = 1; + subpassDesc.pColorAttachments = &colorAttachmentRef; + subpassDesc.pDepthStencilAttachment = &depthStencilAttachmentRef; + + VkRenderPassCreateInfo renderPassInfo = { VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO }; + renderPassInfo.attachmentCount = (uint32_t)_countof(attachments); + renderPassInfo.pAttachments = attachments; + renderPassInfo.subpassCount = 1; + renderPassInfo.pSubpasses = &subpassDesc; + renderPassInfo.dependencyCount = 0; + ERR_GUARD_VULKAN( vkCreateRenderPass(g_hDevice, &renderPassInfo, nullptr, &g_hRenderPass) ); + } + + // Create pipeline + { + std::vector vertShaderCode; + LoadShader(vertShaderCode, "Shader.vert.spv"); + VkShaderModuleCreateInfo shaderModuleInfo = { VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO }; + shaderModuleInfo.codeSize = vertShaderCode.size(); + shaderModuleInfo.pCode = (const uint32_t*)vertShaderCode.data(); + VkShaderModule hVertShaderModule = VK_NULL_HANDLE; + ERR_GUARD_VULKAN( vkCreateShaderModule(g_hDevice, &shaderModuleInfo, nullptr, &hVertShaderModule) ); + + std::vector hFragShaderCode; + LoadShader(hFragShaderCode, "Shader.frag.spv"); + shaderModuleInfo.codeSize = hFragShaderCode.size(); + shaderModuleInfo.pCode = (const uint32_t*)hFragShaderCode.data(); + VkShaderModule fragShaderModule = VK_NULL_HANDLE; + ERR_GUARD_VULKAN( vkCreateShaderModule(g_hDevice, &shaderModuleInfo, nullptr, &fragShaderModule) ); + + VkPipelineShaderStageCreateInfo vertPipelineShaderStageInfo = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO }; + vertPipelineShaderStageInfo.stage = VK_SHADER_STAGE_VERTEX_BIT; + vertPipelineShaderStageInfo.module = hVertShaderModule; + vertPipelineShaderStageInfo.pName = "main"; + + VkPipelineShaderStageCreateInfo fragPipelineShaderStageInfo = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO }; + fragPipelineShaderStageInfo.stage = VK_SHADER_STAGE_FRAGMENT_BIT; + fragPipelineShaderStageInfo.module = fragShaderModule; + fragPipelineShaderStageInfo.pName = "main"; + + VkPipelineShaderStageCreateInfo pipelineShaderStageInfos[] = { + vertPipelineShaderStageInfo, + fragPipelineShaderStageInfo + }; + + VkVertexInputBindingDescription bindingDescription = {}; + bindingDescription.binding = 0; + bindingDescription.stride = sizeof(Vertex); + bindingDescription.inputRate = VK_VERTEX_INPUT_RATE_VERTEX; + + VkVertexInputAttributeDescription attributeDescriptions[3]; + ZeroMemory(attributeDescriptions, sizeof(attributeDescriptions)); + + attributeDescriptions[0].binding = 0; + attributeDescriptions[0].location = 0; + attributeDescriptions[0].format = VK_FORMAT_R32G32B32_SFLOAT; + attributeDescriptions[0].offset = offsetof(Vertex, pos); + + attributeDescriptions[1].binding = 0; + attributeDescriptions[1].location = 1; + attributeDescriptions[1].format = VK_FORMAT_R32G32B32_SFLOAT; + attributeDescriptions[1].offset = offsetof(Vertex, color); + + attributeDescriptions[2].binding = 0; + attributeDescriptions[2].location = 2; + attributeDescriptions[2].format = VK_FORMAT_R32G32_SFLOAT; + attributeDescriptions[2].offset = offsetof(Vertex, texCoord); + + VkPipelineVertexInputStateCreateInfo pipelineVertexInputStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO }; + pipelineVertexInputStateInfo.vertexBindingDescriptionCount = 1; + pipelineVertexInputStateInfo.pVertexBindingDescriptions = &bindingDescription; + pipelineVertexInputStateInfo.vertexAttributeDescriptionCount = _countof(attributeDescriptions); + pipelineVertexInputStateInfo.pVertexAttributeDescriptions = attributeDescriptions; + + VkPipelineInputAssemblyStateCreateInfo pipelineInputAssemblyStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO }; + pipelineInputAssemblyStateInfo.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; + pipelineInputAssemblyStateInfo.primitiveRestartEnable = VK_TRUE; + + VkViewport viewport = {}; + viewport.x = 0.f; + viewport.y = 0.f; + viewport.width = (float)g_Extent.width; + viewport.height = (float)g_Extent.height; + viewport.minDepth = 0.f; + viewport.maxDepth = 1.f; + + VkRect2D scissor = {}; + scissor.offset.x = 0; + scissor.offset.y = 0; + scissor.extent = g_Extent; + + VkPipelineViewportStateCreateInfo pipelineViewportStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO }; + pipelineViewportStateInfo.viewportCount = 1; + pipelineViewportStateInfo.pViewports = &viewport; + pipelineViewportStateInfo.scissorCount = 1; + pipelineViewportStateInfo.pScissors = &scissor; + + VkPipelineRasterizationStateCreateInfo pipelineRasterizationStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO }; + pipelineRasterizationStateInfo.depthClampEnable = VK_FALSE; + pipelineRasterizationStateInfo.rasterizerDiscardEnable = VK_FALSE; + pipelineRasterizationStateInfo.polygonMode = VK_POLYGON_MODE_FILL; + pipelineRasterizationStateInfo.lineWidth = 1.f; + pipelineRasterizationStateInfo.cullMode = VK_CULL_MODE_BACK_BIT; + pipelineRasterizationStateInfo.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + pipelineRasterizationStateInfo.depthBiasEnable = VK_FALSE; + pipelineRasterizationStateInfo.depthBiasConstantFactor = 0.f; + pipelineRasterizationStateInfo.depthBiasClamp = 0.f; + pipelineRasterizationStateInfo.depthBiasSlopeFactor = 0.f; + + VkPipelineMultisampleStateCreateInfo pipelineMultisampleStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO }; + pipelineMultisampleStateInfo.sampleShadingEnable = VK_FALSE; + pipelineMultisampleStateInfo.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + pipelineMultisampleStateInfo.minSampleShading = 1.f; + pipelineMultisampleStateInfo.pSampleMask = nullptr; + pipelineMultisampleStateInfo.alphaToCoverageEnable = VK_FALSE; + pipelineMultisampleStateInfo.alphaToOneEnable = VK_FALSE; + + VkPipelineColorBlendAttachmentState pipelineColorBlendAttachmentState = {}; + pipelineColorBlendAttachmentState.colorWriteMask = + VK_COLOR_COMPONENT_R_BIT | + VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | + VK_COLOR_COMPONENT_A_BIT; + pipelineColorBlendAttachmentState.blendEnable = VK_FALSE; + pipelineColorBlendAttachmentState.srcColorBlendFactor = VK_BLEND_FACTOR_ONE; // Optional + pipelineColorBlendAttachmentState.dstColorBlendFactor = VK_BLEND_FACTOR_ZERO; // Optional + pipelineColorBlendAttachmentState.colorBlendOp = VK_BLEND_OP_ADD; // Optional + pipelineColorBlendAttachmentState.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE; // Optional + pipelineColorBlendAttachmentState.dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO; // Optional + pipelineColorBlendAttachmentState.alphaBlendOp = VK_BLEND_OP_ADD; // Optional + + VkPipelineColorBlendStateCreateInfo pipelineColorBlendStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO }; + pipelineColorBlendStateInfo.logicOpEnable = VK_FALSE; + pipelineColorBlendStateInfo.logicOp = VK_LOGIC_OP_COPY; + pipelineColorBlendStateInfo.attachmentCount = 1; + pipelineColorBlendStateInfo.pAttachments = &pipelineColorBlendAttachmentState; + + VkPipelineDepthStencilStateCreateInfo depthStencilStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO }; + depthStencilStateInfo.depthTestEnable = VK_TRUE; + depthStencilStateInfo.depthWriteEnable = VK_TRUE; + depthStencilStateInfo.depthCompareOp = VK_COMPARE_OP_LESS; + depthStencilStateInfo.depthBoundsTestEnable = VK_FALSE; + depthStencilStateInfo.stencilTestEnable = VK_FALSE; + + VkGraphicsPipelineCreateInfo pipelineInfo = { VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO }; + pipelineInfo.stageCount = 2; + pipelineInfo.pStages = pipelineShaderStageInfos; + pipelineInfo.pVertexInputState = &pipelineVertexInputStateInfo; + pipelineInfo.pInputAssemblyState = &pipelineInputAssemblyStateInfo; + pipelineInfo.pViewportState = &pipelineViewportStateInfo; + pipelineInfo.pRasterizationState = &pipelineRasterizationStateInfo; + pipelineInfo.pMultisampleState = &pipelineMultisampleStateInfo; + pipelineInfo.pDepthStencilState = &depthStencilStateInfo; + pipelineInfo.pColorBlendState = &pipelineColorBlendStateInfo; + pipelineInfo.pDynamicState = nullptr; + pipelineInfo.layout = g_hPipelineLayout; + pipelineInfo.renderPass = g_hRenderPass; + pipelineInfo.subpass = 0; + pipelineInfo.basePipelineHandle = VK_NULL_HANDLE; + pipelineInfo.basePipelineIndex = -1; + ERR_GUARD_VULKAN( vkCreateGraphicsPipelines( + g_hDevice, + VK_NULL_HANDLE, + 1, + &pipelineInfo, nullptr, + &g_hPipeline) ); + + vkDestroyShaderModule(g_hDevice, fragShaderModule, nullptr); + vkDestroyShaderModule(g_hDevice, hVertShaderModule, nullptr); + } + + // Create frambuffers + + for(size_t i = g_Framebuffers.size(); i--; ) + vkDestroyFramebuffer(g_hDevice, g_Framebuffers[i], nullptr); + g_Framebuffers.clear(); + + g_Framebuffers.resize(g_SwapchainImageViews.size()); + for(size_t i = 0; i < g_SwapchainImages.size(); ++i) + { + VkImageView attachments[] = { g_SwapchainImageViews[i], g_hDepthImageView }; + + VkFramebufferCreateInfo framebufferInfo = { VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO }; + framebufferInfo.renderPass = g_hRenderPass; + framebufferInfo.attachmentCount = (uint32_t)_countof(attachments); + framebufferInfo.pAttachments = attachments; + framebufferInfo.width = g_Extent.width; + framebufferInfo.height = g_Extent.height; + framebufferInfo.layers = 1; + ERR_GUARD_VULKAN( vkCreateFramebuffer(g_hDevice, &framebufferInfo, nullptr, &g_Framebuffers[i]) ); + } + + // Create semaphores + + if(g_hImageAvailableSemaphore != VK_NULL_HANDLE) + { + vkDestroySemaphore(g_hDevice, g_hImageAvailableSemaphore, nullptr); + g_hImageAvailableSemaphore = VK_NULL_HANDLE; + } + if(g_hRenderFinishedSemaphore != VK_NULL_HANDLE) + { + vkDestroySemaphore(g_hDevice, g_hRenderFinishedSemaphore, nullptr); + g_hRenderFinishedSemaphore = VK_NULL_HANDLE; + } + + VkSemaphoreCreateInfo semaphoreInfo = { VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO }; + ERR_GUARD_VULKAN( vkCreateSemaphore(g_hDevice, &semaphoreInfo, nullptr, &g_hImageAvailableSemaphore) ); + ERR_GUARD_VULKAN( vkCreateSemaphore(g_hDevice, &semaphoreInfo, nullptr, &g_hRenderFinishedSemaphore) ); +} + +static void DestroySwapchain(bool destroyActualSwapchain) +{ + if(g_hImageAvailableSemaphore != VK_NULL_HANDLE) + { + vkDestroySemaphore(g_hDevice, g_hImageAvailableSemaphore, nullptr); + g_hImageAvailableSemaphore = VK_NULL_HANDLE; + } + if(g_hRenderFinishedSemaphore != VK_NULL_HANDLE) + { + vkDestroySemaphore(g_hDevice, g_hRenderFinishedSemaphore, nullptr); + g_hRenderFinishedSemaphore = VK_NULL_HANDLE; + } + + for(size_t i = g_Framebuffers.size(); i--; ) + vkDestroyFramebuffer(g_hDevice, g_Framebuffers[i], nullptr); + g_Framebuffers.clear(); + + if(g_hDepthImageView != VK_NULL_HANDLE) + { + vkDestroyImageView(g_hDevice, g_hDepthImageView, nullptr); + g_hDepthImageView = VK_NULL_HANDLE; + } + if(g_hDepthImage != VK_NULL_HANDLE) + { + vmaDestroyImage(g_hAllocator, g_hDepthImage, g_hDepthImageAlloc); + g_hDepthImage = VK_NULL_HANDLE; + } + + if(g_hPipeline != VK_NULL_HANDLE) + { + vkDestroyPipeline(g_hDevice, g_hPipeline, nullptr); + g_hPipeline = VK_NULL_HANDLE; + } + + if(g_hRenderPass != VK_NULL_HANDLE) + { + vkDestroyRenderPass(g_hDevice, g_hRenderPass, nullptr); + g_hRenderPass = VK_NULL_HANDLE; + } + + if(g_hPipelineLayout != VK_NULL_HANDLE) + { + vkDestroyPipelineLayout(g_hDevice, g_hPipelineLayout, nullptr); + g_hPipelineLayout = VK_NULL_HANDLE; + } + + for(size_t i = g_SwapchainImageViews.size(); i--; ) + vkDestroyImageView(g_hDevice, g_SwapchainImageViews[i], nullptr); + g_SwapchainImageViews.clear(); + + if(destroyActualSwapchain && (g_hSwapchain != VK_NULL_HANDLE)) + { + vkDestroySwapchainKHR(g_hDevice, g_hSwapchain, nullptr); + g_hSwapchain = VK_NULL_HANDLE; + } +} + +static void InitializeApplication() +{ + uint32_t instanceLayerPropCount = 0; + ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, nullptr) ); + std::vector instanceLayerProps(instanceLayerPropCount); + if(instanceLayerPropCount > 0) + { + ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, instanceLayerProps.data()) ); + } + + if(g_EnableValidationLayer == true) + { + if(IsLayerSupported(instanceLayerProps.data(), instanceLayerProps.size(), VALIDATION_LAYER_NAME) == false) + { + printf("Layer \"%s\" not supported.", VALIDATION_LAYER_NAME); + g_EnableValidationLayer = false; + } + } + + std::vector instanceExtensions; + instanceExtensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME); + instanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); + + std::vector instanceLayers; + if(g_EnableValidationLayer == true) + { + instanceLayers.push_back(VALIDATION_LAYER_NAME); + instanceExtensions.push_back("VK_EXT_debug_report"); + } + + VkApplicationInfo appInfo = { VK_STRUCTURE_TYPE_APPLICATION_INFO }; + appInfo.pApplicationName = APP_TITLE_A; + appInfo.applicationVersion = VK_MAKE_VERSION(1, 0, 0); + appInfo.pEngineName = "Adam Sawicki Engine"; + appInfo.engineVersion = VK_MAKE_VERSION(1, 0, 0); + appInfo.apiVersion = VK_API_VERSION_1_0; + + VkInstanceCreateInfo instInfo = { VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO }; + instInfo.pApplicationInfo = &appInfo; + instInfo.enabledExtensionCount = static_cast(instanceExtensions.size()); + instInfo.ppEnabledExtensionNames = instanceExtensions.data(); + instInfo.enabledLayerCount = static_cast(instanceLayers.size()); + instInfo.ppEnabledLayerNames = instanceLayers.data(); + + ERR_GUARD_VULKAN( vkCreateInstance(&instInfo, NULL, &g_hVulkanInstance) ); + + // Create VkSurfaceKHR. + VkWin32SurfaceCreateInfoKHR surfaceInfo = { VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR }; + surfaceInfo.hinstance = g_hAppInstance; + surfaceInfo.hwnd = g_hWnd; + VkResult result = vkCreateWin32SurfaceKHR(g_hVulkanInstance, &surfaceInfo, NULL, &g_hSurface); + assert(result == VK_SUCCESS); + + if(g_EnableValidationLayer == true) + RegisterDebugCallbacks(); + + // Find physical device + + uint32_t deviceCount = 0; + ERR_GUARD_VULKAN( vkEnumeratePhysicalDevices(g_hVulkanInstance, &deviceCount, nullptr) ); + assert(deviceCount > 0); + + std::vector physicalDevices(deviceCount); + ERR_GUARD_VULKAN( vkEnumeratePhysicalDevices(g_hVulkanInstance, &deviceCount, physicalDevices.data()) ); + + g_hPhysicalDevice = physicalDevices[0]; + + // Query for features + + VkPhysicalDeviceProperties physicalDeviceProperties = {}; + vkGetPhysicalDeviceProperties(g_hPhysicalDevice, &physicalDeviceProperties); + + //VkPhysicalDeviceFeatures physicalDeviceFreatures = {}; + //vkGetPhysicalDeviceFeatures(g_PhysicalDevice, &physicalDeviceFreatures); + + // Find queue family index + + uint32_t queueFamilyCount = 0; + vkGetPhysicalDeviceQueueFamilyProperties(g_hPhysicalDevice, &queueFamilyCount, nullptr); + assert(queueFamilyCount > 0); + std::vector queueFamilies(queueFamilyCount); + vkGetPhysicalDeviceQueueFamilyProperties(g_hPhysicalDevice, &queueFamilyCount, queueFamilies.data()); + for(uint32_t i = 0; + (i < queueFamilyCount) && + (g_GraphicsQueueFamilyIndex == UINT_MAX || g_PresentQueueFamilyIndex == UINT_MAX); + ++i) + { + if(queueFamilies[i].queueCount > 0) + { + if((g_GraphicsQueueFamilyIndex != 0) && + ((queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0)) + { + g_GraphicsQueueFamilyIndex = i; + } + + VkBool32 surfaceSupported = 0; + VkResult res = vkGetPhysicalDeviceSurfaceSupportKHR(g_hPhysicalDevice, i, g_hSurface, &surfaceSupported); + if((res >= 0) && (surfaceSupported == VK_TRUE)) + { + g_PresentQueueFamilyIndex = i; + } + } + } + assert(g_GraphicsQueueFamilyIndex != UINT_MAX); + + // Create logical device + + const float queuePriority = 1.f; + + VkDeviceQueueCreateInfo deviceQueueCreateInfo[2] = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO }; + deviceQueueCreateInfo[0].queueFamilyIndex = g_GraphicsQueueFamilyIndex; + deviceQueueCreateInfo[0].queueCount = 1; + deviceQueueCreateInfo[0].pQueuePriorities = &queuePriority; + deviceQueueCreateInfo[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + deviceQueueCreateInfo[1].queueFamilyIndex = g_PresentQueueFamilyIndex; + deviceQueueCreateInfo[1].queueCount = 1; + deviceQueueCreateInfo[1].pQueuePriorities = &queuePriority; + + VkPhysicalDeviceFeatures deviceFeatures = {}; + deviceFeatures.fillModeNonSolid = VK_TRUE; + deviceFeatures.samplerAnisotropy = VK_TRUE; + + // Determine list of device extensions to enable. + std::vector enabledDeviceExtensions; + enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); + { + uint32_t propertyCount = 0; + ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(g_hPhysicalDevice, nullptr, &propertyCount, nullptr) ); + + if(propertyCount) + { + std::vector properties{propertyCount}; + ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(g_hPhysicalDevice, nullptr, &propertyCount, properties.data()) ); + + for(uint32_t i = 0; i < propertyCount; ++i) + { + if(strcmp(properties[i].extensionName, VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME) == 0) + { + enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); + VK_KHR_get_memory_requirements2_enabled = true; + } + else if(strcmp(properties[i].extensionName, VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME) == 0) + { + enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); + VK_KHR_dedicated_allocation_enabled = true; + } + } + } + } + + VkDeviceCreateInfo deviceCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO }; + deviceCreateInfo.enabledLayerCount = 0; + deviceCreateInfo.ppEnabledLayerNames = nullptr; + deviceCreateInfo.enabledExtensionCount = (uint32_t)enabledDeviceExtensions.size(); + deviceCreateInfo.ppEnabledExtensionNames = !enabledDeviceExtensions.empty() ? enabledDeviceExtensions.data() : nullptr; + deviceCreateInfo.queueCreateInfoCount = g_PresentQueueFamilyIndex != g_GraphicsQueueFamilyIndex ? 2 : 1; + deviceCreateInfo.pQueueCreateInfos = deviceQueueCreateInfo; + deviceCreateInfo.pEnabledFeatures = &deviceFeatures; + + ERR_GUARD_VULKAN( vkCreateDevice(g_hPhysicalDevice, &deviceCreateInfo, nullptr, &g_hDevice) ); + + // Create memory allocator + + VmaAllocatorCreateInfo allocatorInfo = {}; + allocatorInfo.physicalDevice = g_hPhysicalDevice; + allocatorInfo.device = g_hDevice; + + if(VK_KHR_dedicated_allocation_enabled) + { + allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; + } + + VkAllocationCallbacks cpuAllocationCallbacks = {}; + if(USE_CUSTOM_CPU_ALLOCATION_CALLBACKS) + { + cpuAllocationCallbacks.pUserData = CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA; + cpuAllocationCallbacks.pfnAllocation = &CustomCpuAllocation; + cpuAllocationCallbacks.pfnReallocation = &CustomCpuReallocation; + cpuAllocationCallbacks.pfnFree = &CustomCpuFree; + allocatorInfo.pAllocationCallbacks = &cpuAllocationCallbacks; + } + + ERR_GUARD_VULKAN( vmaCreateAllocator(&allocatorInfo, &g_hAllocator) ); + + // Retrieve queue (doesn't need to be destroyed) + + vkGetDeviceQueue(g_hDevice, g_GraphicsQueueFamilyIndex, 0, &g_hGraphicsQueue); + vkGetDeviceQueue(g_hDevice, g_PresentQueueFamilyIndex, 0, &g_hPresentQueue); + assert(g_hGraphicsQueue); + assert(g_hPresentQueue); + + // Create command pool + + VkCommandPoolCreateInfo commandPoolInfo = { VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO }; + commandPoolInfo.queueFamilyIndex = g_GraphicsQueueFamilyIndex; + commandPoolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + ERR_GUARD_VULKAN( vkCreateCommandPool(g_hDevice, &commandPoolInfo, nullptr, &g_hCommandPool) ); + + VkCommandBufferAllocateInfo commandBufferInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO }; + commandBufferInfo.commandPool = g_hCommandPool; + commandBufferInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + commandBufferInfo.commandBufferCount = COMMAND_BUFFER_COUNT; + ERR_GUARD_VULKAN( vkAllocateCommandBuffers(g_hDevice, &commandBufferInfo, g_MainCommandBuffers) ); + + VkFenceCreateInfo fenceInfo = { VK_STRUCTURE_TYPE_FENCE_CREATE_INFO }; + fenceInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT; + for(size_t i = 0; i < COMMAND_BUFFER_COUNT; ++i) + { + ERR_GUARD_VULKAN( vkCreateFence(g_hDevice, &fenceInfo, nullptr, &g_MainCommandBufferExecutedFances[i]) ); + } + + commandBufferInfo.commandBufferCount = 1; + ERR_GUARD_VULKAN( vkAllocateCommandBuffers(g_hDevice, &commandBufferInfo, &g_hTemporaryCommandBuffer) ); + + // Create texture sampler + + VkSamplerCreateInfo samplerInfo = { VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO }; + samplerInfo.magFilter = VK_FILTER_LINEAR; + samplerInfo.minFilter = VK_FILTER_LINEAR; + samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_REPEAT; + samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_REPEAT; + samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT; + samplerInfo.anisotropyEnable = VK_TRUE; + samplerInfo.maxAnisotropy = 16; + samplerInfo.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK; + samplerInfo.unnormalizedCoordinates = VK_FALSE; + samplerInfo.compareEnable = VK_FALSE; + samplerInfo.compareOp = VK_COMPARE_OP_ALWAYS; + samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR; + samplerInfo.mipLodBias = 0.f; + samplerInfo.minLod = 0.f; + samplerInfo.maxLod = FLT_MAX; + ERR_GUARD_VULKAN( vkCreateSampler(g_hDevice, &samplerInfo, nullptr, &g_hSampler) ); + + CreateTexture(128, 128); + CreateMesh(); + + VkDescriptorSetLayoutBinding samplerLayoutBinding = {}; + samplerLayoutBinding.binding = 1; + samplerLayoutBinding.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + samplerLayoutBinding.descriptorCount = 1; + samplerLayoutBinding.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; + + VkDescriptorSetLayoutCreateInfo descriptorSetLayoutInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO }; + descriptorSetLayoutInfo.bindingCount = 1; + descriptorSetLayoutInfo.pBindings = &samplerLayoutBinding; + ERR_GUARD_VULKAN( vkCreateDescriptorSetLayout(g_hDevice, &descriptorSetLayoutInfo, nullptr, &g_hDescriptorSetLayout) ); + + // Create descriptor pool + + VkDescriptorPoolSize descriptorPoolSizes[2]; + ZeroMemory(descriptorPoolSizes, sizeof(descriptorPoolSizes)); + descriptorPoolSizes[0].type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; + descriptorPoolSizes[0].descriptorCount = 1; + descriptorPoolSizes[1].type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + descriptorPoolSizes[1].descriptorCount = 1; + + VkDescriptorPoolCreateInfo descriptorPoolInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO }; + descriptorPoolInfo.poolSizeCount = (uint32_t)_countof(descriptorPoolSizes); + descriptorPoolInfo.pPoolSizes = descriptorPoolSizes; + descriptorPoolInfo.maxSets = 1; + ERR_GUARD_VULKAN( vkCreateDescriptorPool(g_hDevice, &descriptorPoolInfo, nullptr, &g_hDescriptorPool) ); + + // Create descriptor set layout + + VkDescriptorSetLayout descriptorSetLayouts[] = { g_hDescriptorSetLayout }; + VkDescriptorSetAllocateInfo descriptorSetInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO }; + descriptorSetInfo.descriptorPool = g_hDescriptorPool; + descriptorSetInfo.descriptorSetCount = 1; + descriptorSetInfo.pSetLayouts = descriptorSetLayouts; + ERR_GUARD_VULKAN( vkAllocateDescriptorSets(g_hDevice, &descriptorSetInfo, &g_hDescriptorSet) ); + + VkDescriptorImageInfo descriptorImageInfo = {}; + descriptorImageInfo.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + descriptorImageInfo.imageView = g_hTextureImageView; + descriptorImageInfo.sampler = g_hSampler; + + VkWriteDescriptorSet writeDescriptorSet = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET }; + writeDescriptorSet.dstSet = g_hDescriptorSet; + writeDescriptorSet.dstBinding = 1; + writeDescriptorSet.dstArrayElement = 0; + writeDescriptorSet.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + writeDescriptorSet.descriptorCount = 1; + writeDescriptorSet.pImageInfo = &descriptorImageInfo; + + vkUpdateDescriptorSets(g_hDevice, 1, &writeDescriptorSet, 0, nullptr); + + CreateSwapchain(); +} + +static void FinalizeApplication() +{ + vkDeviceWaitIdle(g_hDevice); + + DestroySwapchain(true); + + if(g_hDescriptorPool != VK_NULL_HANDLE) + { + vkDestroyDescriptorPool(g_hDevice, g_hDescriptorPool, nullptr); + g_hDescriptorPool = VK_NULL_HANDLE; + } + + if(g_hDescriptorSetLayout != VK_NULL_HANDLE) + { + vkDestroyDescriptorSetLayout(g_hDevice, g_hDescriptorSetLayout, nullptr); + g_hDescriptorSetLayout = VK_NULL_HANDLE; + } + + if(g_hTextureImageView != VK_NULL_HANDLE) + { + vkDestroyImageView(g_hDevice, g_hTextureImageView, nullptr); + g_hTextureImageView = VK_NULL_HANDLE; + } + if(g_hTextureImage != VK_NULL_HANDLE) + { + vmaDestroyImage(g_hAllocator, g_hTextureImage, g_hTextureImageAlloc); + g_hTextureImage = VK_NULL_HANDLE; + } + + if(g_hIndexBuffer != VK_NULL_HANDLE) + { + vmaDestroyBuffer(g_hAllocator, g_hIndexBuffer, g_hIndexBufferAlloc); + g_hIndexBuffer = VK_NULL_HANDLE; + } + if(g_hVertexBuffer != VK_NULL_HANDLE) + { + vmaDestroyBuffer(g_hAllocator, g_hVertexBuffer, g_hVertexBufferAlloc); + g_hVertexBuffer = VK_NULL_HANDLE; + } + + if(g_hSampler != VK_NULL_HANDLE) + { + vkDestroySampler(g_hDevice, g_hSampler, nullptr); + g_hSampler = VK_NULL_HANDLE; + } + + for(size_t i = COMMAND_BUFFER_COUNT; i--; ) + { + if(g_MainCommandBufferExecutedFances[i] != VK_NULL_HANDLE) + { + vkDestroyFence(g_hDevice, g_MainCommandBufferExecutedFances[i], nullptr); + g_MainCommandBufferExecutedFances[i] = VK_NULL_HANDLE; + } + } + if(g_MainCommandBuffers[0] != VK_NULL_HANDLE) + { + vkFreeCommandBuffers(g_hDevice, g_hCommandPool, COMMAND_BUFFER_COUNT, g_MainCommandBuffers); + ZeroMemory(g_MainCommandBuffers, sizeof(g_MainCommandBuffers)); + } + if(g_hTemporaryCommandBuffer != VK_NULL_HANDLE) + { + vkFreeCommandBuffers(g_hDevice, g_hCommandPool, 1, &g_hTemporaryCommandBuffer); + g_hTemporaryCommandBuffer = VK_NULL_HANDLE; + } + + if(g_hCommandPool != VK_NULL_HANDLE) + { + vkDestroyCommandPool(g_hDevice, g_hCommandPool, nullptr); + g_hCommandPool = VK_NULL_HANDLE; + } + + if(g_hAllocator != VK_NULL_HANDLE) + { + vmaDestroyAllocator(g_hAllocator); + g_hAllocator = nullptr; + } + + if(g_hDevice != VK_NULL_HANDLE) + { + vkDestroyDevice(g_hDevice, nullptr); + g_hDevice = nullptr; + } + + if(g_pvkDestroyDebugReportCallbackEXT && g_hCallback != VK_NULL_HANDLE) + { + g_pvkDestroyDebugReportCallbackEXT(g_hVulkanInstance, g_hCallback, nullptr); + g_hCallback = VK_NULL_HANDLE; + } + + if(g_hSurface != VK_NULL_HANDLE) + { + vkDestroySurfaceKHR(g_hVulkanInstance, g_hSurface, NULL); + g_hSurface = VK_NULL_HANDLE; + } + + if(g_hVulkanInstance != VK_NULL_HANDLE) + { + vkDestroyInstance(g_hVulkanInstance, NULL); + g_hVulkanInstance = VK_NULL_HANDLE; + } +} + +static void PrintAllocatorStats() +{ +#if VMA_STATS_STRING_ENABLED + char* statsString = nullptr; + vmaBuildStatsString(g_hAllocator, &statsString, true); + printf("%s\n", statsString); + vmaFreeStatsString(g_hAllocator, statsString); +#endif +} + +static void RecreateSwapChain() +{ + vkDeviceWaitIdle(g_hDevice); + DestroySwapchain(false); + CreateSwapchain(); +} + +static void DrawFrame() +{ + // Begin main command buffer + size_t cmdBufIndex = (g_NextCommandBufferIndex++) % COMMAND_BUFFER_COUNT; + VkCommandBuffer hCommandBuffer = g_MainCommandBuffers[cmdBufIndex]; + VkFence hCommandBufferExecutedFence = g_MainCommandBufferExecutedFances[cmdBufIndex]; + + ERR_GUARD_VULKAN( vkWaitForFences(g_hDevice, 1, &hCommandBufferExecutedFence, VK_TRUE, UINT64_MAX) ); + ERR_GUARD_VULKAN( vkResetFences(g_hDevice, 1, &hCommandBufferExecutedFence) ); + + VkCommandBufferBeginInfo commandBufferBeginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + commandBufferBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + ERR_GUARD_VULKAN( vkBeginCommandBuffer(hCommandBuffer, &commandBufferBeginInfo) ); + + // Acquire swapchain image + uint32_t imageIndex = 0; + VkResult res = vkAcquireNextImageKHR(g_hDevice, g_hSwapchain, UINT64_MAX, g_hImageAvailableSemaphore, VK_NULL_HANDLE, &imageIndex); + if(res == VK_ERROR_OUT_OF_DATE_KHR) + { + RecreateSwapChain(); + return; + } + else if(res < 0) + { + ERR_GUARD_VULKAN(res); + } + + // Record geometry pass + + VkClearValue clearValues[2]; + ZeroMemory(clearValues, sizeof(clearValues)); + clearValues[0].color.float32[0] = 0.25f; + clearValues[0].color.float32[1] = 0.25f; + clearValues[0].color.float32[2] = 0.5f; + clearValues[0].color.float32[3] = 1.0f; + clearValues[1].depthStencil.depth = 1.0f; + + VkRenderPassBeginInfo renderPassBeginInfo = { VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO }; + renderPassBeginInfo.renderPass = g_hRenderPass; + renderPassBeginInfo.framebuffer = g_Framebuffers[imageIndex]; + renderPassBeginInfo.renderArea.offset.x = 0; + renderPassBeginInfo.renderArea.offset.y = 0; + renderPassBeginInfo.renderArea.extent = g_Extent; + renderPassBeginInfo.clearValueCount = (uint32_t)_countof(clearValues); + renderPassBeginInfo.pClearValues = clearValues; + vkCmdBeginRenderPass(hCommandBuffer, &renderPassBeginInfo, VK_SUBPASS_CONTENTS_INLINE); + + vkCmdBindPipeline( + hCommandBuffer, + VK_PIPELINE_BIND_POINT_GRAPHICS, + g_hPipeline); + + mat4 view = mat4::LookAt( + vec3(0.f, 0.f, 0.f), + vec3(0.f, -2.f, 4.f), + vec3(0.f, 1.f, 0.f)); + mat4 proj = mat4::Perspective( + 1.0471975511966f, // 60 degrees + (float)g_Extent.width / (float)g_Extent.height, + 0.1f, + 1000.f); + mat4 viewProj = view * proj; + + vkCmdBindDescriptorSets( + hCommandBuffer, + VK_PIPELINE_BIND_POINT_GRAPHICS, + g_hPipelineLayout, + 0, + 1, + &g_hDescriptorSet, + 0, + nullptr); + + float rotationAngle = (float)GetTickCount() * 0.001f * (float)PI * 0.2f; + mat4 model = mat4::RotationY(rotationAngle); + + UniformBufferObject ubo = {}; + ubo.ModelViewProj = model * viewProj; + vkCmdPushConstants(hCommandBuffer, g_hPipelineLayout, VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(UniformBufferObject), &ubo); + + VkBuffer vertexBuffers[] = { g_hVertexBuffer }; + VkDeviceSize offsets[] = { 0 }; + vkCmdBindVertexBuffers(hCommandBuffer, 0, 1, vertexBuffers, offsets); + + vkCmdBindIndexBuffer(hCommandBuffer, g_hIndexBuffer, 0, VK_INDEX_TYPE_UINT16); + + vkCmdDrawIndexed(hCommandBuffer, g_IndexCount, 1, 0, 0, 0); + + vkCmdEndRenderPass(hCommandBuffer); + + vkEndCommandBuffer(hCommandBuffer); + + // Submit command buffer + + VkSemaphore submitWaitSemaphores[] = { g_hImageAvailableSemaphore }; + VkPipelineStageFlags submitWaitStages[] = { VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT }; + VkSemaphore submitSignalSemaphores[] = { g_hRenderFinishedSemaphore }; + VkSubmitInfo submitInfo = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; + submitInfo.waitSemaphoreCount = 1; + submitInfo.pWaitSemaphores = submitWaitSemaphores; + submitInfo.pWaitDstStageMask = submitWaitStages; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &hCommandBuffer; + submitInfo.signalSemaphoreCount = _countof(submitSignalSemaphores); + submitInfo.pSignalSemaphores = submitSignalSemaphores; + ERR_GUARD_VULKAN( vkQueueSubmit(g_hGraphicsQueue, 1, &submitInfo, hCommandBufferExecutedFence) ); + + VkSemaphore presentWaitSemaphores[] = { g_hRenderFinishedSemaphore }; + + VkSwapchainKHR swapchains[] = { g_hSwapchain }; + VkPresentInfoKHR presentInfo = { VK_STRUCTURE_TYPE_PRESENT_INFO_KHR }; + presentInfo.waitSemaphoreCount = _countof(presentWaitSemaphores); + presentInfo.pWaitSemaphores = presentWaitSemaphores; + presentInfo.swapchainCount = 1; + presentInfo.pSwapchains = swapchains; + presentInfo.pImageIndices = &imageIndex; + presentInfo.pResults = nullptr; + res = vkQueuePresentKHR(g_hPresentQueue, &presentInfo); + if(res == VK_ERROR_OUT_OF_DATE_KHR) + { + RecreateSwapChain(); + } + else + ERR_GUARD_VULKAN(res); +} + +static void HandlePossibleSizeChange() +{ + RECT clientRect; + GetClientRect(g_hWnd, &clientRect); + LONG newSizeX = clientRect.right - clientRect.left; + LONG newSizeY = clientRect.bottom - clientRect.top; + if((newSizeX > 0) && + (newSizeY > 0) && + ((newSizeX != g_SizeX) || (newSizeY != g_SizeY))) + { + g_SizeX = newSizeX; + g_SizeY = newSizeY; + + RecreateSwapChain(); + } +} + +static LRESULT WINAPI WndProc(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam) +{ + switch(msg) + { + case WM_CREATE: + // This is intentionally assigned here because we are now inside CreateWindow, before it returns. + g_hWnd = hWnd; + InitializeApplication(); + PrintAllocatorStats(); + return 0; + + case WM_DESTROY: + FinalizeApplication(); + PostQuitMessage(0); + return 0; + + // This prevents app from freezing when left Alt is pressed + // (which normally enters modal menu loop). + case WM_SYSKEYDOWN: + case WM_SYSKEYUP: + return 0; + + case WM_SIZE: + if((wParam == SIZE_MAXIMIZED) || (wParam == SIZE_RESTORED)) + HandlePossibleSizeChange(); + return 0; + + case WM_EXITSIZEMOVE: + HandlePossibleSizeChange(); + return 0; + + case WM_KEYDOWN: + switch(wParam) + { + case VK_ESCAPE: + PostMessage(hWnd, WM_CLOSE, 0, 0); + break; + case 'T': + Test(); + break; + } + return 0; + + default: + break; + } + + return DefWindowProc(hWnd, msg, wParam, lParam); +} + +int main() +{ + g_hAppInstance = (HINSTANCE)GetModuleHandle(NULL); + + WNDCLASSEX wndClassDesc = { sizeof(WNDCLASSEX) }; + wndClassDesc.style = CS_VREDRAW | CS_HREDRAW | CS_DBLCLKS; + wndClassDesc.hbrBackground = NULL; + wndClassDesc.hCursor = LoadCursor(NULL, IDC_CROSS); + wndClassDesc.hIcon = LoadIcon(NULL, IDI_APPLICATION); + wndClassDesc.hInstance = g_hAppInstance; + wndClassDesc.lpfnWndProc = WndProc; + wndClassDesc.lpszClassName = WINDOW_CLASS_NAME; + + const ATOM hWndClass = RegisterClassEx(&wndClassDesc); + assert(hWndClass); + + const DWORD style = WS_VISIBLE | WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_MINIMIZEBOX | WS_MAXIMIZEBOX | WS_THICKFRAME; + const DWORD exStyle = 0; + + RECT rect = { 0, 0, g_SizeX, g_SizeY }; + AdjustWindowRectEx(&rect, style, FALSE, exStyle); + + CreateWindowEx( + exStyle, WINDOW_CLASS_NAME, APP_TITLE_W, style, + CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, + NULL, NULL, g_hAppInstance, NULL); + + MSG msg; + for(;;) + { + if(PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) + { + if(msg.message == WM_QUIT) + break; + TranslateMessage(&msg); + DispatchMessage(&msg); + } + if(g_hDevice != VK_NULL_HANDLE) + DrawFrame(); + } + + return 0; +} +#endif + +static const int RESULT_EXCEPTION = -1000; +static const int RESULT_ERROR_COMMAND_LINE = -1; +static const int RESULT_ERROR_SOURCE_FILE = -2; +static const int RESULT_ERROR_FORMAT = -3; + +//////////////////////////////////////////////////////////////////////////////// +// LineSplit class + +class LineSplit +{ +public: + LineSplit(const char* data, size_t numBytes) : + m_Data(data), + m_NumBytes(numBytes), + m_NextLineBeg(0), + m_NextLineIndex(0) + { + } + + bool GetNextLine(const char *&outBeg, const char*&outEnd); + size_t GetNextLineIndex() const { return m_NextLineIndex; } + +private: + const char* const m_Data; + const size_t m_NumBytes; + size_t m_NextLineBeg; + size_t m_NextLineIndex; +}; + +bool LineSplit::GetNextLine(const char *&outBeg, const char*&outEnd) +{ + if(m_NextLineBeg < m_NumBytes) + { + outBeg = m_Data + m_NextLineBeg; + size_t currLineEnd = m_NextLineBeg; + while(currLineEnd < m_NumBytes && m_Data[currLineEnd] != '\n') + ++currLineEnd; + outEnd = m_Data + currLineEnd; + m_NextLineBeg = currLineEnd + 1; // Past '\n' + ++m_NextLineIndex; + return true; + } + else + return false; +} + + +//////////////////////////////////////////////////////////////////////////////// +// CsvSplit class + +class CsvSplit +{ +public: + static const size_t RANGE_COUNT_MAX = 32; + + void Set(const char* beg, const char* end, size_t maxCount = RANGE_COUNT_MAX); + + size_t GetCount() const { return m_Count; } + void GetRange(size_t index, const char*& outBeg, const char*& outEnd) const + { + outBeg = m_Str + m_Ranges[index * 2]; + outEnd = m_Str + m_Ranges[index * 2 + 1]; + } + +private: + const char* m_Str = nullptr; + size_t m_Count = 0; + size_t m_Ranges[RANGE_COUNT_MAX * 2]; // Pairs of begin-end. +}; + +void CsvSplit::Set(const char* beg, const char* end, size_t maxCount) +{ + assert(maxCount <= RANGE_COUNT_MAX); + m_Str = beg; + const size_t strLen = end - beg; + size_t rangeIndex = 0; + size_t charIndex = 0; + while(charIndex < strLen && rangeIndex < maxCount) + { + m_Ranges[rangeIndex * 2] = charIndex; + while(charIndex < strLen && (rangeIndex + 1 == maxCount || m_Str[charIndex] != ',')) + ++charIndex; + m_Ranges[rangeIndex * 2 + 1] = charIndex; + ++rangeIndex; + ++charIndex; // Past ',' + } + m_Count = rangeIndex; +} + +//////////////////////////////////////////////////////////////////////////////// +// class Player + +class Player +{ +public: + Player(); + void Init(); + ~Player(); + + void ExecuteLine(size_t lineNumber, const char* lineBeg, const char* lineEnd); + +private: + static const size_t MAX_WARNINGS_TO_SHOW = 16; + + // Increments warning counter. Returns true if warning message should be printed. + bool IssueWarning() { return m_WarningCount++ < MAX_WARNINGS_TO_SHOW; } + + size_t m_WarningCount = 0; +}; + +Player::Player() +{ +} + +void Player::Init() +{ +} + +Player::~Player() +{ + if(m_WarningCount > 0) + printf("WARNING: %zu more warnings not shown.\n", m_WarningCount - MAX_WARNINGS_TO_SHOW); +} + +void Player::ExecuteLine(size_t lineNumber, const char* lineBeg, const char* lineEnd) +{ + CsvSplit csvSplit; + csvSplit.Set(lineBeg, lineEnd); + + if(csvSplit.GetCount() >= 4) + { + } + else + { + if(IssueWarning()) + printf("Line %zu: Too few columns.\n", lineNumber); + } +} + + +//////////////////////////////////////////////////////////////////////////////// +// Main functions + +static void PrintCommandLineSyntax() +{ + printf("Command line syntax:\n" + " VmaReplay \n"); +} + +static inline bool StrRangeEq(const char* lhsBeg, const char* lhsEnd, const char* rhsSz) +{ + const size_t rhsLen = strlen(rhsSz); + return rhsLen == lhsEnd - lhsBeg && + memcmp(lhsBeg, rhsSz, rhsLen) == 0; +} + +static int ProcessFile(const char* data, size_t numBytes) +{ + // Begin stats. + printf("File size: %zu B\n", numBytes); + + LineSplit lineSplit(data, numBytes); + const char* lineBeg; + const char* lineEnd; + + if(!lineSplit.GetNextLine(lineBeg, lineEnd) || + !StrRangeEq(lineBeg, lineEnd, "Vulkan Memory Allocator,Calls recording")) + { + printf("ERROR: Incorrect file format.\n"); + return RESULT_ERROR_FORMAT; + } + + if(!lineSplit.GetNextLine(lineBeg, lineEnd) || + !StrRangeEq(lineBeg, lineEnd, "1,0")) + { + printf("ERROR: Incorrect file format version.\n"); + return RESULT_ERROR_FORMAT; + } + + Player player; + player.Init(); + + while(lineSplit.GetNextLine(lineBeg, lineEnd)) + { + player.ExecuteLine(lineSplit.GetNextLineIndex(), lineBeg, lineEnd); + } + + // End stats. + printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); + + return 0; +} + +static int ProcessFile(const char* filePath) +{ + printf("Replaying file \"%s\"...\n", filePath); + int result = 0; + + FILE* file = nullptr; + const errno_t err = fopen_s(&file, filePath, "rb"); + if(err == 0) + { + _fseeki64(file, 0, SEEK_END); + const size_t fileSize = (size_t)_ftelli64(file); + _fseeki64(file, 0, SEEK_SET); + + if(fileSize > 0) + { + std::vector fileContents(fileSize); + fread(fileContents.data(), 1, fileSize, file); + ProcessFile(fileContents.data(), fileContents.size()); + } + else + { + printf("ERROR: Source file is empty.\n"); + result = RESULT_ERROR_SOURCE_FILE; + } + + fclose(file); + } + else + { + printf("ERROR: Couldn't open file (%i).\n", err); + result = RESULT_ERROR_SOURCE_FILE; + } + + return result; +} + +static int main2(int argc, char** argv) +{ + if(argc != 2) + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + + return ProcessFile(argv[1]); +} + +int main(int argc, char** argv) +{ + try + { + main2(argc, argv); + } + catch(const std::exception& e) + { + printf("ERROR: %s\n", e.what()); + return RESULT_EXCEPTION; + } + catch(...) + { + printf("UNKNOWN ERROR\n"); + return RESULT_EXCEPTION; + } + + return 0; +} diff --git a/src/VmaReplay/VmaUsage.cpp b/src/VmaReplay/VmaUsage.cpp new file mode 100644 index 0000000..d2d035b --- /dev/null +++ b/src/VmaReplay/VmaUsage.cpp @@ -0,0 +1,2 @@ +#define VMA_IMPLEMENTATION +#include "VmaUsage.h" diff --git a/src/VmaReplay/VmaUsage.h b/src/VmaReplay/VmaUsage.h new file mode 100644 index 0000000..182132b --- /dev/null +++ b/src/VmaReplay/VmaUsage.h @@ -0,0 +1,27 @@ +#pragma once + +#define NOMINMAX +#define WIN32_LEAN_AND_MEAN +#include +#define VK_USE_PLATFORM_WIN32_KHR + +#include + +//#define VMA_USE_STL_CONTAINERS 1 + +//#define VMA_HEAVY_ASSERT(expr) assert(expr) + +//#define VMA_DEDICATED_ALLOCATION 0 + +//#define VMA_DEBUG_MARGIN 16 +//#define VMA_DEBUG_DETECT_CORRUPTION 1 +//#define VMA_DEBUG_INITIALIZE_ALLOCATIONS 1 + +#pragma warning(push, 4) +#pragma warning(disable: 4127) // conditional expression is constant +#pragma warning(disable: 4100) // unreferenced formal parameter +#pragma warning(disable: 4189) // local variable is initialized but not referenced + +#include "../vk_mem_alloc.h" + +#pragma warning(pop) diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 149445e..5dbad64 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -2649,6 +2649,29 @@ static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF; END OF CONFIGURATION */ +// TEMP ADDED + +void Crash() +{ + int* i = 0; *i = 0; +} + +FILE* g_File; +LARGE_INTEGER g_Freq, g_StartCounter; +VMA_MUTEX g_FileMutex; + +void EnsureFile() +{ + if(!g_File) + { + fopen_s(&g_File, "VMA_Usage_Dump", "wb"); + fprintf(g_File, "%s\n", "Vulkan Memory Allocator,Calls recording"); + fprintf(g_File, "%s\n", "1,0"); + QueryPerformanceFrequency(&g_Freq); + QueryPerformanceCounter(&g_StartCounter); + } +} + static VkAllocationCallbacks VmaEmptyAllocationCallbacks = { VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL }; @@ -4785,6 +4808,7 @@ public: void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats); void SetCurrentFrameIndex(uint32_t frameIndex); + uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); } void MakePoolAllocationsLost( VmaPool hPool, @@ -9338,6 +9362,18 @@ VkResult vmaCreateAllocator( VMA_ASSERT(pCreateInfo && pAllocator); VMA_DEBUG_LOG("vmaCreateAllocator"); *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo); + + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = (*pAllocator)->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaCreateAllocator\n", threadId, time, frameIndex); + fflush(g_File); + } + return VK_SUCCESS; } @@ -9346,6 +9382,17 @@ void vmaDestroyAllocator( { if(allocator != VK_NULL_HANDLE) { + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaDestroyAllocator\n", threadId, time, frameIndex); + fflush(g_File); + } + VMA_DEBUG_LOG("vmaDestroyAllocator"); VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks; vma_delete(&allocationCallbacks, allocator); @@ -9688,7 +9735,27 @@ VkResult vmaCreatePool( VMA_DEBUG_GLOBAL_MUTEX_LOCK - return allocator->CreatePool(pCreateInfo, pPool); + VkResult res = allocator->CreatePool(pCreateInfo, pPool); + + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", threadId, time, frameIndex, + pCreateInfo->memoryTypeIndex, + pCreateInfo->flags, + pCreateInfo->blockSize, + pCreateInfo->minBlockCount, + pCreateInfo->maxBlockCount, + pCreateInfo->frameInUseCount, + (*pPool)); + fflush(g_File); + } + + return res; } void vmaDestroyPool( @@ -9702,6 +9769,18 @@ void vmaDestroyPool( return; } + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", threadId, time, frameIndex, + pool); + fflush(g_File); + } + VMA_DEBUG_LOG("vmaDestroyPool"); VMA_DEBUG_GLOBAL_MUTEX_LOCK @@ -9751,6 +9830,7 @@ VkResult vmaAllocateMemory( VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { + Crash(); VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemory"); @@ -9782,6 +9862,7 @@ VkResult vmaAllocateMemoryForBuffer( VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { + Crash(); VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer"); @@ -9820,6 +9901,7 @@ VkResult vmaAllocateMemoryForImage( VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { + Crash(); VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemoryForImage"); @@ -9845,6 +9927,7 @@ void vmaFreeMemory( VmaAllocator allocator, VmaAllocation allocation) { + Crash(); VMA_ASSERT(allocator); VMA_DEBUG_LOG("vmaFreeMemory"); VMA_DEBUG_GLOBAL_MUTEX_LOCK @@ -9886,6 +9969,19 @@ void vmaSetAllocationUserData( VMA_DEBUG_GLOBAL_MUTEX_LOCK + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", threadId, time, frameIndex, + allocation, + (const char*)pUserData); + fflush(g_File); + } + allocation->SetUserData(allocator, pUserData); } @@ -9893,6 +9989,7 @@ void vmaCreateLostAllocation( VmaAllocator allocator, VmaAllocation* pAllocation) { + Crash(); VMA_ASSERT(allocator && pAllocation); VMA_DEBUG_GLOBAL_MUTEX_LOCK; @@ -9978,6 +10075,7 @@ VkResult vmaBindBufferMemory( VmaAllocation allocation, VkBuffer buffer) { + Crash(); VMA_ASSERT(allocator && allocation && buffer); VMA_DEBUG_LOG("vmaBindBufferMemory"); @@ -9992,6 +10090,7 @@ VkResult vmaBindImageMemory( VmaAllocation allocation, VkImage image) { + Crash(); VMA_ASSERT(allocator && allocation && image); VMA_DEBUG_LOG("vmaBindImageMemory"); @@ -10075,6 +10174,30 @@ VkResult vmaCreateBuffer( { allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } + + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, + pBufferCreateInfo->flags, + pBufferCreateInfo->size, + pBufferCreateInfo->usage, + pBufferCreateInfo->sharingMode, + pAllocationCreateInfo->flags, + pAllocationCreateInfo->usage, + pAllocationCreateInfo->requiredFlags, + pAllocationCreateInfo->preferredFlags, + pAllocationCreateInfo->memoryTypeBits, + pAllocationCreateInfo->pool, + (*pAllocation), + pAllocationCreateInfo->pUserData ? (const char*)pAllocationCreateInfo->pUserData : ""); + fflush(g_File); + } + return VK_SUCCESS; } allocator->FreeMemory(*pAllocation); @@ -10096,6 +10219,19 @@ void vmaDestroyBuffer( VmaAllocation allocation) { VMA_ASSERT(allocator); + + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", threadId, time, frameIndex, + allocation); + fflush(g_File); + } + VMA_DEBUG_LOG("vmaDestroyBuffer"); VMA_DEBUG_GLOBAL_MUTEX_LOCK if(buffer != VK_NULL_HANDLE) @@ -10153,6 +10289,39 @@ VkResult vmaCreateImage( { allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } + + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, + pImageCreateInfo->flags, + pImageCreateInfo->imageType, + pImageCreateInfo->format, + pImageCreateInfo->extent.width, + pImageCreateInfo->extent.height, + pImageCreateInfo->extent.depth, + pImageCreateInfo->mipLevels, + pImageCreateInfo->arrayLayers, + pImageCreateInfo->samples, + pImageCreateInfo->tiling, + pImageCreateInfo->usage, + pImageCreateInfo->sharingMode, + pImageCreateInfo->initialLayout, + pAllocationCreateInfo->flags, + pAllocationCreateInfo->usage, + pAllocationCreateInfo->requiredFlags, + pAllocationCreateInfo->preferredFlags, + pAllocationCreateInfo->memoryTypeBits, + pAllocationCreateInfo->pool, + (*pAllocation), + pAllocationCreateInfo->pUserData ? (const char*)pAllocationCreateInfo->pUserData : ""); + fflush(g_File); + } + return VK_SUCCESS; } allocator->FreeMemory(*pAllocation); @@ -10174,6 +10343,19 @@ void vmaDestroyImage( VmaAllocation allocation) { VMA_ASSERT(allocator); + + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", threadId, time, frameIndex, + allocation); + fflush(g_File); + } + VMA_DEBUG_LOG("vmaDestroyImage"); VMA_DEBUG_GLOBAL_MUTEX_LOCK if(image != VK_NULL_HANDLE) From 0b9bcf0340f0a35a72113455ebb27c7888aa3f58 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 13:50:50 +0200 Subject: [PATCH 05/51] Added struct StrRange for convenience. --- src/VmaReplay/VmaReplay.cpp | 51 +++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 3fd9b4b..e69cf59 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -1762,6 +1762,14 @@ static const int RESULT_ERROR_COMMAND_LINE = -1; static const int RESULT_ERROR_SOURCE_FILE = -2; static const int RESULT_ERROR_FORMAT = -3; +struct StrRange +{ + const char* beg; + const char* end; + + size_t length() const { return end - beg; } +}; + //////////////////////////////////////////////////////////////////////////////// // LineSplit class @@ -1776,7 +1784,7 @@ public: { } - bool GetNextLine(const char *&outBeg, const char*&outEnd); + bool GetNextLine(StrRange& out); size_t GetNextLineIndex() const { return m_NextLineIndex; } private: @@ -1786,15 +1794,15 @@ private: size_t m_NextLineIndex; }; -bool LineSplit::GetNextLine(const char *&outBeg, const char*&outEnd) +bool LineSplit::GetNextLine(StrRange& out) { if(m_NextLineBeg < m_NumBytes) { - outBeg = m_Data + m_NextLineBeg; + out.beg = m_Data + m_NextLineBeg; size_t currLineEnd = m_NextLineBeg; while(currLineEnd < m_NumBytes && m_Data[currLineEnd] != '\n') ++currLineEnd; - outEnd = m_Data + currLineEnd; + out.end = m_Data + currLineEnd; m_NextLineBeg = currLineEnd + 1; // Past '\n' ++m_NextLineIndex; return true; @@ -1812,7 +1820,7 @@ class CsvSplit public: static const size_t RANGE_COUNT_MAX = 32; - void Set(const char* beg, const char* end, size_t maxCount = RANGE_COUNT_MAX); + void Set(const StrRange& line, size_t maxCount = RANGE_COUNT_MAX); size_t GetCount() const { return m_Count; } void GetRange(size_t index, const char*& outBeg, const char*& outEnd) const @@ -1827,11 +1835,11 @@ private: size_t m_Ranges[RANGE_COUNT_MAX * 2]; // Pairs of begin-end. }; -void CsvSplit::Set(const char* beg, const char* end, size_t maxCount) +void CsvSplit::Set(const StrRange& line, size_t maxCount) { assert(maxCount <= RANGE_COUNT_MAX); - m_Str = beg; - const size_t strLen = end - beg; + m_Str = line.beg; + const size_t strLen = line.length(); size_t rangeIndex = 0; size_t charIndex = 0; while(charIndex < strLen && rangeIndex < maxCount) @@ -1856,7 +1864,7 @@ public: void Init(); ~Player(); - void ExecuteLine(size_t lineNumber, const char* lineBeg, const char* lineEnd); + void ExecuteLine(size_t lineNumber, const StrRange& line); private: static const size_t MAX_WARNINGS_TO_SHOW = 16; @@ -1881,10 +1889,10 @@ Player::~Player() printf("WARNING: %zu more warnings not shown.\n", m_WarningCount - MAX_WARNINGS_TO_SHOW); } -void Player::ExecuteLine(size_t lineNumber, const char* lineBeg, const char* lineEnd) +void Player::ExecuteLine(size_t lineNumber, const StrRange& line) { CsvSplit csvSplit; - csvSplit.Set(lineBeg, lineEnd); + csvSplit.Set(line); if(csvSplit.GetCount() >= 4) { @@ -1906,11 +1914,11 @@ static void PrintCommandLineSyntax() " VmaReplay \n"); } -static inline bool StrRangeEq(const char* lhsBeg, const char* lhsEnd, const char* rhsSz) +static inline bool StrRangeEq(const StrRange& lhs, const char* rhsSz) { const size_t rhsLen = strlen(rhsSz); - return rhsLen == lhsEnd - lhsBeg && - memcmp(lhsBeg, rhsSz, rhsLen) == 0; + return rhsLen == lhs.length() && + memcmp(lhs.beg, rhsSz, rhsLen) == 0; } static int ProcessFile(const char* data, size_t numBytes) @@ -1919,18 +1927,17 @@ static int ProcessFile(const char* data, size_t numBytes) printf("File size: %zu B\n", numBytes); LineSplit lineSplit(data, numBytes); - const char* lineBeg; - const char* lineEnd; + StrRange line; - if(!lineSplit.GetNextLine(lineBeg, lineEnd) || - !StrRangeEq(lineBeg, lineEnd, "Vulkan Memory Allocator,Calls recording")) + if(!lineSplit.GetNextLine(line) || + !StrRangeEq(line, "Vulkan Memory Allocator,Calls recording")) { printf("ERROR: Incorrect file format.\n"); return RESULT_ERROR_FORMAT; } - if(!lineSplit.GetNextLine(lineBeg, lineEnd) || - !StrRangeEq(lineBeg, lineEnd, "1,0")) + if(!lineSplit.GetNextLine(line) || + !StrRangeEq(line, "1,0")) { printf("ERROR: Incorrect file format version.\n"); return RESULT_ERROR_FORMAT; @@ -1939,9 +1946,9 @@ static int ProcessFile(const char* data, size_t numBytes) Player player; player.Init(); - while(lineSplit.GetNextLine(lineBeg, lineEnd)) + while(lineSplit.GetNextLine(line)) { - player.ExecuteLine(lineSplit.GetNextLineIndex(), lineBeg, lineEnd); + player.ExecuteLine(lineSplit.GetNextLineIndex(), line); } // End stats. From 9d7d848658563778f9dd94a21edc46800eb7fd36 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 14:13:06 +0200 Subject: [PATCH 06/51] VmaReplay: Initialized Vulkan. --- src/VmaReplay/Common.h | 7 - src/VmaReplay/VmaReplay.cpp | 2012 +++++------------------------------ 2 files changed, 278 insertions(+), 1741 deletions(-) diff --git a/src/VmaReplay/Common.h b/src/VmaReplay/Common.h index c4934b2..70e7dad 100644 --- a/src/VmaReplay/Common.h +++ b/src/VmaReplay/Common.h @@ -24,11 +24,6 @@ typedef std::chrono::high_resolution_clock::duration duration; #define ERR_GUARD_VULKAN(Expr) do { VkResult res__ = (Expr); if (res__ < 0) assert(0); } while(0) -extern VkPhysicalDevice g_hPhysicalDevice; -extern VkDevice g_hDevice; -extern VmaAllocator g_hAllocator; -extern bool g_MemoryAliasingWarningEnabled; - inline float ToFloatSeconds(duration d) { return std::chrono::duration_cast>(d).count(); @@ -46,8 +41,6 @@ static inline T align_up(T val, T align) return (val + align - 1) / align * align; } -static const float PI = 3.14159265358979323846264338327950288419716939937510582f; - class RandomNumberGenerator { public: diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index e69cf59..09e38f7 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -23,1740 +23,6 @@ #include "VmaUsage.h" #include "Common.h" -#if 0 -static const char* const SHADER_PATH1 = "./"; -static const char* const SHADER_PATH2 = "../bin/"; -static const wchar_t* const WINDOW_CLASS_NAME = L"VULKAN_MEMORY_ALLOCATOR_SAMPLE"; -static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_LUNARG_standard_validation"; -static const char* const APP_TITLE_A = "Vulkan Memory Allocator Sample 2.0"; -static const wchar_t* const APP_TITLE_W = L"Vulkan Memory Allocator Sample 2.0"; - -static const bool VSYNC = true; -static const uint32_t COMMAND_BUFFER_COUNT = 2; -static void* const CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA = (void*)(intptr_t)43564544; -static const bool USE_CUSTOM_CPU_ALLOCATION_CALLBACKS = false; - -VkPhysicalDevice g_hPhysicalDevice; -VkDevice g_hDevice; -VmaAllocator g_hAllocator; -bool g_MemoryAliasingWarningEnabled = true; - -static bool g_EnableValidationLayer = true; -static bool VK_KHR_get_memory_requirements2_enabled = false; -static bool VK_KHR_dedicated_allocation_enabled = false; - -static HINSTANCE g_hAppInstance; -static HWND g_hWnd; -static LONG g_SizeX = 1280, g_SizeY = 720; -static VkInstance g_hVulkanInstance; -static VkSurfaceKHR g_hSurface; -static VkQueue g_hPresentQueue; -static VkSurfaceFormatKHR g_SurfaceFormat; -static VkExtent2D g_Extent; -static VkSwapchainKHR g_hSwapchain; -static std::vector g_SwapchainImages; -static std::vector g_SwapchainImageViews; -static std::vector g_Framebuffers; -static VkCommandPool g_hCommandPool; -static VkCommandBuffer g_MainCommandBuffers[COMMAND_BUFFER_COUNT]; -static VkFence g_MainCommandBufferExecutedFances[COMMAND_BUFFER_COUNT]; -static uint32_t g_NextCommandBufferIndex; -static VkSemaphore g_hImageAvailableSemaphore; -static VkSemaphore g_hRenderFinishedSemaphore; -static uint32_t g_GraphicsQueueFamilyIndex = UINT_MAX; -static uint32_t g_PresentQueueFamilyIndex = UINT_MAX; -static VkDescriptorSetLayout g_hDescriptorSetLayout; -static VkDescriptorPool g_hDescriptorPool; -static VkDescriptorSet g_hDescriptorSet; // Automatically destroyed with m_DescriptorPool. -static VkSampler g_hSampler; -static VkFormat g_DepthFormat; -static VkImage g_hDepthImage; -static VmaAllocation g_hDepthImageAlloc; -static VkImageView g_hDepthImageView; - -static VkSurfaceCapabilitiesKHR g_SurfaceCapabilities; -static std::vector g_SurfaceFormats; -static std::vector g_PresentModes; - -static PFN_vkCreateDebugReportCallbackEXT g_pvkCreateDebugReportCallbackEXT; -static PFN_vkDebugReportMessageEXT g_pvkDebugReportMessageEXT; -static PFN_vkDestroyDebugReportCallbackEXT g_pvkDestroyDebugReportCallbackEXT; -static VkDebugReportCallbackEXT g_hCallback; - -static VkQueue g_hGraphicsQueue; -static VkCommandBuffer g_hTemporaryCommandBuffer; - -static VkPipelineLayout g_hPipelineLayout; -static VkRenderPass g_hRenderPass; -static VkPipeline g_hPipeline; - -static VkBuffer g_hVertexBuffer; -static VmaAllocation g_hVertexBufferAlloc; -static VkBuffer g_hIndexBuffer; -static VmaAllocation g_hIndexBufferAlloc; -static uint32_t g_VertexCount; -static uint32_t g_IndexCount; - -static VkImage g_hTextureImage; -static VmaAllocation g_hTextureImageAlloc; -static VkImageView g_hTextureImageView; - -static void* CustomCpuAllocation( - void* pUserData, size_t size, size_t alignment, - VkSystemAllocationScope allocationScope) -{ - assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); - return _aligned_malloc(size, alignment); -} - -static void* CustomCpuReallocation( - void* pUserData, void* pOriginal, size_t size, size_t alignment, - VkSystemAllocationScope allocationScope) -{ - assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); - return _aligned_realloc(pOriginal, size, alignment); -} - -static void CustomCpuFree(void* pUserData, void* pMemory) -{ - assert(pUserData == CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA); - _aligned_free(pMemory); -} - -static void BeginSingleTimeCommands() -{ - VkCommandBufferBeginInfo cmdBufBeginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; - cmdBufBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - ERR_GUARD_VULKAN( vkBeginCommandBuffer(g_hTemporaryCommandBuffer, &cmdBufBeginInfo) ); -} - -static void EndSingleTimeCommands() -{ - ERR_GUARD_VULKAN( vkEndCommandBuffer(g_hTemporaryCommandBuffer) ); - - VkSubmitInfo submitInfo = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; - submitInfo.commandBufferCount = 1; - submitInfo.pCommandBuffers = &g_hTemporaryCommandBuffer; - - ERR_GUARD_VULKAN( vkQueueSubmit(g_hGraphicsQueue, 1, &submitInfo, VK_NULL_HANDLE) ); - ERR_GUARD_VULKAN( vkQueueWaitIdle(g_hGraphicsQueue) ); -} - -static void LoadShader(std::vector& out, const char* fileName) -{ - std::ifstream file(std::string(SHADER_PATH1) + fileName, std::ios::ate | std::ios::binary); - if(file.is_open() == false) - file.open(std::string(SHADER_PATH2) + fileName, std::ios::ate | std::ios::binary); - assert(file.is_open()); - size_t fileSize = (size_t)file.tellg(); - if(fileSize > 0) - { - out.resize(fileSize); - file.seekg(0); - file.read(out.data(), fileSize); - file.close(); - } - else - out.clear(); -} - -VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( - VkDebugReportFlagsEXT flags, - VkDebugReportObjectTypeEXT objectType, - uint64_t object, - size_t location, - int32_t messageCode, - const char* pLayerPrefix, - const char* pMessage, - void* pUserData) -{ - // "Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug." - if(!g_MemoryAliasingWarningEnabled && flags == VK_DEBUG_REPORT_WARNING_BIT_EXT && - (strstr(pMessage, " is aliased with non-linear ") || strstr(pMessage, " is aliased with linear "))) - { - return VK_FALSE; - } - - // Ignoring because when VK_KHR_dedicated_allocation extension is enabled, - // vkGetBufferMemoryRequirements2KHR function is used instead, while Validation - // Layer seems to be unaware of it. - if (strstr(pMessage, "but vkGetBufferMemoryRequirements() has not been called on that buffer") != nullptr) - { - return VK_FALSE; - } - if (strstr(pMessage, "but vkGetImageMemoryRequirements() has not been called on that image") != nullptr) - { - return VK_FALSE; - } - - switch(flags) - { - case VK_DEBUG_REPORT_WARNING_BIT_EXT: - SetConsoleColor(CONSOLE_COLOR::WARNING); - break; - case VK_DEBUG_REPORT_ERROR_BIT_EXT: - SetConsoleColor(CONSOLE_COLOR::ERROR_); - break; - default: - SetConsoleColor(CONSOLE_COLOR::INFO); - } - - printf("%s \xBA %s\n", pLayerPrefix, pMessage); - - SetConsoleColor(CONSOLE_COLOR::NORMAL); - - if(flags == VK_DEBUG_REPORT_WARNING_BIT_EXT || - flags == VK_DEBUG_REPORT_ERROR_BIT_EXT) - { - OutputDebugStringA(pMessage); - OutputDebugStringA("\n"); - } - - return VK_FALSE; -} - -static VkSurfaceFormatKHR ChooseSurfaceFormat() -{ - assert(!g_SurfaceFormats.empty()); - - if((g_SurfaceFormats.size() == 1) && (g_SurfaceFormats[0].format == VK_FORMAT_UNDEFINED)) - { - VkSurfaceFormatKHR result = { VK_FORMAT_B8G8R8A8_UNORM, VK_COLOR_SPACE_SRGB_NONLINEAR_KHR }; - return result; - } - - for(const auto& format : g_SurfaceFormats) - { - if((format.format == VK_FORMAT_B8G8R8A8_UNORM) && - (format.colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR)) - { - return format; - } - } - - return g_SurfaceFormats[0]; -} - -VkPresentModeKHR ChooseSwapPresentMode() -{ - VkPresentModeKHR preferredMode = VSYNC ? VK_PRESENT_MODE_MAILBOX_KHR : VK_PRESENT_MODE_IMMEDIATE_KHR; - - if(std::find(g_PresentModes.begin(), g_PresentModes.end(), preferredMode) != - g_PresentModes.end()) - { - return preferredMode; - } - - return VK_PRESENT_MODE_FIFO_KHR; -} - -static VkExtent2D ChooseSwapExtent() -{ - if(g_SurfaceCapabilities.currentExtent.width != UINT_MAX) - return g_SurfaceCapabilities.currentExtent; - - VkExtent2D result = { - std::max(g_SurfaceCapabilities.minImageExtent.width, - std::min(g_SurfaceCapabilities.maxImageExtent.width, (uint32_t)g_SizeX)), - std::max(g_SurfaceCapabilities.minImageExtent.height, - std::min(g_SurfaceCapabilities.maxImageExtent.height, (uint32_t)g_SizeY)) }; - return result; -} - -struct Vertex -{ - float pos[3]; - float color[3]; - float texCoord[2]; -}; - -static void CreateMesh() -{ - assert(g_hAllocator); - - static Vertex vertices[] = { - // -X - { { -1.f, -1.f, -1.f}, {1.0f, 0.0f, 0.0f}, {0.f, 0.f} }, - { { -1.f, -1.f, 1.f}, {1.0f, 0.0f, 0.0f}, {1.f, 0.f} }, - { { -1.f, 1.f, -1.f}, {1.0f, 0.0f, 0.0f}, {0.f, 1.f} }, - { { -1.f, 1.f, 1.f}, {1.0f, 0.0f, 0.0f}, {1.f, 1.f} }, - // +X - { { 1.f, -1.f, 1.f}, {0.0f, 1.0f, 0.0f}, {0.f, 0.f} }, - { { 1.f, -1.f, -1.f}, {0.0f, 1.0f, 0.0f}, {1.f, 0.f} }, - { { 1.f, 1.f, 1.f}, {0.0f, 1.0f, 0.0f}, {0.f, 1.f} }, - { { 1.f, 1.f, -1.f}, {0.0f, 1.0f, 0.0f}, {1.f, 1.f} }, - // -Z - { { 1.f, -1.f, -1.f}, {0.0f, 0.0f, 1.0f}, {0.f, 0.f} }, - { {-1.f, -1.f, -1.f}, {0.0f, 0.0f, 1.0f}, {1.f, 0.f} }, - { { 1.f, 1.f, -1.f}, {0.0f, 0.0f, 1.0f}, {0.f, 1.f} }, - { {-1.f, 1.f, -1.f}, {0.0f, 0.0f, 1.0f}, {1.f, 1.f} }, - // +Z - { {-1.f, -1.f, 1.f}, {1.0f, 1.0f, 0.0f}, {0.f, 0.f} }, - { { 1.f, -1.f, 1.f}, {1.0f, 1.0f, 0.0f}, {1.f, 0.f} }, - { {-1.f, 1.f, 1.f}, {1.0f, 1.0f, 0.0f}, {0.f, 1.f} }, - { { 1.f, 1.f, 1.f}, {1.0f, 1.0f, 0.0f}, {1.f, 1.f} }, - // -Y - { {-1.f, -1.f, -1.f}, {0.0f, 1.0f, 1.0f}, {0.f, 0.f} }, - { { 1.f, -1.f, -1.f}, {0.0f, 1.0f, 1.0f}, {1.f, 0.f} }, - { {-1.f, -1.f, 1.f}, {0.0f, 1.0f, 1.0f}, {0.f, 1.f} }, - { { 1.f, -1.f, 1.f}, {0.0f, 1.0f, 1.0f}, {1.f, 1.f} }, - // +Y - { { 1.f, 1.f, -1.f}, {1.0f, 0.0f, 1.0f}, {0.f, 0.f} }, - { {-1.f, 1.f, -1.f}, {1.0f, 0.0f, 1.0f}, {1.f, 0.f} }, - { { 1.f, 1.f, 1.f}, {1.0f, 0.0f, 1.0f}, {0.f, 1.f} }, - { {-1.f, 1.f, 1.f}, {1.0f, 0.0f, 1.0f}, {1.f, 1.f} }, - }; - static uint16_t indices[] = { - 0, 1, 2, 3, USHRT_MAX, - 4, 5, 6, 7, USHRT_MAX, - 8, 9, 10, 11, USHRT_MAX, - 12, 13, 14, 15, USHRT_MAX, - 16, 17, 18, 19, USHRT_MAX, - 20, 21, 22, 23, USHRT_MAX, - }; - - size_t vertexBufferSize = sizeof(Vertex) * _countof(vertices); - size_t indexBufferSize = sizeof(uint16_t) * _countof(indices); - g_IndexCount = (uint32_t)_countof(indices); - - // Create vertex buffer - - VkBufferCreateInfo vbInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; - vbInfo.size = vertexBufferSize; - vbInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; - vbInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - - VmaAllocationCreateInfo vbAllocCreateInfo = {}; - vbAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; - vbAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; - - VkBuffer stagingVertexBuffer = VK_NULL_HANDLE; - VmaAllocation stagingVertexBufferAlloc = VK_NULL_HANDLE; - VmaAllocationInfo stagingVertexBufferAllocInfo = {}; - ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &vbInfo, &vbAllocCreateInfo, &stagingVertexBuffer, &stagingVertexBufferAlloc, &stagingVertexBufferAllocInfo) ); - - memcpy(stagingVertexBufferAllocInfo.pMappedData, vertices, vertexBufferSize); - - // No need to flush stagingVertexBuffer memory because CPU_ONLY memory is always HOST_COHERENT. - - vbInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; - vbAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; - vbAllocCreateInfo.flags = 0; - ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &vbInfo, &vbAllocCreateInfo, &g_hVertexBuffer, &g_hVertexBufferAlloc, nullptr) ); - - // Create index buffer - - VkBufferCreateInfo ibInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; - ibInfo.size = indexBufferSize; - ibInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; - ibInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - - VmaAllocationCreateInfo ibAllocCreateInfo = {}; - ibAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; - ibAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; - - VkBuffer stagingIndexBuffer = VK_NULL_HANDLE; - VmaAllocation stagingIndexBufferAlloc = VK_NULL_HANDLE; - VmaAllocationInfo stagingIndexBufferAllocInfo = {}; - ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &ibInfo, &ibAllocCreateInfo, &stagingIndexBuffer, &stagingIndexBufferAlloc, &stagingIndexBufferAllocInfo) ); - - memcpy(stagingIndexBufferAllocInfo.pMappedData, indices, indexBufferSize); - - // No need to flush stagingIndexBuffer memory because CPU_ONLY memory is always HOST_COHERENT. - - ibInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT; - ibAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; - ibAllocCreateInfo.flags = 0; - ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &ibInfo, &ibAllocCreateInfo, &g_hIndexBuffer, &g_hIndexBufferAlloc, nullptr) ); - - // Copy buffers - - BeginSingleTimeCommands(); - - VkBufferCopy vbCopyRegion = {}; - vbCopyRegion.srcOffset = 0; - vbCopyRegion.dstOffset = 0; - vbCopyRegion.size = vbInfo.size; - vkCmdCopyBuffer(g_hTemporaryCommandBuffer, stagingVertexBuffer, g_hVertexBuffer, 1, &vbCopyRegion); - - VkBufferCopy ibCopyRegion = {}; - ibCopyRegion.srcOffset = 0; - ibCopyRegion.dstOffset = 0; - ibCopyRegion.size = ibInfo.size; - vkCmdCopyBuffer(g_hTemporaryCommandBuffer, stagingIndexBuffer, g_hIndexBuffer, 1, &ibCopyRegion); - - EndSingleTimeCommands(); - - vmaDestroyBuffer(g_hAllocator, stagingIndexBuffer, stagingIndexBufferAlloc); - vmaDestroyBuffer(g_hAllocator, stagingVertexBuffer, stagingVertexBufferAlloc); -} - -static void CreateTexture(uint32_t sizeX, uint32_t sizeY) -{ - // Create Image - - const VkDeviceSize imageSize = sizeX * sizeY * 4; - - VkImageCreateInfo stagingImageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; - stagingImageInfo.imageType = VK_IMAGE_TYPE_2D; - stagingImageInfo.extent.width = sizeX; - stagingImageInfo.extent.height = sizeY; - stagingImageInfo.extent.depth = 1; - stagingImageInfo.mipLevels = 1; - stagingImageInfo.arrayLayers = 1; - stagingImageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; - stagingImageInfo.tiling = VK_IMAGE_TILING_LINEAR; - stagingImageInfo.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; - stagingImageInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT; - stagingImageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - stagingImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; - stagingImageInfo.flags = 0; - - VmaAllocationCreateInfo stagingImageAllocCreateInfo = {}; - stagingImageAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; - stagingImageAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT; - - VkImage stagingImage = VK_NULL_HANDLE; - VmaAllocation stagingImageAlloc = VK_NULL_HANDLE; - VmaAllocationInfo stagingImageAllocInfo = {}; - ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &stagingImageInfo, &stagingImageAllocCreateInfo, &stagingImage, &stagingImageAlloc, &stagingImageAllocInfo) ); - - VkImageSubresource imageSubresource = {}; - imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageSubresource.mipLevel = 0; - imageSubresource.arrayLayer = 0; - - VkSubresourceLayout imageLayout = {}; - vkGetImageSubresourceLayout(g_hDevice, stagingImage, &imageSubresource, &imageLayout); - - char* const pMipLevelData = (char*)stagingImageAllocInfo.pMappedData + imageLayout.offset; - uint8_t* pRowData = (uint8_t*)pMipLevelData; - for(uint32_t y = 0; y < sizeY; ++y) - { - uint32_t* pPixelData = (uint32_t*)pRowData; - for(uint32_t x = 0; x < sizeY; ++x) - { - *pPixelData = - ((x & 0x18) == 0x08 ? 0x000000FF : 0x00000000) | - ((x & 0x18) == 0x10 ? 0x0000FFFF : 0x00000000) | - ((y & 0x18) == 0x08 ? 0x0000FF00 : 0x00000000) | - ((y & 0x18) == 0x10 ? 0x00FF0000 : 0x00000000); - ++pPixelData; - } - pRowData += imageLayout.rowPitch; - } - - // No need to flush stagingImage memory because CPU_ONLY memory is always HOST_COHERENT. - - // Create g_hTextureImage in GPU memory. - - VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; - imageInfo.imageType = VK_IMAGE_TYPE_2D; - imageInfo.extent.width = sizeX; - imageInfo.extent.height = sizeY; - imageInfo.extent.depth = 1; - imageInfo.mipLevels = 1; - imageInfo.arrayLayers = 1; - imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; - imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; - imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; - imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; - imageInfo.flags = 0; - - VmaAllocationCreateInfo imageAllocCreateInfo = {}; - imageAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; - - ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &imageInfo, &imageAllocCreateInfo, &g_hTextureImage, &g_hTextureImageAlloc, nullptr) ); - - // Transition image layouts, copy image. - - BeginSingleTimeCommands(); - - VkImageMemoryBarrier imgMemBarrier = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; - imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_PREINITIALIZED; - imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; - imgMemBarrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - imgMemBarrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - imgMemBarrier.image = stagingImage; - imgMemBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imgMemBarrier.subresourceRange.baseMipLevel = 0; - imgMemBarrier.subresourceRange.levelCount = 1; - imgMemBarrier.subresourceRange.baseArrayLayer = 0; - imgMemBarrier.subresourceRange.layerCount = 1; - imgMemBarrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; - imgMemBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - - vkCmdPipelineBarrier( - g_hTemporaryCommandBuffer, - VK_PIPELINE_STAGE_HOST_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, - 0, nullptr, - 0, nullptr, - 1, &imgMemBarrier); - - imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; - imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - imgMemBarrier.image = g_hTextureImage; - imgMemBarrier.srcAccessMask = 0; - imgMemBarrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - - vkCmdPipelineBarrier( - g_hTemporaryCommandBuffer, - VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, - 0, nullptr, - 0, nullptr, - 1, &imgMemBarrier); - - VkImageCopy imageCopy = {}; - imageCopy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageCopy.srcSubresource.baseArrayLayer = 0; - imageCopy.srcSubresource.mipLevel = 0; - imageCopy.srcSubresource.layerCount = 1; - imageCopy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageCopy.dstSubresource.baseArrayLayer = 0; - imageCopy.dstSubresource.mipLevel = 0; - imageCopy.dstSubresource.layerCount = 1; - imageCopy.srcOffset.x = 0; - imageCopy.srcOffset.y = 0; - imageCopy.srcOffset.z = 0; - imageCopy.dstOffset.x = 0; - imageCopy.dstOffset.y = 0; - imageCopy.dstOffset.z = 0; - imageCopy.extent.width = sizeX; - imageCopy.extent.height = sizeY; - imageCopy.extent.depth = 1; - vkCmdCopyImage( - g_hTemporaryCommandBuffer, - stagingImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - g_hTextureImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - 1, &imageCopy); - - imgMemBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - imgMemBarrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - imgMemBarrier.image = g_hTextureImage; - imgMemBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - imgMemBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; - - vkCmdPipelineBarrier( - g_hTemporaryCommandBuffer, - VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - 0, - 0, nullptr, - 0, nullptr, - 1, &imgMemBarrier); - - EndSingleTimeCommands(); - - vmaDestroyImage(g_hAllocator, stagingImage, stagingImageAlloc); - - // Create ImageView - - VkImageViewCreateInfo textureImageViewInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; - textureImageViewInfo.image = g_hTextureImage; - textureImageViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; - textureImageViewInfo.format = VK_FORMAT_R8G8B8A8_UNORM; - textureImageViewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - textureImageViewInfo.subresourceRange.baseMipLevel = 0; - textureImageViewInfo.subresourceRange.levelCount = 1; - textureImageViewInfo.subresourceRange.baseArrayLayer = 0; - textureImageViewInfo.subresourceRange.layerCount = 1; - ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &textureImageViewInfo, nullptr, &g_hTextureImageView) ); -} - -struct UniformBufferObject -{ - mat4 ModelViewProj; -}; - -static void RegisterDebugCallbacks() -{ - g_pvkCreateDebugReportCallbackEXT = - reinterpret_cast - (vkGetInstanceProcAddr(g_hVulkanInstance, "vkCreateDebugReportCallbackEXT")); - g_pvkDebugReportMessageEXT = - reinterpret_cast - (vkGetInstanceProcAddr(g_hVulkanInstance, "vkDebugReportMessageEXT")); - g_pvkDestroyDebugReportCallbackEXT = - reinterpret_cast - (vkGetInstanceProcAddr(g_hVulkanInstance, "vkDestroyDebugReportCallbackEXT")); - assert(g_pvkCreateDebugReportCallbackEXT); - assert(g_pvkDebugReportMessageEXT); - assert(g_pvkDestroyDebugReportCallbackEXT); - - VkDebugReportCallbackCreateInfoEXT callbackCreateInfo; - callbackCreateInfo.sType = VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT; - callbackCreateInfo.pNext = nullptr; - callbackCreateInfo.flags = //VK_DEBUG_REPORT_INFORMATION_BIT_EXT | - VK_DEBUG_REPORT_ERROR_BIT_EXT | - VK_DEBUG_REPORT_WARNING_BIT_EXT | - VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT /*| - VK_DEBUG_REPORT_DEBUG_BIT_EXT*/; - callbackCreateInfo.pfnCallback = &MyDebugReportCallback; - callbackCreateInfo.pUserData = nullptr; - - ERR_GUARD_VULKAN( g_pvkCreateDebugReportCallbackEXT(g_hVulkanInstance, &callbackCreateInfo, nullptr, &g_hCallback) ); -} - -static bool IsLayerSupported(const VkLayerProperties* pProps, size_t propCount, const char* pLayerName) -{ - const VkLayerProperties* propsEnd = pProps + propCount; - return std::find_if( - pProps, - propsEnd, - [pLayerName](const VkLayerProperties& prop) -> bool { - return strcmp(pLayerName, prop.layerName) == 0; - }) != propsEnd; -} - -static VkFormat FindSupportedFormat( - const std::vector& candidates, - VkImageTiling tiling, - VkFormatFeatureFlags features) -{ - for (VkFormat format : candidates) - { - VkFormatProperties props; - vkGetPhysicalDeviceFormatProperties(g_hPhysicalDevice, format, &props); - - if ((tiling == VK_IMAGE_TILING_LINEAR) && - ((props.linearTilingFeatures & features) == features)) - { - return format; - } - else if ((tiling == VK_IMAGE_TILING_OPTIMAL) && - ((props.optimalTilingFeatures & features) == features)) - { - return format; - } - } - return VK_FORMAT_UNDEFINED; -} - -static VkFormat FindDepthFormat() -{ - std::vector formats; - formats.push_back(VK_FORMAT_D32_SFLOAT); - formats.push_back(VK_FORMAT_D32_SFLOAT_S8_UINT); - formats.push_back(VK_FORMAT_D24_UNORM_S8_UINT); - - return FindSupportedFormat( - formats, - VK_IMAGE_TILING_OPTIMAL, - VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT); -} - -static void CreateSwapchain() -{ - // Query surface formats. - - ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfaceCapabilitiesKHR(g_hPhysicalDevice, g_hSurface, &g_SurfaceCapabilities) ); - - uint32_t formatCount = 0; - ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfaceFormatsKHR(g_hPhysicalDevice, g_hSurface, &formatCount, nullptr) ); - g_SurfaceFormats.resize(formatCount); - ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfaceFormatsKHR(g_hPhysicalDevice, g_hSurface, &formatCount, g_SurfaceFormats.data()) ); - - uint32_t presentModeCount = 0; - ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfacePresentModesKHR(g_hPhysicalDevice, g_hSurface, &presentModeCount, nullptr) ); - g_PresentModes.resize(presentModeCount); - ERR_GUARD_VULKAN( vkGetPhysicalDeviceSurfacePresentModesKHR(g_hPhysicalDevice, g_hSurface, &presentModeCount, g_PresentModes.data()) ); - - // Create swap chain - - g_SurfaceFormat = ChooseSurfaceFormat(); - VkPresentModeKHR presentMode = ChooseSwapPresentMode(); - g_Extent = ChooseSwapExtent(); - - uint32_t imageCount = g_SurfaceCapabilities.minImageCount + 1; - if((g_SurfaceCapabilities.maxImageCount > 0) && - (imageCount > g_SurfaceCapabilities.maxImageCount)) - { - imageCount = g_SurfaceCapabilities.maxImageCount; - } - - VkSwapchainCreateInfoKHR swapChainInfo = { VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR }; - swapChainInfo.surface = g_hSurface; - swapChainInfo.minImageCount = imageCount; - swapChainInfo.imageFormat = g_SurfaceFormat.format; - swapChainInfo.imageColorSpace = g_SurfaceFormat.colorSpace; - swapChainInfo.imageExtent = g_Extent; - swapChainInfo.imageArrayLayers = 1; - swapChainInfo.imageUsage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; - swapChainInfo.preTransform = g_SurfaceCapabilities.currentTransform; - swapChainInfo.compositeAlpha = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR; - swapChainInfo.presentMode = presentMode; - swapChainInfo.clipped = VK_TRUE; - swapChainInfo.oldSwapchain = g_hSwapchain; - - uint32_t queueFamilyIndices[] = { g_GraphicsQueueFamilyIndex, g_PresentQueueFamilyIndex }; - if(g_PresentQueueFamilyIndex != g_GraphicsQueueFamilyIndex) - { - swapChainInfo.imageSharingMode = VK_SHARING_MODE_CONCURRENT; - swapChainInfo.queueFamilyIndexCount = 2; - swapChainInfo.pQueueFamilyIndices = queueFamilyIndices; - } - else - { - swapChainInfo.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE; - } - - VkSwapchainKHR hNewSwapchain = VK_NULL_HANDLE; - ERR_GUARD_VULKAN( vkCreateSwapchainKHR(g_hDevice, &swapChainInfo, nullptr, &hNewSwapchain) ); - if(g_hSwapchain != VK_NULL_HANDLE) - vkDestroySwapchainKHR(g_hDevice, g_hSwapchain, nullptr); - g_hSwapchain = hNewSwapchain; - - // Retrieve swapchain images. - - uint32_t swapchainImageCount = 0; - ERR_GUARD_VULKAN( vkGetSwapchainImagesKHR(g_hDevice, g_hSwapchain, &swapchainImageCount, nullptr) ); - g_SwapchainImages.resize(swapchainImageCount); - ERR_GUARD_VULKAN( vkGetSwapchainImagesKHR(g_hDevice, g_hSwapchain, &swapchainImageCount, g_SwapchainImages.data()) ); - - // Create swapchain image views. - - for(size_t i = g_SwapchainImageViews.size(); i--; ) - vkDestroyImageView(g_hDevice, g_SwapchainImageViews[i], nullptr); - g_SwapchainImageViews.clear(); - - VkImageViewCreateInfo swapchainImageViewInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; - g_SwapchainImageViews.resize(swapchainImageCount); - for(uint32_t i = 0; i < swapchainImageCount; ++i) - { - swapchainImageViewInfo.image = g_SwapchainImages[i]; - swapchainImageViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; - swapchainImageViewInfo.format = g_SurfaceFormat.format; - swapchainImageViewInfo.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; - swapchainImageViewInfo.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; - swapchainImageViewInfo.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; - swapchainImageViewInfo.components.a = VK_COMPONENT_SWIZZLE_IDENTITY; - swapchainImageViewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - swapchainImageViewInfo.subresourceRange.baseMipLevel = 0; - swapchainImageViewInfo.subresourceRange.levelCount = 1; - swapchainImageViewInfo.subresourceRange.baseArrayLayer = 0; - swapchainImageViewInfo.subresourceRange.layerCount = 1; - ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &swapchainImageViewInfo, nullptr, &g_SwapchainImageViews[i]) ); - } - - // Create depth buffer - - g_DepthFormat = FindDepthFormat(); - assert(g_DepthFormat != VK_FORMAT_UNDEFINED); - - VkImageCreateInfo depthImageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; - depthImageInfo.imageType = VK_IMAGE_TYPE_2D; - depthImageInfo.extent.width = g_Extent.width; - depthImageInfo.extent.height = g_Extent.height; - depthImageInfo.extent.depth = 1; - depthImageInfo.mipLevels = 1; - depthImageInfo.arrayLayers = 1; - depthImageInfo.format = g_DepthFormat; - depthImageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; - depthImageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - depthImageInfo.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT; - depthImageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - depthImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; - depthImageInfo.flags = 0; - - VmaAllocationCreateInfo depthImageAllocCreateInfo = {}; - depthImageAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; - - ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &depthImageInfo, &depthImageAllocCreateInfo, &g_hDepthImage, &g_hDepthImageAlloc, nullptr) ); - - VkImageViewCreateInfo depthImageViewInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; - depthImageViewInfo.image = g_hDepthImage; - depthImageViewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; - depthImageViewInfo.format = g_DepthFormat; - depthImageViewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT; - depthImageViewInfo.subresourceRange.baseMipLevel = 0; - depthImageViewInfo.subresourceRange.levelCount = 1; - depthImageViewInfo.subresourceRange.baseArrayLayer = 0; - depthImageViewInfo.subresourceRange.layerCount = 1; - - ERR_GUARD_VULKAN( vkCreateImageView(g_hDevice, &depthImageViewInfo, nullptr, &g_hDepthImageView) ); - - // Create pipeline layout - { - if(g_hPipelineLayout != VK_NULL_HANDLE) - { - vkDestroyPipelineLayout(g_hDevice, g_hPipelineLayout, nullptr); - g_hPipelineLayout = VK_NULL_HANDLE; - } - - VkPushConstantRange pushConstantRanges[1]; - ZeroMemory(&pushConstantRanges, sizeof pushConstantRanges); - pushConstantRanges[0].offset = 0; - pushConstantRanges[0].size = sizeof(UniformBufferObject); - pushConstantRanges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT; - - VkDescriptorSetLayout descriptorSetLayouts[] = { g_hDescriptorSetLayout }; - VkPipelineLayoutCreateInfo pipelineLayoutInfo = { VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO }; - pipelineLayoutInfo.setLayoutCount = 1; - pipelineLayoutInfo.pSetLayouts = descriptorSetLayouts; - pipelineLayoutInfo.pushConstantRangeCount = 1; - pipelineLayoutInfo.pPushConstantRanges = pushConstantRanges; - ERR_GUARD_VULKAN( vkCreatePipelineLayout(g_hDevice, &pipelineLayoutInfo, nullptr, &g_hPipelineLayout) ); - } - - // Create render pass - { - if(g_hRenderPass != VK_NULL_HANDLE) - { - vkDestroyRenderPass(g_hDevice, g_hRenderPass, nullptr); - g_hRenderPass = VK_NULL_HANDLE; - } - - VkAttachmentDescription attachments[2]; - ZeroMemory(attachments, sizeof(attachments)); - - attachments[0].format = g_SurfaceFormat.format; - attachments[0].samples = VK_SAMPLE_COUNT_1_BIT; - attachments[0].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; - attachments[0].storeOp = VK_ATTACHMENT_STORE_OP_STORE; - attachments[0].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; - attachments[0].stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; - attachments[0].initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - attachments[0].finalLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; - - attachments[1].format = g_DepthFormat; - attachments[1].samples = VK_SAMPLE_COUNT_1_BIT; - attachments[1].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; - attachments[1].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; - attachments[1].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; - attachments[1].stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; - attachments[1].initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - attachments[1].finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; - - VkAttachmentReference colorAttachmentRef = {}; - colorAttachmentRef.attachment = 0; - colorAttachmentRef.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - - VkAttachmentReference depthStencilAttachmentRef = {}; - depthStencilAttachmentRef.attachment = 1; - depthStencilAttachmentRef.layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; - - VkSubpassDescription subpassDesc = {}; - subpassDesc.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; - subpassDesc.colorAttachmentCount = 1; - subpassDesc.pColorAttachments = &colorAttachmentRef; - subpassDesc.pDepthStencilAttachment = &depthStencilAttachmentRef; - - VkRenderPassCreateInfo renderPassInfo = { VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO }; - renderPassInfo.attachmentCount = (uint32_t)_countof(attachments); - renderPassInfo.pAttachments = attachments; - renderPassInfo.subpassCount = 1; - renderPassInfo.pSubpasses = &subpassDesc; - renderPassInfo.dependencyCount = 0; - ERR_GUARD_VULKAN( vkCreateRenderPass(g_hDevice, &renderPassInfo, nullptr, &g_hRenderPass) ); - } - - // Create pipeline - { - std::vector vertShaderCode; - LoadShader(vertShaderCode, "Shader.vert.spv"); - VkShaderModuleCreateInfo shaderModuleInfo = { VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO }; - shaderModuleInfo.codeSize = vertShaderCode.size(); - shaderModuleInfo.pCode = (const uint32_t*)vertShaderCode.data(); - VkShaderModule hVertShaderModule = VK_NULL_HANDLE; - ERR_GUARD_VULKAN( vkCreateShaderModule(g_hDevice, &shaderModuleInfo, nullptr, &hVertShaderModule) ); - - std::vector hFragShaderCode; - LoadShader(hFragShaderCode, "Shader.frag.spv"); - shaderModuleInfo.codeSize = hFragShaderCode.size(); - shaderModuleInfo.pCode = (const uint32_t*)hFragShaderCode.data(); - VkShaderModule fragShaderModule = VK_NULL_HANDLE; - ERR_GUARD_VULKAN( vkCreateShaderModule(g_hDevice, &shaderModuleInfo, nullptr, &fragShaderModule) ); - - VkPipelineShaderStageCreateInfo vertPipelineShaderStageInfo = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO }; - vertPipelineShaderStageInfo.stage = VK_SHADER_STAGE_VERTEX_BIT; - vertPipelineShaderStageInfo.module = hVertShaderModule; - vertPipelineShaderStageInfo.pName = "main"; - - VkPipelineShaderStageCreateInfo fragPipelineShaderStageInfo = { VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO }; - fragPipelineShaderStageInfo.stage = VK_SHADER_STAGE_FRAGMENT_BIT; - fragPipelineShaderStageInfo.module = fragShaderModule; - fragPipelineShaderStageInfo.pName = "main"; - - VkPipelineShaderStageCreateInfo pipelineShaderStageInfos[] = { - vertPipelineShaderStageInfo, - fragPipelineShaderStageInfo - }; - - VkVertexInputBindingDescription bindingDescription = {}; - bindingDescription.binding = 0; - bindingDescription.stride = sizeof(Vertex); - bindingDescription.inputRate = VK_VERTEX_INPUT_RATE_VERTEX; - - VkVertexInputAttributeDescription attributeDescriptions[3]; - ZeroMemory(attributeDescriptions, sizeof(attributeDescriptions)); - - attributeDescriptions[0].binding = 0; - attributeDescriptions[0].location = 0; - attributeDescriptions[0].format = VK_FORMAT_R32G32B32_SFLOAT; - attributeDescriptions[0].offset = offsetof(Vertex, pos); - - attributeDescriptions[1].binding = 0; - attributeDescriptions[1].location = 1; - attributeDescriptions[1].format = VK_FORMAT_R32G32B32_SFLOAT; - attributeDescriptions[1].offset = offsetof(Vertex, color); - - attributeDescriptions[2].binding = 0; - attributeDescriptions[2].location = 2; - attributeDescriptions[2].format = VK_FORMAT_R32G32_SFLOAT; - attributeDescriptions[2].offset = offsetof(Vertex, texCoord); - - VkPipelineVertexInputStateCreateInfo pipelineVertexInputStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO }; - pipelineVertexInputStateInfo.vertexBindingDescriptionCount = 1; - pipelineVertexInputStateInfo.pVertexBindingDescriptions = &bindingDescription; - pipelineVertexInputStateInfo.vertexAttributeDescriptionCount = _countof(attributeDescriptions); - pipelineVertexInputStateInfo.pVertexAttributeDescriptions = attributeDescriptions; - - VkPipelineInputAssemblyStateCreateInfo pipelineInputAssemblyStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO }; - pipelineInputAssemblyStateInfo.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; - pipelineInputAssemblyStateInfo.primitiveRestartEnable = VK_TRUE; - - VkViewport viewport = {}; - viewport.x = 0.f; - viewport.y = 0.f; - viewport.width = (float)g_Extent.width; - viewport.height = (float)g_Extent.height; - viewport.minDepth = 0.f; - viewport.maxDepth = 1.f; - - VkRect2D scissor = {}; - scissor.offset.x = 0; - scissor.offset.y = 0; - scissor.extent = g_Extent; - - VkPipelineViewportStateCreateInfo pipelineViewportStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO }; - pipelineViewportStateInfo.viewportCount = 1; - pipelineViewportStateInfo.pViewports = &viewport; - pipelineViewportStateInfo.scissorCount = 1; - pipelineViewportStateInfo.pScissors = &scissor; - - VkPipelineRasterizationStateCreateInfo pipelineRasterizationStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO }; - pipelineRasterizationStateInfo.depthClampEnable = VK_FALSE; - pipelineRasterizationStateInfo.rasterizerDiscardEnable = VK_FALSE; - pipelineRasterizationStateInfo.polygonMode = VK_POLYGON_MODE_FILL; - pipelineRasterizationStateInfo.lineWidth = 1.f; - pipelineRasterizationStateInfo.cullMode = VK_CULL_MODE_BACK_BIT; - pipelineRasterizationStateInfo.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; - pipelineRasterizationStateInfo.depthBiasEnable = VK_FALSE; - pipelineRasterizationStateInfo.depthBiasConstantFactor = 0.f; - pipelineRasterizationStateInfo.depthBiasClamp = 0.f; - pipelineRasterizationStateInfo.depthBiasSlopeFactor = 0.f; - - VkPipelineMultisampleStateCreateInfo pipelineMultisampleStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO }; - pipelineMultisampleStateInfo.sampleShadingEnable = VK_FALSE; - pipelineMultisampleStateInfo.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; - pipelineMultisampleStateInfo.minSampleShading = 1.f; - pipelineMultisampleStateInfo.pSampleMask = nullptr; - pipelineMultisampleStateInfo.alphaToCoverageEnable = VK_FALSE; - pipelineMultisampleStateInfo.alphaToOneEnable = VK_FALSE; - - VkPipelineColorBlendAttachmentState pipelineColorBlendAttachmentState = {}; - pipelineColorBlendAttachmentState.colorWriteMask = - VK_COLOR_COMPONENT_R_BIT | - VK_COLOR_COMPONENT_G_BIT | - VK_COLOR_COMPONENT_B_BIT | - VK_COLOR_COMPONENT_A_BIT; - pipelineColorBlendAttachmentState.blendEnable = VK_FALSE; - pipelineColorBlendAttachmentState.srcColorBlendFactor = VK_BLEND_FACTOR_ONE; // Optional - pipelineColorBlendAttachmentState.dstColorBlendFactor = VK_BLEND_FACTOR_ZERO; // Optional - pipelineColorBlendAttachmentState.colorBlendOp = VK_BLEND_OP_ADD; // Optional - pipelineColorBlendAttachmentState.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE; // Optional - pipelineColorBlendAttachmentState.dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO; // Optional - pipelineColorBlendAttachmentState.alphaBlendOp = VK_BLEND_OP_ADD; // Optional - - VkPipelineColorBlendStateCreateInfo pipelineColorBlendStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO }; - pipelineColorBlendStateInfo.logicOpEnable = VK_FALSE; - pipelineColorBlendStateInfo.logicOp = VK_LOGIC_OP_COPY; - pipelineColorBlendStateInfo.attachmentCount = 1; - pipelineColorBlendStateInfo.pAttachments = &pipelineColorBlendAttachmentState; - - VkPipelineDepthStencilStateCreateInfo depthStencilStateInfo = { VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO }; - depthStencilStateInfo.depthTestEnable = VK_TRUE; - depthStencilStateInfo.depthWriteEnable = VK_TRUE; - depthStencilStateInfo.depthCompareOp = VK_COMPARE_OP_LESS; - depthStencilStateInfo.depthBoundsTestEnable = VK_FALSE; - depthStencilStateInfo.stencilTestEnable = VK_FALSE; - - VkGraphicsPipelineCreateInfo pipelineInfo = { VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO }; - pipelineInfo.stageCount = 2; - pipelineInfo.pStages = pipelineShaderStageInfos; - pipelineInfo.pVertexInputState = &pipelineVertexInputStateInfo; - pipelineInfo.pInputAssemblyState = &pipelineInputAssemblyStateInfo; - pipelineInfo.pViewportState = &pipelineViewportStateInfo; - pipelineInfo.pRasterizationState = &pipelineRasterizationStateInfo; - pipelineInfo.pMultisampleState = &pipelineMultisampleStateInfo; - pipelineInfo.pDepthStencilState = &depthStencilStateInfo; - pipelineInfo.pColorBlendState = &pipelineColorBlendStateInfo; - pipelineInfo.pDynamicState = nullptr; - pipelineInfo.layout = g_hPipelineLayout; - pipelineInfo.renderPass = g_hRenderPass; - pipelineInfo.subpass = 0; - pipelineInfo.basePipelineHandle = VK_NULL_HANDLE; - pipelineInfo.basePipelineIndex = -1; - ERR_GUARD_VULKAN( vkCreateGraphicsPipelines( - g_hDevice, - VK_NULL_HANDLE, - 1, - &pipelineInfo, nullptr, - &g_hPipeline) ); - - vkDestroyShaderModule(g_hDevice, fragShaderModule, nullptr); - vkDestroyShaderModule(g_hDevice, hVertShaderModule, nullptr); - } - - // Create frambuffers - - for(size_t i = g_Framebuffers.size(); i--; ) - vkDestroyFramebuffer(g_hDevice, g_Framebuffers[i], nullptr); - g_Framebuffers.clear(); - - g_Framebuffers.resize(g_SwapchainImageViews.size()); - for(size_t i = 0; i < g_SwapchainImages.size(); ++i) - { - VkImageView attachments[] = { g_SwapchainImageViews[i], g_hDepthImageView }; - - VkFramebufferCreateInfo framebufferInfo = { VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO }; - framebufferInfo.renderPass = g_hRenderPass; - framebufferInfo.attachmentCount = (uint32_t)_countof(attachments); - framebufferInfo.pAttachments = attachments; - framebufferInfo.width = g_Extent.width; - framebufferInfo.height = g_Extent.height; - framebufferInfo.layers = 1; - ERR_GUARD_VULKAN( vkCreateFramebuffer(g_hDevice, &framebufferInfo, nullptr, &g_Framebuffers[i]) ); - } - - // Create semaphores - - if(g_hImageAvailableSemaphore != VK_NULL_HANDLE) - { - vkDestroySemaphore(g_hDevice, g_hImageAvailableSemaphore, nullptr); - g_hImageAvailableSemaphore = VK_NULL_HANDLE; - } - if(g_hRenderFinishedSemaphore != VK_NULL_HANDLE) - { - vkDestroySemaphore(g_hDevice, g_hRenderFinishedSemaphore, nullptr); - g_hRenderFinishedSemaphore = VK_NULL_HANDLE; - } - - VkSemaphoreCreateInfo semaphoreInfo = { VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO }; - ERR_GUARD_VULKAN( vkCreateSemaphore(g_hDevice, &semaphoreInfo, nullptr, &g_hImageAvailableSemaphore) ); - ERR_GUARD_VULKAN( vkCreateSemaphore(g_hDevice, &semaphoreInfo, nullptr, &g_hRenderFinishedSemaphore) ); -} - -static void DestroySwapchain(bool destroyActualSwapchain) -{ - if(g_hImageAvailableSemaphore != VK_NULL_HANDLE) - { - vkDestroySemaphore(g_hDevice, g_hImageAvailableSemaphore, nullptr); - g_hImageAvailableSemaphore = VK_NULL_HANDLE; - } - if(g_hRenderFinishedSemaphore != VK_NULL_HANDLE) - { - vkDestroySemaphore(g_hDevice, g_hRenderFinishedSemaphore, nullptr); - g_hRenderFinishedSemaphore = VK_NULL_HANDLE; - } - - for(size_t i = g_Framebuffers.size(); i--; ) - vkDestroyFramebuffer(g_hDevice, g_Framebuffers[i], nullptr); - g_Framebuffers.clear(); - - if(g_hDepthImageView != VK_NULL_HANDLE) - { - vkDestroyImageView(g_hDevice, g_hDepthImageView, nullptr); - g_hDepthImageView = VK_NULL_HANDLE; - } - if(g_hDepthImage != VK_NULL_HANDLE) - { - vmaDestroyImage(g_hAllocator, g_hDepthImage, g_hDepthImageAlloc); - g_hDepthImage = VK_NULL_HANDLE; - } - - if(g_hPipeline != VK_NULL_HANDLE) - { - vkDestroyPipeline(g_hDevice, g_hPipeline, nullptr); - g_hPipeline = VK_NULL_HANDLE; - } - - if(g_hRenderPass != VK_NULL_HANDLE) - { - vkDestroyRenderPass(g_hDevice, g_hRenderPass, nullptr); - g_hRenderPass = VK_NULL_HANDLE; - } - - if(g_hPipelineLayout != VK_NULL_HANDLE) - { - vkDestroyPipelineLayout(g_hDevice, g_hPipelineLayout, nullptr); - g_hPipelineLayout = VK_NULL_HANDLE; - } - - for(size_t i = g_SwapchainImageViews.size(); i--; ) - vkDestroyImageView(g_hDevice, g_SwapchainImageViews[i], nullptr); - g_SwapchainImageViews.clear(); - - if(destroyActualSwapchain && (g_hSwapchain != VK_NULL_HANDLE)) - { - vkDestroySwapchainKHR(g_hDevice, g_hSwapchain, nullptr); - g_hSwapchain = VK_NULL_HANDLE; - } -} - -static void InitializeApplication() -{ - uint32_t instanceLayerPropCount = 0; - ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, nullptr) ); - std::vector instanceLayerProps(instanceLayerPropCount); - if(instanceLayerPropCount > 0) - { - ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, instanceLayerProps.data()) ); - } - - if(g_EnableValidationLayer == true) - { - if(IsLayerSupported(instanceLayerProps.data(), instanceLayerProps.size(), VALIDATION_LAYER_NAME) == false) - { - printf("Layer \"%s\" not supported.", VALIDATION_LAYER_NAME); - g_EnableValidationLayer = false; - } - } - - std::vector instanceExtensions; - instanceExtensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME); - instanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); - - std::vector instanceLayers; - if(g_EnableValidationLayer == true) - { - instanceLayers.push_back(VALIDATION_LAYER_NAME); - instanceExtensions.push_back("VK_EXT_debug_report"); - } - - VkApplicationInfo appInfo = { VK_STRUCTURE_TYPE_APPLICATION_INFO }; - appInfo.pApplicationName = APP_TITLE_A; - appInfo.applicationVersion = VK_MAKE_VERSION(1, 0, 0); - appInfo.pEngineName = "Adam Sawicki Engine"; - appInfo.engineVersion = VK_MAKE_VERSION(1, 0, 0); - appInfo.apiVersion = VK_API_VERSION_1_0; - - VkInstanceCreateInfo instInfo = { VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO }; - instInfo.pApplicationInfo = &appInfo; - instInfo.enabledExtensionCount = static_cast(instanceExtensions.size()); - instInfo.ppEnabledExtensionNames = instanceExtensions.data(); - instInfo.enabledLayerCount = static_cast(instanceLayers.size()); - instInfo.ppEnabledLayerNames = instanceLayers.data(); - - ERR_GUARD_VULKAN( vkCreateInstance(&instInfo, NULL, &g_hVulkanInstance) ); - - // Create VkSurfaceKHR. - VkWin32SurfaceCreateInfoKHR surfaceInfo = { VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR }; - surfaceInfo.hinstance = g_hAppInstance; - surfaceInfo.hwnd = g_hWnd; - VkResult result = vkCreateWin32SurfaceKHR(g_hVulkanInstance, &surfaceInfo, NULL, &g_hSurface); - assert(result == VK_SUCCESS); - - if(g_EnableValidationLayer == true) - RegisterDebugCallbacks(); - - // Find physical device - - uint32_t deviceCount = 0; - ERR_GUARD_VULKAN( vkEnumeratePhysicalDevices(g_hVulkanInstance, &deviceCount, nullptr) ); - assert(deviceCount > 0); - - std::vector physicalDevices(deviceCount); - ERR_GUARD_VULKAN( vkEnumeratePhysicalDevices(g_hVulkanInstance, &deviceCount, physicalDevices.data()) ); - - g_hPhysicalDevice = physicalDevices[0]; - - // Query for features - - VkPhysicalDeviceProperties physicalDeviceProperties = {}; - vkGetPhysicalDeviceProperties(g_hPhysicalDevice, &physicalDeviceProperties); - - //VkPhysicalDeviceFeatures physicalDeviceFreatures = {}; - //vkGetPhysicalDeviceFeatures(g_PhysicalDevice, &physicalDeviceFreatures); - - // Find queue family index - - uint32_t queueFamilyCount = 0; - vkGetPhysicalDeviceQueueFamilyProperties(g_hPhysicalDevice, &queueFamilyCount, nullptr); - assert(queueFamilyCount > 0); - std::vector queueFamilies(queueFamilyCount); - vkGetPhysicalDeviceQueueFamilyProperties(g_hPhysicalDevice, &queueFamilyCount, queueFamilies.data()); - for(uint32_t i = 0; - (i < queueFamilyCount) && - (g_GraphicsQueueFamilyIndex == UINT_MAX || g_PresentQueueFamilyIndex == UINT_MAX); - ++i) - { - if(queueFamilies[i].queueCount > 0) - { - if((g_GraphicsQueueFamilyIndex != 0) && - ((queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0)) - { - g_GraphicsQueueFamilyIndex = i; - } - - VkBool32 surfaceSupported = 0; - VkResult res = vkGetPhysicalDeviceSurfaceSupportKHR(g_hPhysicalDevice, i, g_hSurface, &surfaceSupported); - if((res >= 0) && (surfaceSupported == VK_TRUE)) - { - g_PresentQueueFamilyIndex = i; - } - } - } - assert(g_GraphicsQueueFamilyIndex != UINT_MAX); - - // Create logical device - - const float queuePriority = 1.f; - - VkDeviceQueueCreateInfo deviceQueueCreateInfo[2] = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO }; - deviceQueueCreateInfo[0].queueFamilyIndex = g_GraphicsQueueFamilyIndex; - deviceQueueCreateInfo[0].queueCount = 1; - deviceQueueCreateInfo[0].pQueuePriorities = &queuePriority; - deviceQueueCreateInfo[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; - deviceQueueCreateInfo[1].queueFamilyIndex = g_PresentQueueFamilyIndex; - deviceQueueCreateInfo[1].queueCount = 1; - deviceQueueCreateInfo[1].pQueuePriorities = &queuePriority; - - VkPhysicalDeviceFeatures deviceFeatures = {}; - deviceFeatures.fillModeNonSolid = VK_TRUE; - deviceFeatures.samplerAnisotropy = VK_TRUE; - - // Determine list of device extensions to enable. - std::vector enabledDeviceExtensions; - enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); - { - uint32_t propertyCount = 0; - ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(g_hPhysicalDevice, nullptr, &propertyCount, nullptr) ); - - if(propertyCount) - { - std::vector properties{propertyCount}; - ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(g_hPhysicalDevice, nullptr, &propertyCount, properties.data()) ); - - for(uint32_t i = 0; i < propertyCount; ++i) - { - if(strcmp(properties[i].extensionName, VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME) == 0) - { - enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); - VK_KHR_get_memory_requirements2_enabled = true; - } - else if(strcmp(properties[i].extensionName, VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME) == 0) - { - enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); - VK_KHR_dedicated_allocation_enabled = true; - } - } - } - } - - VkDeviceCreateInfo deviceCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO }; - deviceCreateInfo.enabledLayerCount = 0; - deviceCreateInfo.ppEnabledLayerNames = nullptr; - deviceCreateInfo.enabledExtensionCount = (uint32_t)enabledDeviceExtensions.size(); - deviceCreateInfo.ppEnabledExtensionNames = !enabledDeviceExtensions.empty() ? enabledDeviceExtensions.data() : nullptr; - deviceCreateInfo.queueCreateInfoCount = g_PresentQueueFamilyIndex != g_GraphicsQueueFamilyIndex ? 2 : 1; - deviceCreateInfo.pQueueCreateInfos = deviceQueueCreateInfo; - deviceCreateInfo.pEnabledFeatures = &deviceFeatures; - - ERR_GUARD_VULKAN( vkCreateDevice(g_hPhysicalDevice, &deviceCreateInfo, nullptr, &g_hDevice) ); - - // Create memory allocator - - VmaAllocatorCreateInfo allocatorInfo = {}; - allocatorInfo.physicalDevice = g_hPhysicalDevice; - allocatorInfo.device = g_hDevice; - - if(VK_KHR_dedicated_allocation_enabled) - { - allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; - } - - VkAllocationCallbacks cpuAllocationCallbacks = {}; - if(USE_CUSTOM_CPU_ALLOCATION_CALLBACKS) - { - cpuAllocationCallbacks.pUserData = CUSTOM_CPU_ALLOCATION_CALLBACK_USER_DATA; - cpuAllocationCallbacks.pfnAllocation = &CustomCpuAllocation; - cpuAllocationCallbacks.pfnReallocation = &CustomCpuReallocation; - cpuAllocationCallbacks.pfnFree = &CustomCpuFree; - allocatorInfo.pAllocationCallbacks = &cpuAllocationCallbacks; - } - - ERR_GUARD_VULKAN( vmaCreateAllocator(&allocatorInfo, &g_hAllocator) ); - - // Retrieve queue (doesn't need to be destroyed) - - vkGetDeviceQueue(g_hDevice, g_GraphicsQueueFamilyIndex, 0, &g_hGraphicsQueue); - vkGetDeviceQueue(g_hDevice, g_PresentQueueFamilyIndex, 0, &g_hPresentQueue); - assert(g_hGraphicsQueue); - assert(g_hPresentQueue); - - // Create command pool - - VkCommandPoolCreateInfo commandPoolInfo = { VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO }; - commandPoolInfo.queueFamilyIndex = g_GraphicsQueueFamilyIndex; - commandPoolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; - ERR_GUARD_VULKAN( vkCreateCommandPool(g_hDevice, &commandPoolInfo, nullptr, &g_hCommandPool) ); - - VkCommandBufferAllocateInfo commandBufferInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO }; - commandBufferInfo.commandPool = g_hCommandPool; - commandBufferInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - commandBufferInfo.commandBufferCount = COMMAND_BUFFER_COUNT; - ERR_GUARD_VULKAN( vkAllocateCommandBuffers(g_hDevice, &commandBufferInfo, g_MainCommandBuffers) ); - - VkFenceCreateInfo fenceInfo = { VK_STRUCTURE_TYPE_FENCE_CREATE_INFO }; - fenceInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT; - for(size_t i = 0; i < COMMAND_BUFFER_COUNT; ++i) - { - ERR_GUARD_VULKAN( vkCreateFence(g_hDevice, &fenceInfo, nullptr, &g_MainCommandBufferExecutedFances[i]) ); - } - - commandBufferInfo.commandBufferCount = 1; - ERR_GUARD_VULKAN( vkAllocateCommandBuffers(g_hDevice, &commandBufferInfo, &g_hTemporaryCommandBuffer) ); - - // Create texture sampler - - VkSamplerCreateInfo samplerInfo = { VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO }; - samplerInfo.magFilter = VK_FILTER_LINEAR; - samplerInfo.minFilter = VK_FILTER_LINEAR; - samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_REPEAT; - samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_REPEAT; - samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT; - samplerInfo.anisotropyEnable = VK_TRUE; - samplerInfo.maxAnisotropy = 16; - samplerInfo.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK; - samplerInfo.unnormalizedCoordinates = VK_FALSE; - samplerInfo.compareEnable = VK_FALSE; - samplerInfo.compareOp = VK_COMPARE_OP_ALWAYS; - samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR; - samplerInfo.mipLodBias = 0.f; - samplerInfo.minLod = 0.f; - samplerInfo.maxLod = FLT_MAX; - ERR_GUARD_VULKAN( vkCreateSampler(g_hDevice, &samplerInfo, nullptr, &g_hSampler) ); - - CreateTexture(128, 128); - CreateMesh(); - - VkDescriptorSetLayoutBinding samplerLayoutBinding = {}; - samplerLayoutBinding.binding = 1; - samplerLayoutBinding.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; - samplerLayoutBinding.descriptorCount = 1; - samplerLayoutBinding.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; - - VkDescriptorSetLayoutCreateInfo descriptorSetLayoutInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO }; - descriptorSetLayoutInfo.bindingCount = 1; - descriptorSetLayoutInfo.pBindings = &samplerLayoutBinding; - ERR_GUARD_VULKAN( vkCreateDescriptorSetLayout(g_hDevice, &descriptorSetLayoutInfo, nullptr, &g_hDescriptorSetLayout) ); - - // Create descriptor pool - - VkDescriptorPoolSize descriptorPoolSizes[2]; - ZeroMemory(descriptorPoolSizes, sizeof(descriptorPoolSizes)); - descriptorPoolSizes[0].type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; - descriptorPoolSizes[0].descriptorCount = 1; - descriptorPoolSizes[1].type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; - descriptorPoolSizes[1].descriptorCount = 1; - - VkDescriptorPoolCreateInfo descriptorPoolInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO }; - descriptorPoolInfo.poolSizeCount = (uint32_t)_countof(descriptorPoolSizes); - descriptorPoolInfo.pPoolSizes = descriptorPoolSizes; - descriptorPoolInfo.maxSets = 1; - ERR_GUARD_VULKAN( vkCreateDescriptorPool(g_hDevice, &descriptorPoolInfo, nullptr, &g_hDescriptorPool) ); - - // Create descriptor set layout - - VkDescriptorSetLayout descriptorSetLayouts[] = { g_hDescriptorSetLayout }; - VkDescriptorSetAllocateInfo descriptorSetInfo = { VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO }; - descriptorSetInfo.descriptorPool = g_hDescriptorPool; - descriptorSetInfo.descriptorSetCount = 1; - descriptorSetInfo.pSetLayouts = descriptorSetLayouts; - ERR_GUARD_VULKAN( vkAllocateDescriptorSets(g_hDevice, &descriptorSetInfo, &g_hDescriptorSet) ); - - VkDescriptorImageInfo descriptorImageInfo = {}; - descriptorImageInfo.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - descriptorImageInfo.imageView = g_hTextureImageView; - descriptorImageInfo.sampler = g_hSampler; - - VkWriteDescriptorSet writeDescriptorSet = { VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET }; - writeDescriptorSet.dstSet = g_hDescriptorSet; - writeDescriptorSet.dstBinding = 1; - writeDescriptorSet.dstArrayElement = 0; - writeDescriptorSet.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; - writeDescriptorSet.descriptorCount = 1; - writeDescriptorSet.pImageInfo = &descriptorImageInfo; - - vkUpdateDescriptorSets(g_hDevice, 1, &writeDescriptorSet, 0, nullptr); - - CreateSwapchain(); -} - -static void FinalizeApplication() -{ - vkDeviceWaitIdle(g_hDevice); - - DestroySwapchain(true); - - if(g_hDescriptorPool != VK_NULL_HANDLE) - { - vkDestroyDescriptorPool(g_hDevice, g_hDescriptorPool, nullptr); - g_hDescriptorPool = VK_NULL_HANDLE; - } - - if(g_hDescriptorSetLayout != VK_NULL_HANDLE) - { - vkDestroyDescriptorSetLayout(g_hDevice, g_hDescriptorSetLayout, nullptr); - g_hDescriptorSetLayout = VK_NULL_HANDLE; - } - - if(g_hTextureImageView != VK_NULL_HANDLE) - { - vkDestroyImageView(g_hDevice, g_hTextureImageView, nullptr); - g_hTextureImageView = VK_NULL_HANDLE; - } - if(g_hTextureImage != VK_NULL_HANDLE) - { - vmaDestroyImage(g_hAllocator, g_hTextureImage, g_hTextureImageAlloc); - g_hTextureImage = VK_NULL_HANDLE; - } - - if(g_hIndexBuffer != VK_NULL_HANDLE) - { - vmaDestroyBuffer(g_hAllocator, g_hIndexBuffer, g_hIndexBufferAlloc); - g_hIndexBuffer = VK_NULL_HANDLE; - } - if(g_hVertexBuffer != VK_NULL_HANDLE) - { - vmaDestroyBuffer(g_hAllocator, g_hVertexBuffer, g_hVertexBufferAlloc); - g_hVertexBuffer = VK_NULL_HANDLE; - } - - if(g_hSampler != VK_NULL_HANDLE) - { - vkDestroySampler(g_hDevice, g_hSampler, nullptr); - g_hSampler = VK_NULL_HANDLE; - } - - for(size_t i = COMMAND_BUFFER_COUNT; i--; ) - { - if(g_MainCommandBufferExecutedFances[i] != VK_NULL_HANDLE) - { - vkDestroyFence(g_hDevice, g_MainCommandBufferExecutedFances[i], nullptr); - g_MainCommandBufferExecutedFances[i] = VK_NULL_HANDLE; - } - } - if(g_MainCommandBuffers[0] != VK_NULL_HANDLE) - { - vkFreeCommandBuffers(g_hDevice, g_hCommandPool, COMMAND_BUFFER_COUNT, g_MainCommandBuffers); - ZeroMemory(g_MainCommandBuffers, sizeof(g_MainCommandBuffers)); - } - if(g_hTemporaryCommandBuffer != VK_NULL_HANDLE) - { - vkFreeCommandBuffers(g_hDevice, g_hCommandPool, 1, &g_hTemporaryCommandBuffer); - g_hTemporaryCommandBuffer = VK_NULL_HANDLE; - } - - if(g_hCommandPool != VK_NULL_HANDLE) - { - vkDestroyCommandPool(g_hDevice, g_hCommandPool, nullptr); - g_hCommandPool = VK_NULL_HANDLE; - } - - if(g_hAllocator != VK_NULL_HANDLE) - { - vmaDestroyAllocator(g_hAllocator); - g_hAllocator = nullptr; - } - - if(g_hDevice != VK_NULL_HANDLE) - { - vkDestroyDevice(g_hDevice, nullptr); - g_hDevice = nullptr; - } - - if(g_pvkDestroyDebugReportCallbackEXT && g_hCallback != VK_NULL_HANDLE) - { - g_pvkDestroyDebugReportCallbackEXT(g_hVulkanInstance, g_hCallback, nullptr); - g_hCallback = VK_NULL_HANDLE; - } - - if(g_hSurface != VK_NULL_HANDLE) - { - vkDestroySurfaceKHR(g_hVulkanInstance, g_hSurface, NULL); - g_hSurface = VK_NULL_HANDLE; - } - - if(g_hVulkanInstance != VK_NULL_HANDLE) - { - vkDestroyInstance(g_hVulkanInstance, NULL); - g_hVulkanInstance = VK_NULL_HANDLE; - } -} - -static void PrintAllocatorStats() -{ -#if VMA_STATS_STRING_ENABLED - char* statsString = nullptr; - vmaBuildStatsString(g_hAllocator, &statsString, true); - printf("%s\n", statsString); - vmaFreeStatsString(g_hAllocator, statsString); -#endif -} - -static void RecreateSwapChain() -{ - vkDeviceWaitIdle(g_hDevice); - DestroySwapchain(false); - CreateSwapchain(); -} - -static void DrawFrame() -{ - // Begin main command buffer - size_t cmdBufIndex = (g_NextCommandBufferIndex++) % COMMAND_BUFFER_COUNT; - VkCommandBuffer hCommandBuffer = g_MainCommandBuffers[cmdBufIndex]; - VkFence hCommandBufferExecutedFence = g_MainCommandBufferExecutedFances[cmdBufIndex]; - - ERR_GUARD_VULKAN( vkWaitForFences(g_hDevice, 1, &hCommandBufferExecutedFence, VK_TRUE, UINT64_MAX) ); - ERR_GUARD_VULKAN( vkResetFences(g_hDevice, 1, &hCommandBufferExecutedFence) ); - - VkCommandBufferBeginInfo commandBufferBeginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; - commandBufferBeginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - ERR_GUARD_VULKAN( vkBeginCommandBuffer(hCommandBuffer, &commandBufferBeginInfo) ); - - // Acquire swapchain image - uint32_t imageIndex = 0; - VkResult res = vkAcquireNextImageKHR(g_hDevice, g_hSwapchain, UINT64_MAX, g_hImageAvailableSemaphore, VK_NULL_HANDLE, &imageIndex); - if(res == VK_ERROR_OUT_OF_DATE_KHR) - { - RecreateSwapChain(); - return; - } - else if(res < 0) - { - ERR_GUARD_VULKAN(res); - } - - // Record geometry pass - - VkClearValue clearValues[2]; - ZeroMemory(clearValues, sizeof(clearValues)); - clearValues[0].color.float32[0] = 0.25f; - clearValues[0].color.float32[1] = 0.25f; - clearValues[0].color.float32[2] = 0.5f; - clearValues[0].color.float32[3] = 1.0f; - clearValues[1].depthStencil.depth = 1.0f; - - VkRenderPassBeginInfo renderPassBeginInfo = { VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO }; - renderPassBeginInfo.renderPass = g_hRenderPass; - renderPassBeginInfo.framebuffer = g_Framebuffers[imageIndex]; - renderPassBeginInfo.renderArea.offset.x = 0; - renderPassBeginInfo.renderArea.offset.y = 0; - renderPassBeginInfo.renderArea.extent = g_Extent; - renderPassBeginInfo.clearValueCount = (uint32_t)_countof(clearValues); - renderPassBeginInfo.pClearValues = clearValues; - vkCmdBeginRenderPass(hCommandBuffer, &renderPassBeginInfo, VK_SUBPASS_CONTENTS_INLINE); - - vkCmdBindPipeline( - hCommandBuffer, - VK_PIPELINE_BIND_POINT_GRAPHICS, - g_hPipeline); - - mat4 view = mat4::LookAt( - vec3(0.f, 0.f, 0.f), - vec3(0.f, -2.f, 4.f), - vec3(0.f, 1.f, 0.f)); - mat4 proj = mat4::Perspective( - 1.0471975511966f, // 60 degrees - (float)g_Extent.width / (float)g_Extent.height, - 0.1f, - 1000.f); - mat4 viewProj = view * proj; - - vkCmdBindDescriptorSets( - hCommandBuffer, - VK_PIPELINE_BIND_POINT_GRAPHICS, - g_hPipelineLayout, - 0, - 1, - &g_hDescriptorSet, - 0, - nullptr); - - float rotationAngle = (float)GetTickCount() * 0.001f * (float)PI * 0.2f; - mat4 model = mat4::RotationY(rotationAngle); - - UniformBufferObject ubo = {}; - ubo.ModelViewProj = model * viewProj; - vkCmdPushConstants(hCommandBuffer, g_hPipelineLayout, VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(UniformBufferObject), &ubo); - - VkBuffer vertexBuffers[] = { g_hVertexBuffer }; - VkDeviceSize offsets[] = { 0 }; - vkCmdBindVertexBuffers(hCommandBuffer, 0, 1, vertexBuffers, offsets); - - vkCmdBindIndexBuffer(hCommandBuffer, g_hIndexBuffer, 0, VK_INDEX_TYPE_UINT16); - - vkCmdDrawIndexed(hCommandBuffer, g_IndexCount, 1, 0, 0, 0); - - vkCmdEndRenderPass(hCommandBuffer); - - vkEndCommandBuffer(hCommandBuffer); - - // Submit command buffer - - VkSemaphore submitWaitSemaphores[] = { g_hImageAvailableSemaphore }; - VkPipelineStageFlags submitWaitStages[] = { VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT }; - VkSemaphore submitSignalSemaphores[] = { g_hRenderFinishedSemaphore }; - VkSubmitInfo submitInfo = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; - submitInfo.waitSemaphoreCount = 1; - submitInfo.pWaitSemaphores = submitWaitSemaphores; - submitInfo.pWaitDstStageMask = submitWaitStages; - submitInfo.commandBufferCount = 1; - submitInfo.pCommandBuffers = &hCommandBuffer; - submitInfo.signalSemaphoreCount = _countof(submitSignalSemaphores); - submitInfo.pSignalSemaphores = submitSignalSemaphores; - ERR_GUARD_VULKAN( vkQueueSubmit(g_hGraphicsQueue, 1, &submitInfo, hCommandBufferExecutedFence) ); - - VkSemaphore presentWaitSemaphores[] = { g_hRenderFinishedSemaphore }; - - VkSwapchainKHR swapchains[] = { g_hSwapchain }; - VkPresentInfoKHR presentInfo = { VK_STRUCTURE_TYPE_PRESENT_INFO_KHR }; - presentInfo.waitSemaphoreCount = _countof(presentWaitSemaphores); - presentInfo.pWaitSemaphores = presentWaitSemaphores; - presentInfo.swapchainCount = 1; - presentInfo.pSwapchains = swapchains; - presentInfo.pImageIndices = &imageIndex; - presentInfo.pResults = nullptr; - res = vkQueuePresentKHR(g_hPresentQueue, &presentInfo); - if(res == VK_ERROR_OUT_OF_DATE_KHR) - { - RecreateSwapChain(); - } - else - ERR_GUARD_VULKAN(res); -} - -static void HandlePossibleSizeChange() -{ - RECT clientRect; - GetClientRect(g_hWnd, &clientRect); - LONG newSizeX = clientRect.right - clientRect.left; - LONG newSizeY = clientRect.bottom - clientRect.top; - if((newSizeX > 0) && - (newSizeY > 0) && - ((newSizeX != g_SizeX) || (newSizeY != g_SizeY))) - { - g_SizeX = newSizeX; - g_SizeY = newSizeY; - - RecreateSwapChain(); - } -} - -static LRESULT WINAPI WndProc(HWND hWnd, UINT msg, WPARAM wParam, LPARAM lParam) -{ - switch(msg) - { - case WM_CREATE: - // This is intentionally assigned here because we are now inside CreateWindow, before it returns. - g_hWnd = hWnd; - InitializeApplication(); - PrintAllocatorStats(); - return 0; - - case WM_DESTROY: - FinalizeApplication(); - PostQuitMessage(0); - return 0; - - // This prevents app from freezing when left Alt is pressed - // (which normally enters modal menu loop). - case WM_SYSKEYDOWN: - case WM_SYSKEYUP: - return 0; - - case WM_SIZE: - if((wParam == SIZE_MAXIMIZED) || (wParam == SIZE_RESTORED)) - HandlePossibleSizeChange(); - return 0; - - case WM_EXITSIZEMOVE: - HandlePossibleSizeChange(); - return 0; - - case WM_KEYDOWN: - switch(wParam) - { - case VK_ESCAPE: - PostMessage(hWnd, WM_CLOSE, 0, 0); - break; - case 'T': - Test(); - break; - } - return 0; - - default: - break; - } - - return DefWindowProc(hWnd, msg, wParam, lParam); -} - -int main() -{ - g_hAppInstance = (HINSTANCE)GetModuleHandle(NULL); - - WNDCLASSEX wndClassDesc = { sizeof(WNDCLASSEX) }; - wndClassDesc.style = CS_VREDRAW | CS_HREDRAW | CS_DBLCLKS; - wndClassDesc.hbrBackground = NULL; - wndClassDesc.hCursor = LoadCursor(NULL, IDC_CROSS); - wndClassDesc.hIcon = LoadIcon(NULL, IDI_APPLICATION); - wndClassDesc.hInstance = g_hAppInstance; - wndClassDesc.lpfnWndProc = WndProc; - wndClassDesc.lpszClassName = WINDOW_CLASS_NAME; - - const ATOM hWndClass = RegisterClassEx(&wndClassDesc); - assert(hWndClass); - - const DWORD style = WS_VISIBLE | WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU | WS_MINIMIZEBOX | WS_MAXIMIZEBOX | WS_THICKFRAME; - const DWORD exStyle = 0; - - RECT rect = { 0, 0, g_SizeX, g_SizeY }; - AdjustWindowRectEx(&rect, style, FALSE, exStyle); - - CreateWindowEx( - exStyle, WINDOW_CLASS_NAME, APP_TITLE_W, style, - CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, - NULL, NULL, g_hAppInstance, NULL); - - MSG msg; - for(;;) - { - if(PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) - { - if(msg.message == WM_QUIT) - break; - TranslateMessage(&msg); - DispatchMessage(&msg); - } - if(g_hDevice != VK_NULL_HANDLE) - DrawFrame(); - } - - return 0; -} -#endif - static const int RESULT_EXCEPTION = -1000; static const int RESULT_ERROR_COMMAND_LINE = -1; static const int RESULT_ERROR_SOURCE_FILE = -2; @@ -1857,6 +123,58 @@ void CsvSplit::Set(const StrRange& line, size_t maxCount) //////////////////////////////////////////////////////////////////////////////// // class Player +static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_LUNARG_standard_validation"; + +static bool g_MemoryAliasingWarningEnabled = true; +static bool g_EnableValidationLayer = true; +static bool VK_KHR_get_memory_requirements2_enabled = false; +static bool VK_KHR_dedicated_allocation_enabled = false; + +static VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( + VkDebugReportFlagsEXT flags, + VkDebugReportObjectTypeEXT objectType, + uint64_t object, + size_t location, + int32_t messageCode, + const char* pLayerPrefix, + const char* pMessage, + void* pUserData) +{ + // "Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug." + if(!g_MemoryAliasingWarningEnabled && flags == VK_DEBUG_REPORT_WARNING_BIT_EXT && + (strstr(pMessage, " is aliased with non-linear ") || strstr(pMessage, " is aliased with linear "))) + { + return VK_FALSE; + } + + // Ignoring because when VK_KHR_dedicated_allocation extension is enabled, + // vkGetBufferMemoryRequirements2KHR function is used instead, while Validation + // Layer seems to be unaware of it. + if (strstr(pMessage, "but vkGetBufferMemoryRequirements() has not been called on that buffer") != nullptr) + { + return VK_FALSE; + } + if (strstr(pMessage, "but vkGetImageMemoryRequirements() has not been called on that image") != nullptr) + { + return VK_FALSE; + } + + printf("%s \xBA %s\n", pLayerPrefix, pMessage); + + return VK_FALSE; +} + +static bool IsLayerSupported(const VkLayerProperties* pProps, size_t propCount, const char* pLayerName) +{ + const VkLayerProperties* propsEnd = pProps + propCount; + return std::find_if( + pProps, + propsEnd, + [pLayerName](const VkLayerProperties& prop) -> bool { + return strcmp(pLayerName, prop.layerName) == 0; + }) != propsEnd; +} + class Player { public: @@ -1873,6 +191,21 @@ private: bool IssueWarning() { return m_WarningCount++ < MAX_WARNINGS_TO_SHOW; } size_t m_WarningCount = 0; + + VkInstance m_VulkanInstance = VK_NULL_HANDLE; + VkPhysicalDevice m_PhysicalDevice = VK_NULL_HANDLE; + uint32_t m_GraphicsQueueFamilyIndex = UINT_MAX; + VkDevice m_Device = VK_NULL_HANDLE; + VmaAllocator m_Allocator = VK_NULL_HANDLE; + + PFN_vkCreateDebugReportCallbackEXT m_pvkCreateDebugReportCallbackEXT; + PFN_vkDebugReportMessageEXT m_pvkDebugReportMessageEXT; + PFN_vkDestroyDebugReportCallbackEXT m_pvkDestroyDebugReportCallbackEXT; + VkDebugReportCallbackEXT m_hCallback; + + void InitVulkan(); + void FinalizeVulkan(); + void RegisterDebugCallbacks(); }; Player::Player() @@ -1881,10 +214,13 @@ Player::Player() void Player::Init() { + InitVulkan(); } Player::~Player() { + FinalizeVulkan(); + if(m_WarningCount > 0) printf("WARNING: %zu more warnings not shown.\n", m_WarningCount - MAX_WARNINGS_TO_SHOW); } @@ -1896,6 +232,7 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) if(csvSplit.GetCount() >= 4) { + // TODO } else { @@ -1904,6 +241,211 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) } } +void Player::InitVulkan() +{ + printf("Initializing Vulkan...\n"); + + uint32_t instanceLayerPropCount = 0; + ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, nullptr) ); + std::vector instanceLayerProps(instanceLayerPropCount); + if(instanceLayerPropCount > 0) + { + ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, instanceLayerProps.data()) ); + } + + if(g_EnableValidationLayer == true) + { + if(IsLayerSupported(instanceLayerProps.data(), instanceLayerProps.size(), VALIDATION_LAYER_NAME) == false) + { + printf("WARNING: Layer \"%s\" not supported.\n", VALIDATION_LAYER_NAME); + g_EnableValidationLayer = false; + } + } + + std::vector instanceExtensions; + //instanceExtensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME); + //instanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); + + std::vector instanceLayers; + if(g_EnableValidationLayer == true) + { + instanceLayers.push_back(VALIDATION_LAYER_NAME); + instanceExtensions.push_back("VK_EXT_debug_report"); + } + + VkApplicationInfo appInfo = { VK_STRUCTURE_TYPE_APPLICATION_INFO }; + appInfo.pApplicationName = "VmaReplay"; + appInfo.applicationVersion = VK_MAKE_VERSION(1, 0, 0); + appInfo.pEngineName = "Vulkan Memory Allocator"; + appInfo.engineVersion = VK_MAKE_VERSION(1, 0, 0); + appInfo.apiVersion = VK_API_VERSION_1_0; + + VkInstanceCreateInfo instInfo = { VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO }; + instInfo.pApplicationInfo = &appInfo; + instInfo.enabledExtensionCount = (uint32_t)instanceExtensions.size(); + instInfo.ppEnabledExtensionNames = instanceExtensions.data(); + instInfo.enabledLayerCount = (uint32_t)instanceLayers.size(); + instInfo.ppEnabledLayerNames = instanceLayers.data(); + + ERR_GUARD_VULKAN( vkCreateInstance(&instInfo, NULL, &m_VulkanInstance) ); + + // Find physical device + + uint32_t deviceCount = 0; + ERR_GUARD_VULKAN( vkEnumeratePhysicalDevices(m_VulkanInstance, &deviceCount, nullptr) ); + assert(deviceCount > 0); + + std::vector physicalDevices(deviceCount); + ERR_GUARD_VULKAN( vkEnumeratePhysicalDevices(m_VulkanInstance, &deviceCount, physicalDevices.data()) ); + + m_PhysicalDevice = physicalDevices[0]; + + // Find queue family index + + uint32_t queueFamilyCount = 0; + vkGetPhysicalDeviceQueueFamilyProperties(m_PhysicalDevice, &queueFamilyCount, nullptr); + assert(queueFamilyCount > 0); + std::vector queueFamilies(queueFamilyCount); + vkGetPhysicalDeviceQueueFamilyProperties(m_PhysicalDevice, &queueFamilyCount, queueFamilies.data()); + for(uint32_t i = 0; i < queueFamilyCount; ++i) + { + if(queueFamilies[i].queueCount > 0) + { + if((queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) + { + m_GraphicsQueueFamilyIndex = i; + break; + } + } + } + assert(m_GraphicsQueueFamilyIndex != UINT_MAX); + + // Create logical device + + const float queuePriority = 1.f; + + VkDeviceQueueCreateInfo deviceQueueCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO }; + deviceQueueCreateInfo.queueFamilyIndex = m_GraphicsQueueFamilyIndex; + deviceQueueCreateInfo.queueCount = 1; + deviceQueueCreateInfo.pQueuePriorities = &queuePriority; + + // Enable something what may interact with memory/buffer/image support. + VkPhysicalDeviceFeatures deviceFeatures = {}; + deviceFeatures.fullDrawIndexUint32 = VK_TRUE; + deviceFeatures.imageCubeArray = VK_TRUE; + deviceFeatures.geometryShader = VK_TRUE; + deviceFeatures.tessellationShader = VK_TRUE; + deviceFeatures.multiDrawIndirect = VK_TRUE; + deviceFeatures.textureCompressionBC = VK_TRUE; + + // Determine list of device extensions to enable. + std::vector enabledDeviceExtensions; + enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); + { + uint32_t propertyCount = 0; + ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(m_PhysicalDevice, nullptr, &propertyCount, nullptr) ); + + if(propertyCount) + { + std::vector properties{propertyCount}; + ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(m_PhysicalDevice, nullptr, &propertyCount, properties.data()) ); + + for(uint32_t i = 0; i < propertyCount; ++i) + { + if(strcmp(properties[i].extensionName, VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME) == 0) + { + enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); + VK_KHR_get_memory_requirements2_enabled = true; + } + else if(strcmp(properties[i].extensionName, VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME) == 0) + { + enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); + VK_KHR_dedicated_allocation_enabled = true; + } + } + } + } + + VkDeviceCreateInfo deviceCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO }; + deviceCreateInfo.enabledExtensionCount = (uint32_t)enabledDeviceExtensions.size(); + deviceCreateInfo.ppEnabledExtensionNames = !enabledDeviceExtensions.empty() ? enabledDeviceExtensions.data() : nullptr; + deviceCreateInfo.queueCreateInfoCount = 1; + deviceCreateInfo.pQueueCreateInfos = &deviceQueueCreateInfo; + deviceCreateInfo.pEnabledFeatures = &deviceFeatures; + + ERR_GUARD_VULKAN( vkCreateDevice(m_PhysicalDevice, &deviceCreateInfo, nullptr, &m_Device) ); + + // Create memory allocator + + VmaAllocatorCreateInfo allocatorInfo = {}; + allocatorInfo.physicalDevice = m_PhysicalDevice; + allocatorInfo.device = m_Device; + + if(VK_KHR_dedicated_allocation_enabled) + { + allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; + } + + ERR_GUARD_VULKAN( vmaCreateAllocator(&allocatorInfo, &m_Allocator) ); +} + +void Player::FinalizeVulkan() +{ + vkDeviceWaitIdle(m_Device); + + if(m_Allocator != VK_NULL_HANDLE) + { + vmaDestroyAllocator(m_Allocator); + m_Allocator = nullptr; + } + + if(m_Device != VK_NULL_HANDLE) + { + vkDestroyDevice(m_Device, nullptr); + m_Device = nullptr; + } + + if(m_pvkDestroyDebugReportCallbackEXT && m_hCallback != VK_NULL_HANDLE) + { + m_pvkDestroyDebugReportCallbackEXT(m_VulkanInstance, m_hCallback, nullptr); + m_hCallback = VK_NULL_HANDLE; + } + + if(m_VulkanInstance != VK_NULL_HANDLE) + { + vkDestroyInstance(m_VulkanInstance, NULL); + m_VulkanInstance = VK_NULL_HANDLE; + } +} + +void Player::RegisterDebugCallbacks() +{ + m_pvkCreateDebugReportCallbackEXT = + reinterpret_cast + (vkGetInstanceProcAddr(m_VulkanInstance, "vkCreateDebugReportCallbackEXT")); + m_pvkDebugReportMessageEXT = + reinterpret_cast + (vkGetInstanceProcAddr(m_VulkanInstance, "vkDebugReportMessageEXT")); + m_pvkDestroyDebugReportCallbackEXT = + reinterpret_cast + (vkGetInstanceProcAddr(m_VulkanInstance, "vkDestroyDebugReportCallbackEXT")); + assert(m_pvkCreateDebugReportCallbackEXT); + assert(m_pvkDebugReportMessageEXT); + assert(m_pvkDestroyDebugReportCallbackEXT); + + VkDebugReportCallbackCreateInfoEXT callbackCreateInfo; + callbackCreateInfo.sType = VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT; + callbackCreateInfo.pNext = nullptr; + callbackCreateInfo.flags = //VK_DEBUG_REPORT_INFORMATION_BIT_EXT | + VK_DEBUG_REPORT_ERROR_BIT_EXT | + VK_DEBUG_REPORT_WARNING_BIT_EXT | + VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT /*| + VK_DEBUG_REPORT_DEBUG_BIT_EXT*/; + callbackCreateInfo.pfnCallback = &MyDebugReportCallback; + callbackCreateInfo.pUserData = nullptr; + + ERR_GUARD_VULKAN( m_pvkCreateDebugReportCallbackEXT(m_VulkanInstance, &callbackCreateInfo, nullptr, &m_hCallback) ); +} //////////////////////////////////////////////////////////////////////////////// // Main functions @@ -1990,6 +532,8 @@ static int ProcessFile(const char* filePath) result = RESULT_ERROR_SOURCE_FILE; } + printf("Done.\n"); + return result; } From ec5fecf51a5ad6921caf089f1d1f99cab00fd1be Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 16:20:34 +0200 Subject: [PATCH 07/51] Implemented playback of current format. --- src/VmaReplay/VmaReplay.cpp | 454 ++++++++++++++++++++++++++++++++++-- 1 file changed, 430 insertions(+), 24 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 09e38f7..6fc13b0 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -22,6 +22,7 @@ #include "VmaUsage.h" #include "Common.h" +#include static const int RESULT_EXCEPTION = -1000; static const int RESULT_ERROR_COMMAND_LINE = -1; @@ -36,6 +37,35 @@ struct StrRange size_t length() const { return end - beg; } }; +static inline bool StrRangeEq(const StrRange& lhs, const char* rhsSz) +{ + const size_t rhsLen = strlen(rhsSz); + return rhsLen == lhs.length() && + memcmp(lhs.beg, rhsSz, rhsLen) == 0; +} + +static inline bool StrRangeToUint(const StrRange& s, uint32_t& out) +{ + // TODO handle failure. + char* end = (char*)s.end; + out = (uint32_t)strtoul(s.beg, &end, 10); + return true; +} +static inline bool StrRangeToUint(const StrRange& s, uint64_t& out) +{ + // TODO handle failure. + char* end = (char*)s.end; + out = (uint64_t)strtoull(s.beg, &end, 10); + return true; +} +static inline bool StrRangeToPtr(const StrRange& s, uint64_t& out) +{ + // TODO handle failure. + char* end = (char*)s.end; + out = (uint64_t)strtoull(s.beg, &end, 16); + return true; +} + //////////////////////////////////////////////////////////////////////////////// // LineSplit class @@ -89,10 +119,11 @@ public: void Set(const StrRange& line, size_t maxCount = RANGE_COUNT_MAX); size_t GetCount() const { return m_Count; } - void GetRange(size_t index, const char*& outBeg, const char*& outEnd) const + StrRange GetRange(size_t index) const { - outBeg = m_Str + m_Ranges[index * 2]; - outEnd = m_Str + m_Ranges[index * 2 + 1]; + return StrRange { + m_Str + m_Ranges[index * 2], + m_Str + m_Ranges[index * 2 + 1] }; } private: @@ -175,6 +206,8 @@ static bool IsLayerSupported(const VkLayerProperties* pProps, size_t propCount, }) != propsEnd; } +static const size_t FIRST_PARAM_INDEX = 4; + class Player { public: @@ -185,10 +218,7 @@ public: void ExecuteLine(size_t lineNumber, const StrRange& line); private: - static const size_t MAX_WARNINGS_TO_SHOW = 16; - - // Increments warning counter. Returns true if warning message should be printed. - bool IssueWarning() { return m_WarningCount++ < MAX_WARNINGS_TO_SHOW; } + static const size_t MAX_WARNINGS_TO_SHOW = 64; size_t m_WarningCount = 0; @@ -203,9 +233,44 @@ private: PFN_vkDestroyDebugReportCallbackEXT m_pvkDestroyDebugReportCallbackEXT; VkDebugReportCallbackEXT m_hCallback; + uint32_t m_VmaFrameIndex = 0; + + // Any of these handles null means it was created in original but couldn't be created now. + struct Pool + { + VmaPool pool; + }; + struct Buffer + { + VkBuffer buffer; + VmaAllocation allocation; + }; + struct Image + { + VkImage image; + VmaAllocation allocation; + }; + + std::unordered_map m_Pools; + std::unordered_map m_Buffers; + std::unordered_map m_Images; + + // Increments warning counter. Returns true if warning message should be printed. + bool IssueWarning() { return m_WarningCount++ < MAX_WARNINGS_TO_SHOW; } + void InitVulkan(); void FinalizeVulkan(); void RegisterDebugCallbacks(); + + // If parmeter count doesn't match, issues warning and returns false. + bool ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound); + + void CreatePool(size_t lineNumber, const CsvSplit& csvSplit); + void DestroyPool(size_t lineNumber, const CsvSplit& csvSplit); + void CreateBuffer(size_t lineNumber, const CsvSplit& csvSplit); + void DestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit); + void CreateImage(size_t lineNumber, const CsvSplit& csvSplit); + void DestroyImage(size_t lineNumber, const CsvSplit& csvSplit); }; Player::Player() @@ -221,7 +286,7 @@ Player::~Player() { FinalizeVulkan(); - if(m_WarningCount > 0) + if(m_WarningCount > MAX_WARNINGS_TO_SHOW) printf("WARNING: %zu more warnings not shown.\n", m_WarningCount - MAX_WARNINGS_TO_SHOW); } @@ -230,9 +295,58 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) CsvSplit csvSplit; csvSplit.Set(line); - if(csvSplit.GetCount() >= 4) + if(csvSplit.GetCount() >= FIRST_PARAM_INDEX) { - // TODO + // Update VMA current frame index. + StrRange frameIndexStr = csvSplit.GetRange(2); + uint32_t frameIndex; + if(StrRangeToUint(frameIndexStr, frameIndex)) + { + if(frameIndex != m_VmaFrameIndex) + { + vmaSetCurrentFrameIndex(m_Allocator, frameIndex); + m_VmaFrameIndex = frameIndex; + } + } + else + { + if(IssueWarning()) + printf("Line %zu: Incorrect frame index.\n", lineNumber); + } + + StrRange functionName = csvSplit.GetRange(3); + + if(StrRangeEq(functionName, "vmaCreateAllocator")) + { + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 0, false)) + { + // Nothing to do. + } + } + else if(StrRangeEq(functionName, "vmaCreatePool")) + CreatePool(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaDestroyPool")) + DestroyPool(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaSetAllocationUserData")) + { + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 2, true)) + { + // Ignore for now. + } + } + else if(StrRangeEq(functionName, "vmaCreateBuffer")) + CreateBuffer(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaDestroyBuffer")) + DestroyBuffer(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaCreateImage")) + CreateImage(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaDestroyImage")) + DestroyImage(lineNumber, csvSplit); + else + { + if(IssueWarning()) + printf("Line %zu: Unknown function.\n", lineNumber); + } } else { @@ -289,6 +403,8 @@ void Player::InitVulkan() ERR_GUARD_VULKAN( vkCreateInstance(&instInfo, NULL, &m_VulkanInstance) ); + RegisterDebugCallbacks(); + // Find physical device uint32_t deviceCount = 0; @@ -340,7 +456,7 @@ void Player::InitVulkan() // Determine list of device extensions to enable. std::vector enabledDeviceExtensions; - enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); + //enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); { uint32_t propertyCount = 0; ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(m_PhysicalDevice, nullptr, &propertyCount, nullptr) ); @@ -391,6 +507,36 @@ void Player::InitVulkan() void Player::FinalizeVulkan() { + if(!m_Images.empty()) + { + printf("WARNING: Images not destroyed: %zu.\n", m_Images.size()); + + for(const auto it : m_Images) + { + vmaDestroyImage(m_Allocator, it.second.image, it.second.allocation); + } + } + + if(!m_Buffers.empty()) + { + printf("WARNING: Buffers not destroyed: %zu.\n", m_Buffers.size()); + + for(const auto it : m_Buffers) + { + vmaDestroyBuffer(m_Allocator, it.second.buffer, it.second.allocation); + } + } + + if(!m_Pools.empty()) + { + printf("WARNING: Pools not destroyed: %zu.\n", m_Pools.size()); + + for(const auto it : m_Pools) + { + vmaDestroyPool(m_Allocator, it.second.pool); + } + } + vkDeviceWaitIdle(m_Device); if(m_Allocator != VK_NULL_HANDLE) @@ -433,20 +579,287 @@ void Player::RegisterDebugCallbacks() assert(m_pvkDebugReportMessageEXT); assert(m_pvkDestroyDebugReportCallbackEXT); - VkDebugReportCallbackCreateInfoEXT callbackCreateInfo; - callbackCreateInfo.sType = VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT; - callbackCreateInfo.pNext = nullptr; + VkDebugReportCallbackCreateInfoEXT callbackCreateInfo = { VK_STRUCTURE_TYPE_DEBUG_REPORT_CREATE_INFO_EXT }; callbackCreateInfo.flags = //VK_DEBUG_REPORT_INFORMATION_BIT_EXT | VK_DEBUG_REPORT_ERROR_BIT_EXT | VK_DEBUG_REPORT_WARNING_BIT_EXT | VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT /*| VK_DEBUG_REPORT_DEBUG_BIT_EXT*/; callbackCreateInfo.pfnCallback = &MyDebugReportCallback; - callbackCreateInfo.pUserData = nullptr; ERR_GUARD_VULKAN( m_pvkCreateDebugReportCallbackEXT(m_VulkanInstance, &callbackCreateInfo, nullptr, &m_hCallback) ); } +bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound) +{ + bool ok; + if(lastUnbound) + ok = csvSplit.GetCount() >= FIRST_PARAM_INDEX + expectedParamCount - 1; + else + ok = csvSplit.GetCount() == FIRST_PARAM_INDEX + expectedParamCount; + + if(!ok) + { + if(IssueWarning()) + printf("Line %zu: Incorrect number of function parameters.\n", lineNumber); + } + + return ok; +} + +void Player::CreatePool(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 7, false)) + { + VmaPoolCreateInfo poolCreateInfo = {}; + uint64_t origPtr = 0; + + if(StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX), poolCreateInfo.memoryTypeIndex) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), poolCreateInfo.flags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), poolCreateInfo.blockSize) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 3), poolCreateInfo.minBlockCount) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 4), poolCreateInfo.maxBlockCount) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), poolCreateInfo.frameInUseCount) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), origPtr)) + { + Pool poolDesc = {}; + VkResult res = vmaCreatePool(m_Allocator, &poolCreateInfo, &poolDesc.pool); + if(res == VK_SUCCESS) + { + const auto existingIt = m_Pools.find(origPtr); + if(existingIt != m_Pools.end()) + { + if(IssueWarning()) + printf("Line %zu: Pool %llX already exists.\n", lineNumber, origPtr); + } + } + else + { + if(IssueWarning()) + printf("Line %zu: vmaCreatePool failed (%u).\n", lineNumber, res); + } + m_Pools[origPtr] = poolDesc; + } + else + { + if(IssueWarning()) + printf("Line %zu: Invalid parameters for vmaCreatePool.\n", lineNumber); + } + } +} + +void Player::DestroyPool(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origPtr = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + const auto it = m_Pools.find(origPtr); + if(it != m_Pools.end()) + { + vmaDestroyPool(m_Allocator, it->second.pool); + m_Pools.erase(it); + } + else + { + if(IssueWarning()) + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPtr); + } + } + else + { + if(IssueWarning()) + printf("Line %zu: Invalid parameters for vmaDestroyPool.\n", lineNumber); + } + } +} + +void Player::CreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 12, true)) + { + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + VmaAllocationCreateInfo allocCreateInfo = {}; + uint64_t origPool = 0; + uint64_t origPtr = 0; + + if(StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX), bufCreateInfo.flags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), bufCreateInfo.size) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), bufCreateInfo.usage) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 3), (uint32_t&)bufCreateInfo.sharingMode) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 4), allocCreateInfo.flags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), (uint32_t&)allocCreateInfo.usage) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), allocCreateInfo.requiredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 7), allocCreateInfo.preferredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 8), allocCreateInfo.memoryTypeBits) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 9), origPool) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 10), origPtr)) + { + if(origPool != 0) + { + const auto poolIt = m_Pools.find(origPool); + if(poolIt != m_Pools.end()) + allocCreateInfo.pool = poolIt->second.pool; + else + { + if(IssueWarning()) + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); + } + } + + Buffer bufDesc = {}; + VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &bufDesc.buffer, &bufDesc.allocation, nullptr); + if(res == VK_SUCCESS) + { + const auto existingIt = m_Buffers.find(origPtr); + if(existingIt != m_Buffers.end()) + { + if(IssueWarning()) + printf("Line %zu: Allocation %llX for buffer already exists.\n", lineNumber, origPtr); + } + } + else + { + if(IssueWarning()) + printf("Line %zu: vmaCreateBuffer failed (%u).\n", lineNumber, res); + } + m_Buffers[origPtr] = bufDesc; + } + else + { + if(IssueWarning()) + printf("Line %zu: Invalid parameters for vmaCreateBuffer.\n", lineNumber); + } + } +} + +void Player::DestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origAllocPtr = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origAllocPtr)) + { + const auto it = m_Buffers.find(origAllocPtr); + if(it != m_Buffers.end()) + { + vmaDestroyBuffer(m_Allocator, it->second.buffer, it->second.allocation); + m_Buffers.erase(it); + } + else + { + if(IssueWarning()) + printf("Line %zu: Allocation %llX for buffer not found.\n", lineNumber, origAllocPtr); + } + } + else + { + if(IssueWarning()) + printf("Line %zu: Invalid parameters for vmaDestroyBuffer.\n", lineNumber); + } + } +} + +void Player::CreateImage(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 21, true)) + { + VkImageCreateInfo imageCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + VmaAllocationCreateInfo allocCreateInfo = {}; + uint64_t origPool = 0; + uint64_t origPtr = 0; + + if(StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX), imageCreateInfo.flags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), (uint32_t&)imageCreateInfo.imageType) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), (uint32_t&)imageCreateInfo.format) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 3), imageCreateInfo.extent.width) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 4), imageCreateInfo.extent.height) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), imageCreateInfo.extent.depth) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), imageCreateInfo.mipLevels) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 7), imageCreateInfo.arrayLayers) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 8), (uint32_t&)imageCreateInfo.samples) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 9), (uint32_t&)imageCreateInfo.tiling) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 10), imageCreateInfo.usage) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 11), (uint32_t&)imageCreateInfo.sharingMode) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 12), (uint32_t&)imageCreateInfo.initialLayout) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 13), allocCreateInfo.flags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 14), (uint32_t&)allocCreateInfo.usage) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 15), allocCreateInfo.requiredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 16), allocCreateInfo.preferredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 17), allocCreateInfo.memoryTypeBits) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 18), origPool) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 19), origPtr)) + { + if(origPool != 0) + { + const auto poolIt = m_Pools.find(origPool); + if(poolIt != m_Pools.end()) + allocCreateInfo.pool = poolIt->second.pool; + else + { + if(IssueWarning()) + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); + } + } + + Image imageDesc = {}; + VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &imageDesc.image, &imageDesc.allocation, nullptr); + if(res == VK_SUCCESS) + { + const auto existingIt = m_Images.find(origPtr); + if(existingIt != m_Images.end()) + { + if(IssueWarning()) + printf("Line %zu: Allocation %llX for image already exists.\n", lineNumber, origPtr); + } + } + else + { + if(IssueWarning()) + printf("Line %zu: vmaCreateImage failed (%u).\n", lineNumber, res); + } + m_Images[origPtr] = imageDesc; + } + else + { + if(IssueWarning()) + printf("Line %zu: Invalid parameters for vmaCreateImage.\n", lineNumber); + } + } +} + +void Player::DestroyImage(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origAllocPtr = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origAllocPtr)) + { + const auto it = m_Images.find(origAllocPtr); + if(it != m_Images.end()) + { + vmaDestroyImage(m_Allocator, it->second.image, it->second.allocation); + m_Images.erase(it); + } + else + { + if(IssueWarning()) + printf("Line %zu: Allocation %llX for image not found.\n", lineNumber, origAllocPtr); + } + } + else + { + if(IssueWarning()) + printf("Line %zu: Invalid parameters for vmaDestroyBuffer.\n", lineNumber); + } + } +} + + //////////////////////////////////////////////////////////////////////////////// // Main functions @@ -456,13 +869,6 @@ static void PrintCommandLineSyntax() " VmaReplay \n"); } -static inline bool StrRangeEq(const StrRange& lhs, const char* rhsSz) -{ - const size_t rhsLen = strlen(rhsSz); - return rhsLen == lhs.length() && - memcmp(lhs.beg, rhsSz, rhsLen) == 0; -} - static int ProcessFile(const char* data, size_t numBytes) { // Begin stats. @@ -488,12 +894,14 @@ static int ProcessFile(const char* data, size_t numBytes) Player player; player.Init(); + printf("Playing...\n"); while(lineSplit.GetNextLine(line)) { player.ExecuteLine(lineSplit.GetNextLineIndex(), line); } // End stats. + printf("Done.\n"); printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); return 0; @@ -532,8 +940,6 @@ static int ProcessFile(const char* filePath) result = RESULT_ERROR_SOURCE_FILE; } - printf("Done.\n"); - return result; } From 4d63e9d886d4412afbe8115fd74a66e21a5cd636 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 16:52:32 +0200 Subject: [PATCH 08/51] Improved error handling. --- src/VmaReplay/Common.cpp | 29 +-------- src/VmaReplay/Common.h | 8 +-- src/VmaReplay/VmaReplay.cpp | 120 +++++++++++++++++++++++------------- 3 files changed, 82 insertions(+), 75 deletions(-) diff --git a/src/VmaReplay/Common.cpp b/src/VmaReplay/Common.cpp index 8f2baf3..f04aece 100644 --- a/src/VmaReplay/Common.cpp +++ b/src/VmaReplay/Common.cpp @@ -1,19 +1,6 @@ #include "Common.h" -void ReadFile(std::vector& out, const char* fileName) -{ - std::ifstream file(fileName, std::ios::ate | std::ios::binary); - assert(file.is_open()); - size_t fileSize = (size_t)file.tellg(); - if(fileSize > 0) - { - out.resize(fileSize); - file.seekg(0); - file.read(out.data(), fileSize); - } - else - out.clear(); -} +/* void SetConsoleColor(CONSOLE_COLOR color) { @@ -139,16 +126,4 @@ void PrintErrorF(const wchar_t* format, ...) PrintMessageV(CONSOLE_COLOR::WARNING, format, argList); va_end(argList); } - -void SaveFile(const wchar_t* filePath, const void* data, size_t dataSize) -{ - FILE* f = nullptr; - _wfopen_s(&f, filePath, L"wb"); - if(f) - { - fwrite(data, 1, dataSize, f); - fclose(f); - } - else - assert(0); -} +*/ diff --git a/src/VmaReplay/Common.h b/src/VmaReplay/Common.h index 70e7dad..d3ac260 100644 --- a/src/VmaReplay/Common.h +++ b/src/VmaReplay/Common.h @@ -22,8 +22,6 @@ typedef std::chrono::high_resolution_clock::time_point time_point; typedef std::chrono::high_resolution_clock::duration duration; -#define ERR_GUARD_VULKAN(Expr) do { VkResult res__ = (Expr); if (res__ < 0) assert(0); } while(0) - inline float ToFloatSeconds(duration d) { return std::chrono::duration_cast>(d).count(); @@ -41,6 +39,7 @@ static inline T align_up(T val, T align) return (val + align - 1) / align * align; } +/* class RandomNumberGenerator { public: @@ -54,8 +53,6 @@ private: uint32_t GenerateFast() { return m_Value = (m_Value * 196314165 + 907633515); } }; -void ReadFile(std::vector& out, const char* fileName); - enum class CONSOLE_COLOR { INFO, @@ -85,5 +82,4 @@ void PrintWarningF(const char* format, ...); void PrintWarningF(const wchar_t* format, ...); void PrintErrorF(const char* format, ...); void PrintErrorF(const wchar_t* format, ...); - -void SaveFile(const wchar_t* filePath, const void* data, size_t dataSize); +*/ diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 6fc13b0..b4e24ac 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -28,6 +28,7 @@ static const int RESULT_EXCEPTION = -1000; static const int RESULT_ERROR_COMMAND_LINE = -1; static const int RESULT_ERROR_SOURCE_FILE = -2; static const int RESULT_ERROR_FORMAT = -3; +static const int RESULT_ERROR_VULKAN = -4; struct StrRange { @@ -46,24 +47,21 @@ static inline bool StrRangeEq(const StrRange& lhs, const char* rhsSz) static inline bool StrRangeToUint(const StrRange& s, uint32_t& out) { - // TODO handle failure. char* end = (char*)s.end; out = (uint32_t)strtoul(s.beg, &end, 10); - return true; + return end == s.end; } static inline bool StrRangeToUint(const StrRange& s, uint64_t& out) { - // TODO handle failure. char* end = (char*)s.end; out = (uint64_t)strtoull(s.beg, &end, 10); - return true; + return end == s.end; } static inline bool StrRangeToPtr(const StrRange& s, uint64_t& out) { - // TODO handle failure. char* end = (char*)s.end; out = (uint64_t)strtoull(s.beg, &end, 16); - return true; + return end == s.end; } //////////////////////////////////////////////////////////////////////////////// @@ -212,7 +210,7 @@ class Player { public: Player(); - void Init(); + int Init(); ~Player(); void ExecuteLine(size_t lineNumber, const StrRange& line); @@ -258,7 +256,7 @@ private: // Increments warning counter. Returns true if warning message should be printed. bool IssueWarning() { return m_WarningCount++ < MAX_WARNINGS_TO_SHOW; } - void InitVulkan(); + int InitVulkan(); void FinalizeVulkan(); void RegisterDebugCallbacks(); @@ -277,9 +275,9 @@ Player::Player() { } -void Player::Init() +int Player::Init() { - InitVulkan(); + return InitVulkan(); } Player::~Player() @@ -355,16 +353,19 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) } } -void Player::InitVulkan() +int Player::InitVulkan() { printf("Initializing Vulkan...\n"); uint32_t instanceLayerPropCount = 0; - ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, nullptr) ); + VkResult res = vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, nullptr); + assert(res == VK_SUCCESS); + std::vector instanceLayerProps(instanceLayerPropCount); if(instanceLayerPropCount > 0) { - ERR_GUARD_VULKAN( vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, instanceLayerProps.data()) ); + res = vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, instanceLayerProps.data()); + assert(res == VK_SUCCESS); } if(g_EnableValidationLayer == true) @@ -381,7 +382,7 @@ void Player::InitVulkan() //instanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); std::vector instanceLayers; - if(g_EnableValidationLayer == true) + if(g_EnableValidationLayer) { instanceLayers.push_back(VALIDATION_LAYER_NAME); instanceExtensions.push_back("VK_EXT_debug_report"); @@ -401,18 +402,32 @@ void Player::InitVulkan() instInfo.enabledLayerCount = (uint32_t)instanceLayers.size(); instInfo.ppEnabledLayerNames = instanceLayers.data(); - ERR_GUARD_VULKAN( vkCreateInstance(&instInfo, NULL, &m_VulkanInstance) ); + res = vkCreateInstance(&instInfo, NULL, &m_VulkanInstance); + if(res != VK_SUCCESS) + { + printf("ERROR: vkCreateInstance failed (%u)\n", res); + return RESULT_ERROR_VULKAN; + } - RegisterDebugCallbacks(); + if(g_EnableValidationLayer) + RegisterDebugCallbacks(); // Find physical device uint32_t deviceCount = 0; - ERR_GUARD_VULKAN( vkEnumeratePhysicalDevices(m_VulkanInstance, &deviceCount, nullptr) ); - assert(deviceCount > 0); + res = vkEnumeratePhysicalDevices(m_VulkanInstance, &deviceCount, nullptr); + assert(res == VK_SUCCESS); + if(deviceCount == 0) + { + printf("ERROR: No Vulkan physical devices found.\n"); + return RESULT_ERROR_VULKAN; + } + else if(deviceCount > 1) + printf("WARNING: %u Vulkan physical devices found. Choosing first one.\n", deviceCount); std::vector physicalDevices(deviceCount); - ERR_GUARD_VULKAN( vkEnumeratePhysicalDevices(m_VulkanInstance, &deviceCount, physicalDevices.data()) ); + res = vkEnumeratePhysicalDevices(m_VulkanInstance, &deviceCount, physicalDevices.data()); + assert(res == VK_SUCCESS); m_PhysicalDevice = physicalDevices[0]; @@ -420,21 +435,25 @@ void Player::InitVulkan() uint32_t queueFamilyCount = 0; vkGetPhysicalDeviceQueueFamilyProperties(m_PhysicalDevice, &queueFamilyCount, nullptr); - assert(queueFamilyCount > 0); - std::vector queueFamilies(queueFamilyCount); - vkGetPhysicalDeviceQueueFamilyProperties(m_PhysicalDevice, &queueFamilyCount, queueFamilies.data()); - for(uint32_t i = 0; i < queueFamilyCount; ++i) + if(queueFamilyCount) { - if(queueFamilies[i].queueCount > 0) + std::vector queueFamilies(queueFamilyCount); + vkGetPhysicalDeviceQueueFamilyProperties(m_PhysicalDevice, &queueFamilyCount, queueFamilies.data()); + for(uint32_t i = 0; i < queueFamilyCount; ++i) { - if((queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) + if(queueFamilies[i].queueCount > 0 && + (queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) { m_GraphicsQueueFamilyIndex = i; break; } } } - assert(m_GraphicsQueueFamilyIndex != UINT_MAX); + if(m_GraphicsQueueFamilyIndex == UINT_MAX) + { + printf("ERROR: Couldn't find graphics queue.\n"); + return RESULT_ERROR_VULKAN; + } // Create logical device @@ -459,12 +478,14 @@ void Player::InitVulkan() //enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); { uint32_t propertyCount = 0; - ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(m_PhysicalDevice, nullptr, &propertyCount, nullptr) ); + res = vkEnumerateDeviceExtensionProperties(m_PhysicalDevice, nullptr, &propertyCount, nullptr); + assert(res == VK_SUCCESS); if(propertyCount) { std::vector properties{propertyCount}; - ERR_GUARD_VULKAN( vkEnumerateDeviceExtensionProperties(m_PhysicalDevice, nullptr, &propertyCount, properties.data()) ); + res = vkEnumerateDeviceExtensionProperties(m_PhysicalDevice, nullptr, &propertyCount, properties.data()); + assert(res == VK_SUCCESS); for(uint32_t i = 0; i < propertyCount; ++i) { @@ -489,7 +510,12 @@ void Player::InitVulkan() deviceCreateInfo.pQueueCreateInfos = &deviceQueueCreateInfo; deviceCreateInfo.pEnabledFeatures = &deviceFeatures; - ERR_GUARD_VULKAN( vkCreateDevice(m_PhysicalDevice, &deviceCreateInfo, nullptr, &m_Device) ); + res = vkCreateDevice(m_PhysicalDevice, &deviceCreateInfo, nullptr, &m_Device); + if(res != VK_SUCCESS) + { + printf("ERROR: vkCreateDevice failed (%u)\n", res); + return RESULT_ERROR_VULKAN; + } // Create memory allocator @@ -502,7 +528,14 @@ void Player::InitVulkan() allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; } - ERR_GUARD_VULKAN( vmaCreateAllocator(&allocatorInfo, &m_Allocator) ); + res = vmaCreateAllocator(&allocatorInfo, &m_Allocator); + if(res != VK_SUCCESS) + { + printf("ERROR: vmaCreateAllocator failed (%u)\n", res); + return RESULT_ERROR_VULKAN; + } + + return 0; } void Player::FinalizeVulkan() @@ -587,7 +620,8 @@ void Player::RegisterDebugCallbacks() VK_DEBUG_REPORT_DEBUG_BIT_EXT*/; callbackCreateInfo.pfnCallback = &MyDebugReportCallback; - ERR_GUARD_VULKAN( m_pvkCreateDebugReportCallbackEXT(m_VulkanInstance, &callbackCreateInfo, nullptr, &m_hCallback) ); + VkResult res = m_pvkCreateDebugReportCallbackEXT(m_VulkanInstance, &callbackCreateInfo, nullptr, &m_hCallback); + assert(res == VK_SUCCESS); } bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound) @@ -892,24 +926,26 @@ static int ProcessFile(const char* data, size_t numBytes) } Player player; - player.Init(); - - printf("Playing...\n"); - while(lineSplit.GetNextLine(line)) + int result = player.Init(); + if(result == 0) { - player.ExecuteLine(lineSplit.GetNextLineIndex(), line); + printf("Playing...\n"); + while(lineSplit.GetNextLine(line)) + { + player.ExecuteLine(lineSplit.GetNextLineIndex(), line); + } + + // End stats. + printf("Done.\n"); + printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); } - // End stats. - printf("Done.\n"); - printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); - - return 0; + return result; } static int ProcessFile(const char* filePath) { - printf("Replaying file \"%s\"...\n", filePath); + printf("Loading file \"%s\"...\n", filePath); int result = 0; FILE* file = nullptr; From 71f3d067c52480bbec40e1f0f6e2115339a687d1 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 17:09:51 +0200 Subject: [PATCH 09/51] Better tracking of allocations. --- src/VmaReplay/VmaReplay.cpp | 153 ++++++++++++++++-------------------- src/vk_mem_alloc.h | 14 +++- 2 files changed, 80 insertions(+), 87 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index b4e24ac..24222c5 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -233,25 +233,22 @@ private: uint32_t m_VmaFrameIndex = 0; - // Any of these handles null means it was created in original but couldn't be created now. + // Any of these handles null can mean it was created in original but couldn't be created now. struct Pool { VmaPool pool; }; - struct Buffer + struct Allocation { + VmaAllocation allocation; VkBuffer buffer; - VmaAllocation allocation; - }; - struct Image - { VkImage image; - VmaAllocation allocation; }; std::unordered_map m_Pools; - std::unordered_map m_Buffers; - std::unordered_map m_Images; + std::unordered_map m_Allocations; + + void Destroy(const Allocation& alloc); // Increments warning counter. Returns true if warning message should be printed. bool IssueWarning() { return m_WarningCount++ < MAX_WARNINGS_TO_SHOW; } @@ -263,12 +260,15 @@ private: // If parmeter count doesn't match, issues warning and returns false. bool ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound); - void CreatePool(size_t lineNumber, const CsvSplit& csvSplit); - void DestroyPool(size_t lineNumber, const CsvSplit& csvSplit); - void CreateBuffer(size_t lineNumber, const CsvSplit& csvSplit); - void DestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit); - void CreateImage(size_t lineNumber, const CsvSplit& csvSplit); - void DestroyImage(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteDestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } + void ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteDestroyImage(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } + void ExecuteFreeMemory(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } + + void DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit); }; Player::Player() @@ -322,9 +322,9 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) } } else if(StrRangeEq(functionName, "vmaCreatePool")) - CreatePool(lineNumber, csvSplit); + ExecuteCreatePool(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaDestroyPool")) - DestroyPool(lineNumber, csvSplit); + ExecuteDestroyPool(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaSetAllocationUserData")) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 2, true)) @@ -333,13 +333,15 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) } } else if(StrRangeEq(functionName, "vmaCreateBuffer")) - CreateBuffer(lineNumber, csvSplit); + ExecuteCreateBuffer(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaDestroyBuffer")) - DestroyBuffer(lineNumber, csvSplit); + ExecuteDestroyBuffer(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaCreateImage")) - CreateImage(lineNumber, csvSplit); + ExecuteCreateImage(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaDestroyImage")) - DestroyImage(lineNumber, csvSplit); + ExecuteDestroyImage(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaFreeMemory")) + ExecuteFreeMemory(lineNumber, csvSplit); else { if(IssueWarning()) @@ -353,6 +355,21 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) } } +void Player::Destroy(const Allocation& alloc) +{ + if(alloc.buffer) + { + assert(alloc.image == VK_NULL_HANDLE); + vmaDestroyBuffer(m_Allocator, alloc.buffer, alloc.allocation); + } + else if(alloc.image) + { + vmaDestroyImage(m_Allocator, alloc.image, alloc.allocation); + } + else + vmaFreeMemory(m_Allocator, alloc.allocation); +} + int Player::InitVulkan() { printf("Initializing Vulkan...\n"); @@ -540,24 +557,16 @@ int Player::InitVulkan() void Player::FinalizeVulkan() { - if(!m_Images.empty()) + if(!m_Allocations.empty()) { - printf("WARNING: Images not destroyed: %zu.\n", m_Images.size()); + printf("WARNING: Allocations not destroyed: %zu.\n", m_Allocations.size()); - for(const auto it : m_Images) + for(const auto it : m_Allocations) { - vmaDestroyImage(m_Allocator, it.second.image, it.second.allocation); + Destroy(it.second); } - } - if(!m_Buffers.empty()) - { - printf("WARNING: Buffers not destroyed: %zu.\n", m_Buffers.size()); - - for(const auto it : m_Buffers) - { - vmaDestroyBuffer(m_Allocator, it.second.buffer, it.second.allocation); - } + m_Allocations.clear(); } if(!m_Pools.empty()) @@ -565,9 +574,9 @@ void Player::FinalizeVulkan() printf("WARNING: Pools not destroyed: %zu.\n", m_Pools.size()); for(const auto it : m_Pools) - { vmaDestroyPool(m_Allocator, it.second.pool); - } + + m_Pools.clear(); } vkDeviceWaitIdle(m_Device); @@ -641,7 +650,7 @@ bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& c return ok; } -void Player::CreatePool(size_t lineNumber, const CsvSplit& csvSplit) +void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 7, false)) { @@ -682,7 +691,7 @@ void Player::CreatePool(size_t lineNumber, const CsvSplit& csvSplit) } } -void Player::DestroyPool(size_t lineNumber, const CsvSplit& csvSplit) +void Player::ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { @@ -710,7 +719,7 @@ void Player::DestroyPool(size_t lineNumber, const CsvSplit& csvSplit) } } -void Player::CreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) +void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 12, true)) { @@ -743,15 +752,15 @@ void Player::CreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) } } - Buffer bufDesc = {}; - VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &bufDesc.buffer, &bufDesc.allocation, nullptr); + Allocation allocDesc = {}; + VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); if(res == VK_SUCCESS) { - const auto existingIt = m_Buffers.find(origPtr); - if(existingIt != m_Buffers.end()) + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) { if(IssueWarning()) - printf("Line %zu: Allocation %llX for buffer already exists.\n", lineNumber, origPtr); + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); } } else @@ -759,7 +768,7 @@ void Player::CreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) if(IssueWarning()) printf("Line %zu: vmaCreateBuffer failed (%u).\n", lineNumber, res); } - m_Buffers[origPtr] = bufDesc; + m_Allocations[origPtr] = allocDesc; } else { @@ -769,7 +778,7 @@ void Player::CreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) } } -void Player::DestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) +void Player::DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { @@ -777,16 +786,16 @@ void Player::DestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origAllocPtr)) { - const auto it = m_Buffers.find(origAllocPtr); - if(it != m_Buffers.end()) + const auto it = m_Allocations.find(origAllocPtr); + if(it != m_Allocations.end()) { - vmaDestroyBuffer(m_Allocator, it->second.buffer, it->second.allocation); - m_Buffers.erase(it); + Destroy(it->second); + m_Allocations.erase(it); } else { if(IssueWarning()) - printf("Line %zu: Allocation %llX for buffer not found.\n", lineNumber, origAllocPtr); + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origAllocPtr); } } else @@ -797,7 +806,7 @@ void Player::DestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) } } -void Player::CreateImage(size_t lineNumber, const CsvSplit& csvSplit) +void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 21, true)) { @@ -839,15 +848,15 @@ void Player::CreateImage(size_t lineNumber, const CsvSplit& csvSplit) } } - Image imageDesc = {}; - VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &imageDesc.image, &imageDesc.allocation, nullptr); + Allocation allocDesc = {}; + VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &allocDesc.image, &allocDesc.allocation, nullptr); if(res == VK_SUCCESS) { - const auto existingIt = m_Images.find(origPtr); - if(existingIt != m_Images.end()) + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) { if(IssueWarning()) - printf("Line %zu: Allocation %llX for image already exists.\n", lineNumber, origPtr); + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); } } else @@ -855,7 +864,7 @@ void Player::CreateImage(size_t lineNumber, const CsvSplit& csvSplit) if(IssueWarning()) printf("Line %zu: vmaCreateImage failed (%u).\n", lineNumber, res); } - m_Images[origPtr] = imageDesc; + m_Allocations[origPtr] = allocDesc; } else { @@ -865,34 +874,6 @@ void Player::CreateImage(size_t lineNumber, const CsvSplit& csvSplit) } } -void Player::DestroyImage(size_t lineNumber, const CsvSplit& csvSplit) -{ - if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) - { - uint64_t origAllocPtr = 0; - - if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origAllocPtr)) - { - const auto it = m_Images.find(origAllocPtr); - if(it != m_Images.end()) - { - vmaDestroyImage(m_Allocator, it->second.image, it->second.allocation); - m_Images.erase(it); - } - else - { - if(IssueWarning()) - printf("Line %zu: Allocation %llX for image not found.\n", lineNumber, origAllocPtr); - } - } - else - { - if(IssueWarning()) - printf("Line %zu: Invalid parameters for vmaDestroyBuffer.\n", lineNumber); - } - } -} - //////////////////////////////////////////////////////////////////////////////// // Main functions diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 5dbad64..53943bd 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -9927,8 +9927,20 @@ void vmaFreeMemory( VmaAllocator allocator, VmaAllocation allocation) { - Crash(); VMA_ASSERT(allocator); + + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", threadId, time, frameIndex, + allocation); + fflush(g_File); + } + VMA_DEBUG_LOG("vmaFreeMemory"); VMA_DEBUG_GLOBAL_MUTEX_LOCK if(allocation != VK_NULL_HANDLE) From deeb817d7a071904ea33ed32ac69c184a9b4b652 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 17:30:29 +0200 Subject: [PATCH 10/51] Minor fixes. Added basic statistics. --- src/VmaReplay/VmaReplay.cpp | 85 +++++++++++++++++++++++++++++++------ src/vk_mem_alloc.h | 2 +- 2 files changed, 73 insertions(+), 14 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 24222c5..c0144bc 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -149,6 +149,39 @@ void CsvSplit::Set(const StrRange& line, size_t maxCount) m_Count = rangeIndex; } +//////////////////////////////////////////////////////////////////////////////// +// class Statistics + +class Statistics +{ +public: + Statistics() { } + + size_t GetImageCreationCount() const { return m_ImageCreationCount; } + size_t GetBufferCreationCount() const { return m_BufferCreationCount; } + size_t GetAllocationCreationCount() const { return m_AllocationCreationCount; } + + void RegisterCreateImage(); + void RegisterCreateBuffer(); + +private: + size_t m_ImageCreationCount = 0; + size_t m_BufferCreationCount = 0; + size_t m_AllocationCreationCount = 0; // Also includes buffers and images. +}; + +void Statistics::RegisterCreateImage() +{ + ++m_ImageCreationCount; + ++m_AllocationCreationCount; +} + +void Statistics::RegisterCreateBuffer() +{ + ++m_BufferCreationCount; + ++m_AllocationCreationCount; +} + //////////////////////////////////////////////////////////////////////////////// // class Player @@ -248,6 +281,8 @@ private: std::unordered_map m_Pools; std::unordered_map m_Allocations; + Statistics m_Stats; + void Destroy(const Allocation& alloc); // Increments warning counter. Returns true if warning message should be printed. @@ -256,6 +291,7 @@ private: int InitVulkan(); void FinalizeVulkan(); void RegisterDebugCallbacks(); + void PrintStats(); // If parmeter count doesn't match, issues warning and returns false. bool ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound); @@ -282,6 +318,8 @@ int Player::Init() Player::~Player() { + PrintStats(); + FinalizeVulkan(); if(m_WarningCount > MAX_WARNINGS_TO_SHOW) @@ -321,6 +359,13 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) // Nothing to do. } } + else if(StrRangeEq(functionName, "vmaDestroyAllocator")) + { + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 0, false)) + { + // Nothing to do. + } + } else if(StrRangeEq(functionName, "vmaCreatePool")) ExecuteCreatePool(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaDestroyPool")) @@ -633,6 +678,14 @@ void Player::RegisterDebugCallbacks() assert(res == VK_SUCCESS); } +void Player::PrintStats() +{ + printf("Statistics:\n"); + printf(" Total allocations created: %zu\n", m_Stats.GetAllocationCreationCount()); + printf(" Total buffers created: %zu\n", m_Stats.GetBufferCreationCount()); + printf(" Total images created: %zu\n", m_Stats.GetImageCreationCount()); +} + bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound) { bool ok; @@ -756,18 +809,21 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); if(res == VK_SUCCESS) { - const auto existingIt = m_Allocations.find(origPtr); - if(existingIt != m_Allocations.end()) - { - if(IssueWarning()) - printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); - } + m_Stats.RegisterCreateBuffer(); } else { if(IssueWarning()) printf("Line %zu: vmaCreateBuffer failed (%u).\n", lineNumber, res); } + + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) + { + if(IssueWarning()) + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } + m_Allocations[origPtr] = allocDesc; } else @@ -852,18 +908,21 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &allocDesc.image, &allocDesc.allocation, nullptr); if(res == VK_SUCCESS) { - const auto existingIt = m_Allocations.find(origPtr); - if(existingIt != m_Allocations.end()) - { - if(IssueWarning()) - printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); - } + m_Stats.RegisterCreateImage(); } else { if(IssueWarning()) printf("Line %zu: vmaCreateImage failed (%u).\n", lineNumber, res); } + + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) + { + if(IssueWarning()) + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } + m_Allocations[origPtr] = allocDesc; } else @@ -900,7 +959,7 @@ static int ProcessFile(const char* data, size_t numBytes) } if(!lineSplit.GetNextLine(line) || - !StrRangeEq(line, "1,0")) + !(StrRangeEq(line, "1,0") || StrRangeEq(line, "1,1"))) { printf("ERROR: Incorrect file format version.\n"); return RESULT_ERROR_FORMAT; diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 53943bd..834adc5 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -2666,7 +2666,7 @@ void EnsureFile() { fopen_s(&g_File, "VMA_Usage_Dump", "wb"); fprintf(g_File, "%s\n", "Vulkan Memory Allocator,Calls recording"); - fprintf(g_File, "%s\n", "1,0"); + fprintf(g_File, "%s\n", "1,1"); QueryPerformanceFrequency(&g_Freq); QueryPerformanceCounter(&g_StartCounter); } From f188a152de0df5bfbf547997e6d6d989d40643aa Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 7 Aug 2018 17:58:46 +0200 Subject: [PATCH 11/51] Minor fixes. --- src/VmaReplay/VmaReplay.cpp | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index c0144bc..c37b463 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -720,20 +720,19 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) { Pool poolDesc = {}; VkResult res = vmaCreatePool(m_Allocator, &poolCreateInfo, &poolDesc.pool); - if(res == VK_SUCCESS) - { - const auto existingIt = m_Pools.find(origPtr); - if(existingIt != m_Pools.end()) - { - if(IssueWarning()) - printf("Line %zu: Pool %llX already exists.\n", lineNumber, origPtr); - } - } - else + if(res != VK_SUCCESS) { if(IssueWarning()) printf("Line %zu: vmaCreatePool failed (%u).\n", lineNumber, res); } + + const auto existingIt = m_Pools.find(origPtr); + if(existingIt != m_Pools.end()) + { + if(IssueWarning()) + printf("Line %zu: Pool %llX already exists.\n", lineNumber, origPtr); + } + m_Pools[origPtr] = poolDesc; } else @@ -1034,7 +1033,7 @@ int main(int argc, char** argv) { try { - main2(argc, argv); + return main2(argc, argv); } catch(const std::exception& e) { @@ -1046,6 +1045,4 @@ int main(int argc, char** argv) printf("UNKNOWN ERROR\n"); return RESULT_EXCEPTION; } - - return 0; } From 5b48b5efcfc36d7f595d2b8ad51e431413e06c14 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 13 Aug 2018 12:15:50 +0200 Subject: [PATCH 12/51] Code formatting --- src/VmaReplay/VmaReplay.cpp | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index c37b463..6462461 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -347,7 +347,9 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) else { if(IssueWarning()) + { printf("Line %zu: Incorrect frame index.\n", lineNumber); + } } StrRange functionName = csvSplit.GetRange(3); @@ -390,13 +392,17 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) else { if(IssueWarning()) + { printf("Line %zu: Unknown function.\n", lineNumber); + } } } else { if(IssueWarning()) + { printf("Line %zu: Too few columns.\n", lineNumber); + } } } @@ -697,7 +703,9 @@ bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& c if(!ok) { if(IssueWarning()) + { printf("Line %zu: Incorrect number of function parameters.\n", lineNumber); + } } return ok; @@ -723,14 +731,18 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) if(res != VK_SUCCESS) { if(IssueWarning()) + { printf("Line %zu: vmaCreatePool failed (%u).\n", lineNumber, res); + } } const auto existingIt = m_Pools.find(origPtr); if(existingIt != m_Pools.end()) { if(IssueWarning()) + { printf("Line %zu: Pool %llX already exists.\n", lineNumber, origPtr); + } } m_Pools[origPtr] = poolDesc; @@ -738,7 +750,9 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) else { if(IssueWarning()) + { printf("Line %zu: Invalid parameters for vmaCreatePool.\n", lineNumber); + } } } } @@ -760,13 +774,17 @@ void Player::ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit) else { if(IssueWarning()) + { printf("Line %zu: Pool %llX not found.\n", lineNumber, origPtr); + } } } else { if(IssueWarning()) + { printf("Line %zu: Invalid parameters for vmaDestroyPool.\n", lineNumber); + } } } } @@ -800,7 +818,9 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) else { if(IssueWarning()) + { printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); + } } } @@ -813,14 +833,18 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) else { if(IssueWarning()) + { printf("Line %zu: vmaCreateBuffer failed (%u).\n", lineNumber, res); + } } const auto existingIt = m_Allocations.find(origPtr); if(existingIt != m_Allocations.end()) { if(IssueWarning()) + { printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } } m_Allocations[origPtr] = allocDesc; @@ -828,7 +852,9 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) else { if(IssueWarning()) + { printf("Line %zu: Invalid parameters for vmaCreateBuffer.\n", lineNumber); + } } } } @@ -850,13 +876,17 @@ void Player::DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit) else { if(IssueWarning()) + { printf("Line %zu: Allocation %llX not found.\n", lineNumber, origAllocPtr); + } } } else { if(IssueWarning()) + { printf("Line %zu: Invalid parameters for vmaDestroyBuffer.\n", lineNumber); + } } } } @@ -895,11 +925,15 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) { const auto poolIt = m_Pools.find(origPool); if(poolIt != m_Pools.end()) + { allocCreateInfo.pool = poolIt->second.pool; + } else { if(IssueWarning()) + { printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); + } } } @@ -912,14 +946,18 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) else { if(IssueWarning()) + { printf("Line %zu: vmaCreateImage failed (%u).\n", lineNumber, res); + } } const auto existingIt = m_Allocations.find(origPtr); if(existingIt != m_Allocations.end()) { if(IssueWarning()) + { printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } } m_Allocations[origPtr] = allocDesc; @@ -927,7 +965,9 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) else { if(IssueWarning()) + { printf("Line %zu: Invalid parameters for vmaCreateImage.\n", lineNumber); + } } } } From db1f73fee4e21f478e8411bbca6184135f79058e Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 13 Aug 2018 12:22:37 +0200 Subject: [PATCH 13/51] Improved handling of destruction calls with null handle, in vk_mem_alloc as well as VmaReplay. --- src/VmaReplay/VmaReplay.cpp | 42 +++++++++++++++++++++---------------- src/vk_mem_alloc.h | 21 +++++++++++++++---- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 6462461..eaac603 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -765,17 +765,20 @@ void Player::ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit) if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) { - const auto it = m_Pools.find(origPtr); - if(it != m_Pools.end()) + if(origPtr != 0) { - vmaDestroyPool(m_Allocator, it->second.pool); - m_Pools.erase(it); - } - else - { - if(IssueWarning()) + const auto it = m_Pools.find(origPtr); + if(it != m_Pools.end()) { - printf("Line %zu: Pool %llX not found.\n", lineNumber, origPtr); + vmaDestroyPool(m_Allocator, it->second.pool); + m_Pools.erase(it); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPtr); + } } } } @@ -867,17 +870,20 @@ void Player::DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit) if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origAllocPtr)) { - const auto it = m_Allocations.find(origAllocPtr); - if(it != m_Allocations.end()) + if(origAllocPtr != 0) { - Destroy(it->second); - m_Allocations.erase(it); - } - else - { - if(IssueWarning()) + const auto it = m_Allocations.find(origAllocPtr); + if(it != m_Allocations.end()) { - printf("Line %zu: Allocation %llX not found.\n", lineNumber, origAllocPtr); + Destroy(it->second); + m_Allocations.erase(it); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origAllocPtr); + } } } } diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 834adc5..6250dd6 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -9929,6 +9929,11 @@ void vmaFreeMemory( { VMA_ASSERT(allocator); + if(allocation == VK_NULL_HANDLE) + { + return; + } + { VmaMutexLock lock(g_FileMutex, true); EnsureFile(); @@ -9943,10 +9948,7 @@ void vmaFreeMemory( VMA_DEBUG_LOG("vmaFreeMemory"); VMA_DEBUG_GLOBAL_MUTEX_LOCK - if(allocation != VK_NULL_HANDLE) - { - allocator->FreeMemory(allocation); - } + allocator->FreeMemory(allocation); } void vmaGetAllocationInfo( @@ -10232,6 +10234,11 @@ void vmaDestroyBuffer( { VMA_ASSERT(allocator); + if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE) + { + return; + } + { VmaMutexLock lock(g_FileMutex, true); EnsureFile(); @@ -10356,6 +10363,12 @@ void vmaDestroyImage( { VMA_ASSERT(allocator); + if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE) + { + return; + } + + { VmaMutexLock lock(g_FileMutex, true); EnsureFile(); From 652ac275470c9aa9c068cc7627ef647643002b7e Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 13 Aug 2018 13:13:13 +0200 Subject: [PATCH 14/51] Added CmdLineParser, proper parsing of command line parameters, and -v parameter for verbosity. --- src/VmaReplay/Common.cpp | 431 ++++++++++++++++++++++++++++++++++++ src/VmaReplay/Common.h | 61 +++++ src/VmaReplay/VmaReplay.cpp | 139 ++++++++++-- 3 files changed, 616 insertions(+), 15 deletions(-) diff --git a/src/VmaReplay/Common.cpp b/src/VmaReplay/Common.cpp index f04aece..4c1a6b7 100644 --- a/src/VmaReplay/Common.cpp +++ b/src/VmaReplay/Common.cpp @@ -1,5 +1,436 @@ #include "Common.h" +//////////////////////////////////////////////////////////////////////////////// +// class CmdLineParser + +bool CmdLineParser::ReadNextArg(std::string *OutArg) +{ + if (m_argv != NULL) + { + if (m_ArgIndex >= (size_t)m_argc) return false; + + *OutArg = m_argv[m_ArgIndex]; + m_ArgIndex++; + return true; + } + else + { + if (m_ArgIndex >= m_CmdLineLength) return false; + + OutArg->clear(); + bool InsideQuotes = false; + while (m_ArgIndex < m_CmdLineLength) + { + char Ch = m_CmdLine[m_ArgIndex]; + if (Ch == '\\') + { + bool FollowedByQuote = false; + size_t BackslashCount = 1; + size_t TmpIndex = m_ArgIndex + 1; + while (TmpIndex < m_CmdLineLength) + { + char TmpCh = m_CmdLine[TmpIndex]; + if (TmpCh == '\\') + { + BackslashCount++; + TmpIndex++; + } + else if (TmpCh == '"') + { + FollowedByQuote = true; + break; + } + else + break; + } + + if (FollowedByQuote) + { + if (BackslashCount % 2 == 0) + { + for (size_t i = 0; i < BackslashCount / 2; i++) + *OutArg += '\\'; + m_ArgIndex += BackslashCount + 1; + InsideQuotes = !InsideQuotes; + } + else + { + for (size_t i = 0; i < BackslashCount / 2; i++) + *OutArg += '\\'; + *OutArg += '"'; + m_ArgIndex += BackslashCount + 1; + } + } + else + { + for (size_t i = 0; i < BackslashCount; i++) + *OutArg += '\\'; + m_ArgIndex += BackslashCount; + } + } + else if (Ch == '"') + { + InsideQuotes = !InsideQuotes; + m_ArgIndex++; + } + else if (isspace(Ch)) + { + if (InsideQuotes) + { + *OutArg += Ch; + m_ArgIndex++; + } + else + { + m_ArgIndex++; + break; + } + } + else + { + *OutArg += Ch; + m_ArgIndex++; + } + } + + while (m_ArgIndex < m_CmdLineLength && isspace(m_CmdLine[m_ArgIndex])) + m_ArgIndex++; + + return true; + } +} + +CmdLineParser::SHORT_OPT * CmdLineParser::FindShortOpt(char Opt) +{ + for (size_t i = 0; i < m_ShortOpts.size(); i++) + if (m_ShortOpts[i].Opt == Opt) + return &m_ShortOpts[i]; + return NULL; +} + +CmdLineParser::LONG_OPT * CmdLineParser::FindLongOpt(const std::string &Opt) +{ + for (size_t i = 0; i < m_LongOpts.size(); i++) + if (m_LongOpts[i].Opt == Opt) + return &m_LongOpts[i]; + return NULL; +} + +CmdLineParser::CmdLineParser(int argc, char **argv) : + m_argv(argv), + m_CmdLine(NULL), + m_argc(argc), + m_CmdLineLength(0), + m_ArgIndex(1), + m_InsideMultioption(false), + m_LastArgIndex(0), + m_LastOptId(0) +{ + assert(argc > 0); + assert(argv != NULL); +} + +CmdLineParser::CmdLineParser(const char *CmdLine) : + m_argv(NULL), + m_CmdLine(CmdLine), + m_argc(0), + m_ArgIndex(0), + m_InsideMultioption(false), + m_LastArgIndex(0), + m_LastOptId(0) +{ + assert(CmdLine != NULL); + + m_CmdLineLength = strlen(m_CmdLine); + + while (m_ArgIndex < m_CmdLineLength && isspace(m_CmdLine[m_ArgIndex])) + m_ArgIndex++; +} + +void CmdLineParser::RegisterOpt(uint32_t Id, char Opt, bool Parameter) +{ + assert(Opt != '\0'); + + m_ShortOpts.push_back(SHORT_OPT(Id, Opt, Parameter)); +} + +void CmdLineParser::RegisterOpt(uint32_t Id, const std::string &Opt, bool Parameter) +{ + assert(!Opt.empty()); + + m_LongOpts.push_back(LONG_OPT(Id, Opt, Parameter)); +} + +CmdLineParser::RESULT CmdLineParser::ReadNext() +{ + if (m_InsideMultioption) + { + assert(m_LastArgIndex < m_LastArg.length()); + SHORT_OPT *so = FindShortOpt(m_LastArg[m_LastArgIndex]); + if (so == NULL) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + if (so->Parameter) + { + if (m_LastArg.length() == m_LastArgIndex+1) + { + if (!ReadNextArg(&m_LastParameter)) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + m_InsideMultioption = false; + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + else if (m_LastArg[m_LastArgIndex+1] == '=') + { + m_InsideMultioption = false; + m_LastParameter = m_LastArg.substr(m_LastArgIndex+2); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + else + { + m_InsideMultioption = false; + m_LastParameter = m_LastArg.substr(m_LastArgIndex+1); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + } + else + { + if (m_LastArg.length() == m_LastArgIndex+1) + { + m_InsideMultioption = false; + m_LastParameter.clear(); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + else + { + m_LastArgIndex++; + + m_LastParameter.clear(); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + } + } + else + { + if (!ReadNextArg(&m_LastArg)) + { + m_LastParameter.clear(); + m_LastOptId = 0; + return CmdLineParser::RESULT_END; + } + + if (!m_LastArg.empty() && m_LastArg[0] == '-') + { + if (m_LastArg.length() > 1 && m_LastArg[1] == '-') + { + size_t EqualIndex = m_LastArg.find('=', 2); + if (EqualIndex != std::string::npos) + { + LONG_OPT *lo = FindLongOpt(m_LastArg.substr(2, EqualIndex-2)); + if (lo == NULL || lo->Parameter == false) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + m_LastParameter = m_LastArg.substr(EqualIndex+1); + m_LastOptId = lo->Id; + return CmdLineParser::RESULT_OPT; + } + else + { + LONG_OPT *lo = FindLongOpt(m_LastArg.substr(2)); + if (lo == NULL) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + if (lo->Parameter) + { + if (!ReadNextArg(&m_LastParameter)) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + } + else + m_LastParameter.clear(); + m_LastOptId = lo->Id; + return CmdLineParser::RESULT_OPT; + } + } + else + { + if (m_LastArg.length() < 2) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + SHORT_OPT *so = FindShortOpt(m_LastArg[1]); + if (so == NULL) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + if (so->Parameter) + { + if (m_LastArg.length() == 2) + { + if (!ReadNextArg(&m_LastParameter)) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + else if (m_LastArg[2] == '=') + { + m_LastParameter = m_LastArg.substr(3); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + else + { + m_LastParameter = m_LastArg.substr(2); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + } + else + { + if (m_LastArg.length() == 2) + { + m_LastParameter.clear(); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + else + { + m_InsideMultioption = true; + m_LastArgIndex = 2; + + m_LastParameter.clear(); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + } + } + } + else if (!m_LastArg.empty() && m_LastArg[0] == '/') + { + size_t EqualIndex = m_LastArg.find('=', 1); + if (EqualIndex != std::string::npos) + { + if (EqualIndex == 2) + { + SHORT_OPT *so = FindShortOpt(m_LastArg[1]); + if (so != NULL) + { + if (so->Parameter == false) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + m_LastParameter = m_LastArg.substr(EqualIndex+1); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + } + LONG_OPT *lo = FindLongOpt(m_LastArg.substr(1, EqualIndex-1)); + if (lo == NULL || lo->Parameter == false) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + m_LastParameter = m_LastArg.substr(EqualIndex+1); + m_LastOptId = lo->Id; + return CmdLineParser::RESULT_OPT; + } + else + { + if (m_LastArg.length() == 2) + { + SHORT_OPT *so = FindShortOpt(m_LastArg[1]); + if (so != NULL) + { + if (so->Parameter) + { + if (!ReadNextArg(&m_LastParameter)) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + } + else + m_LastParameter.clear(); + m_LastOptId = so->Id; + return CmdLineParser::RESULT_OPT; + } + } + LONG_OPT *lo = FindLongOpt(m_LastArg.substr(1)); + if (lo == NULL) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + if (lo->Parameter) + { + if (!ReadNextArg(&m_LastParameter)) + { + m_LastOptId = 0; + m_LastParameter.clear(); + return CmdLineParser::RESULT_ERROR; + } + } + else + m_LastParameter.clear(); + m_LastOptId = lo->Id; + return CmdLineParser::RESULT_OPT; + } + } + else + { + m_LastOptId = 0; + m_LastParameter = m_LastArg; + return CmdLineParser::RESULT_PARAMETER; + } + } +} + +uint32_t CmdLineParser::GetOptId() +{ + return m_LastOptId; +} + +const std::string & CmdLineParser::GetParameter() +{ + return m_LastParameter; +} + +//////////////////////////////////////////////////////////////////////////////// +// Glolals + /* void SetConsoleColor(CONSOLE_COLOR color) diff --git a/src/VmaReplay/Common.h b/src/VmaReplay/Common.h index d3ac260..afd52c1 100644 --- a/src/VmaReplay/Common.h +++ b/src/VmaReplay/Common.h @@ -39,6 +39,67 @@ static inline T align_up(T val, T align) return (val + align - 1) / align * align; } +class CmdLineParser +{ +public: + enum RESULT + { + RESULT_OPT, + RESULT_PARAMETER, + RESULT_END, + RESULT_ERROR, + }; + + CmdLineParser(int argc, char **argv); + CmdLineParser(const char *CmdLine); + + void RegisterOpt(uint32_t Id, char Opt, bool Parameter); + void RegisterOpt(uint32_t Id, const std::string &Opt, bool Parameter); + + RESULT ReadNext(); + uint32_t GetOptId(); + const std::string & GetParameter(); + +private: + struct SHORT_OPT + { + uint32_t Id; + char Opt; + bool Parameter; + + SHORT_OPT(uint32_t Id, char Opt, bool Parameter) : Id(Id), Opt(Opt), Parameter(Parameter) { } + }; + + struct LONG_OPT + { + uint32_t Id; + std::string Opt; + bool Parameter; + + LONG_OPT(uint32_t Id, std::string Opt, bool Parameter) : Id(Id), Opt(Opt), Parameter(Parameter) { } + }; + + char **m_argv; + const char *m_CmdLine; + int m_argc; + size_t m_CmdLineLength; + size_t m_ArgIndex; + + bool ReadNextArg(std::string *OutArg); + + std::vector m_ShortOpts; + std::vector m_LongOpts; + + SHORT_OPT * FindShortOpt(char Opt); + LONG_OPT * FindLongOpt(const std::string &Opt); + + bool m_InsideMultioption; + std::string m_LastArg; + size_t m_LastArgIndex; + uint32_t m_LastOptId; + std::string m_LastParameter; +}; + /* class RandomNumberGenerator { diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index eaac603..dd88f42 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -30,11 +30,31 @@ static const int RESULT_ERROR_SOURCE_FILE = -2; static const int RESULT_ERROR_FORMAT = -3; static const int RESULT_ERROR_VULKAN = -4; +enum CMD_LINE_OPT +{ + CMD_LINE_OPT_VERBOSITY, +}; + +static enum class VERBOSITY +{ + MINIMUM = 0, + DEFAULT, + MAXIMUM, + COUNT, +} g_Verbosity = VERBOSITY::DEFAULT; + +static std::string g_FilePath; + struct StrRange { const char* beg; const char* end; + StrRange() { } + StrRange(const char* beg, const char* end) : beg(beg), end(end) { } + explicit StrRange(const char* sz) : beg(sz), end(sz + strlen(sz)) { } + explicit StrRange(const std::string& s) : beg(s.data()), end(s.data() + s.length()) { } + size_t length() const { return end - beg; } }; @@ -286,7 +306,7 @@ private: void Destroy(const Allocation& alloc); // Increments warning counter. Returns true if warning message should be printed. - bool IssueWarning() { return m_WarningCount++ < MAX_WARNINGS_TO_SHOW; } + bool IssueWarning(); int InitVulkan(); void FinalizeVulkan(); @@ -318,11 +338,14 @@ int Player::Init() Player::~Player() { - PrintStats(); + if(g_Verbosity > VERBOSITY::MINIMUM) + { + PrintStats(); + } FinalizeVulkan(); - if(m_WarningCount > MAX_WARNINGS_TO_SHOW) + if(g_Verbosity < VERBOSITY::MAXIMUM && m_WarningCount > MAX_WARNINGS_TO_SHOW) printf("WARNING: %zu more warnings not shown.\n", m_WarningCount - MAX_WARNINGS_TO_SHOW); } @@ -421,9 +444,25 @@ void Player::Destroy(const Allocation& alloc) vmaFreeMemory(m_Allocator, alloc.allocation); } +bool Player::IssueWarning() +{ + if(g_Verbosity < VERBOSITY::MAXIMUM) + { + return m_WarningCount++ < MAX_WARNINGS_TO_SHOW; + } + else + { + ++m_WarningCount; + return true; + } +} + int Player::InitVulkan() { - printf("Initializing Vulkan...\n"); + if(g_Verbosity > VERBOSITY::MINIMUM) + { + printf("Initializing Vulkan...\n"); + } uint32_t instanceLayerPropCount = 0; VkResult res = vkEnumerateInstanceLayerProperties(&instanceLayerPropCount, nullptr); @@ -984,14 +1023,24 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) static void PrintCommandLineSyntax() { - printf("Command line syntax:\n" - " VmaReplay \n"); + printf( + "Command line syntax:\n" + " VmaReplay [Options] \n" + "Available options:\n" + " -v - Verbosity level:\n" + " 0 - Minimum verbosity. Prints only warnings and errors.\n" + " 1 - Default verbosity. Prints important messages and statistics.\n" + " 2 - Maximum verbosity. Prints a lot of information.\n" + ); } static int ProcessFile(const char* data, size_t numBytes) { // Begin stats. - printf("File size: %zu B\n", numBytes); + if(g_Verbosity > VERBOSITY::MINIMUM) + { + printf("File size: %zu B\n", numBytes); + } LineSplit lineSplit(data, numBytes); StrRange line; @@ -1014,27 +1063,36 @@ static int ProcessFile(const char* data, size_t numBytes) int result = player.Init(); if(result == 0) { - printf("Playing...\n"); + if(g_Verbosity > VERBOSITY::MINIMUM) + { + printf("Playing...\n"); + } while(lineSplit.GetNextLine(line)) { player.ExecuteLine(lineSplit.GetNextLineIndex(), line); } // End stats. - printf("Done.\n"); - printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); + if(g_Verbosity > VERBOSITY::MINIMUM) + { + printf("Done.\n"); + printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); + } } return result; } -static int ProcessFile(const char* filePath) +static int ProcessFile() { - printf("Loading file \"%s\"...\n", filePath); + if(g_Verbosity > VERBOSITY::MINIMUM) + { + printf("Loading file \"%s\"...\n", g_FilePath.c_str()); + } int result = 0; FILE* file = nullptr; - const errno_t err = fopen_s(&file, filePath, "rb"); + const errno_t err = fopen_s(&file, g_FilePath.c_str(), "rb"); if(err == 0) { _fseeki64(file, 0, SEEK_END); @@ -1066,13 +1124,64 @@ static int ProcessFile(const char* filePath) static int main2(int argc, char** argv) { - if(argc != 2) + CmdLineParser cmdLineParser(argc, argv); + + cmdLineParser.RegisterOpt(CMD_LINE_OPT_VERBOSITY, 'v', true); + + CmdLineParser::RESULT res; + while((res = cmdLineParser.ReadNext()) != CmdLineParser::RESULT_END) + { + switch(res) + { + case CmdLineParser::RESULT_OPT: + switch(cmdLineParser.GetOptId()) + { + case CMD_LINE_OPT_VERBOSITY: + { + uint32_t verbosityVal = UINT32_MAX; + if(StrRangeToUint(StrRange(cmdLineParser.GetParameter()), verbosityVal) && + verbosityVal < (uint32_t)VERBOSITY::COUNT) + { + g_Verbosity = (VERBOSITY)verbosityVal; + } + else + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + } + break; + default: + assert(0); + } + break; + case CmdLineParser::RESULT_PARAMETER: + if(g_FilePath.empty()) + { + g_FilePath = cmdLineParser.GetParameter(); + } + else + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + break; + case CmdLineParser::RESULT_ERROR: + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + break; + default: + assert(0); + } + } + + if(g_FilePath.empty()) { PrintCommandLineSyntax(); return RESULT_ERROR_COMMAND_LINE; } - return ProcessFile(argv[1]); + return ProcessFile(); } int main(int argc, char** argv) From 1016cc682c2d9cc65c5406638d803f9add1a25ad Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 13 Aug 2018 13:30:04 +0200 Subject: [PATCH 15/51] Added function SecondsToFriendlyStr. Printing playback duration. --- src/VmaReplay/Common.cpp | 67 +++++++++++++++++++++++++++++++++++++ src/VmaReplay/Common.h | 2 ++ src/VmaReplay/VmaReplay.cpp | 9 +++++ 3 files changed, 78 insertions(+) diff --git a/src/VmaReplay/Common.cpp b/src/VmaReplay/Common.cpp index 4c1a6b7..dffd2af 100644 --- a/src/VmaReplay/Common.cpp +++ b/src/VmaReplay/Common.cpp @@ -558,3 +558,70 @@ void PrintErrorF(const wchar_t* format, ...) va_end(argList); } */ + +void SecondsToFriendlyStr(float seconds, std::string& out) +{ + if(seconds == 0.f) + { + out = "0"; + return; + } + + if (seconds < 0.f) + { + out = "-"; + seconds = -seconds; + } + else + { + out.clear(); + } + + char s[32]; + + // #.# ns + if(seconds < 1e-6) + { + sprintf_s(s, "%.4f ns", seconds * 1e-9); + out += s; + } + // #.# us + else if(seconds < 1e-3) + { + sprintf_s(s, "%.4f us", seconds * 1e-6); + out += s; + } + // #.# ms + else if(seconds < 1.f) + { + sprintf_s(s, "%.4f ms", seconds * 1e-3); + out += s; + } + else if(seconds < 60.f) + { + sprintf_s(s, "%.4f s", seconds); + out += s; + } + else + { + uint64_t seconds_u = (uint64_t)seconds; + // "#:## min" + if (seconds_u < 3600) + { + uint64_t minutes = seconds_u / 60; + seconds_u -= minutes * 60; + sprintf_s(s, "%llu:%02llu min", minutes, seconds_u); + out += s; + } + // "#:##:## h" + else + { + uint64_t minutes = seconds_u / 60; + seconds_u -= minutes * 60; + uint64_t hours = minutes / 60; + minutes -= hours * 60; + sprintf_s(s, "%llu:%02llu:%02llu h", hours, minutes, seconds_u); + out += s; + } + } +} diff --git a/src/VmaReplay/Common.h b/src/VmaReplay/Common.h index afd52c1..f5f4c44 100644 --- a/src/VmaReplay/Common.h +++ b/src/VmaReplay/Common.h @@ -27,6 +27,8 @@ inline float ToFloatSeconds(duration d) return std::chrono::duration_cast>(d).count(); } +void SecondsToFriendlyStr(float seconds, std::string& out); + template inline T ceil_div(T x, T y) { diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index dd88f42..7470e48 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -1067,15 +1067,24 @@ static int ProcessFile(const char* data, size_t numBytes) { printf("Playing...\n"); } + + const time_point timeBeg = std::chrono::high_resolution_clock::now(); + while(lineSplit.GetNextLine(line)) { player.ExecuteLine(lineSplit.GetNextLineIndex(), line); } + const duration playDuration = std::chrono::high_resolution_clock::now() - timeBeg; + // End stats. if(g_Verbosity > VERBOSITY::MINIMUM) { + std::string playDurationStr; + SecondsToFriendlyStr(ToFloatSeconds(playDuration), playDurationStr); + printf("Done.\n"); + printf("Playback took: %s\n", playDurationStr.c_str()); printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); } } From 03764c6ea016abd585da7d289364b3149ff281dd Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 13 Aug 2018 13:38:46 +0200 Subject: [PATCH 16/51] Statistics: added printing of original recording time. --- src/VmaReplay/Common.cpp | 15 ++++++++------- src/VmaReplay/VmaReplay.cpp | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/VmaReplay/Common.cpp b/src/VmaReplay/Common.cpp index dffd2af..cfb44dc 100644 --- a/src/VmaReplay/Common.cpp +++ b/src/VmaReplay/Common.cpp @@ -579,27 +579,28 @@ void SecondsToFriendlyStr(float seconds, std::string& out) char s[32]; - // #.# ns + // #.### ns if(seconds < 1e-6) { - sprintf_s(s, "%.4f ns", seconds * 1e-9); + sprintf_s(s, "%.3f ns", seconds * 1e-9); out += s; } - // #.# us + // #.### us else if(seconds < 1e-3) { - sprintf_s(s, "%.4f us", seconds * 1e-6); + sprintf_s(s, "%.3f us", seconds * 1e-6); out += s; } - // #.# ms + // #.### ms else if(seconds < 1.f) { - sprintf_s(s, "%.4f ms", seconds * 1e-3); + sprintf_s(s, "%.3f ms", seconds * 1e-3); out += s; } + // #.### s else if(seconds < 60.f) { - sprintf_s(s, "%.4f s", seconds); + sprintf_s(s, "%.3f s", seconds); out += s; } else diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 7470e48..e74ba53 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -56,6 +56,7 @@ struct StrRange explicit StrRange(const std::string& s) : beg(s.data()), end(s.data() + s.length()) { } size_t length() const { return end - beg; } + void to_str(std::string& out) { out.assign(beg, end); } }; static inline bool StrRangeEq(const StrRange& lhs, const char* rhsSz) @@ -83,6 +84,12 @@ static inline bool StrRangeToPtr(const StrRange& s, uint64_t& out) out = (uint64_t)strtoull(s.beg, &end, 16); return end == s.end; } +static inline bool StrRangeToFloat(const StrRange& s, float& out) +{ + char* end = (char*)s.end; + out = strtof(s.beg, &end); + return end == s.end; +} //////////////////////////////////////////////////////////////////////////////// // LineSplit class @@ -301,6 +308,8 @@ private: std::unordered_map m_Pools; std::unordered_map m_Allocations; + // Copy of column [1] from previously parsed line. + std::string m_LastLineTimeStr; Statistics m_Stats; void Destroy(const Allocation& alloc); @@ -356,6 +365,8 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) if(csvSplit.GetCount() >= FIRST_PARAM_INDEX) { + csvSplit.GetRange(1).to_str(m_LastLineTimeStr); + // Update VMA current frame index. StrRange frameIndexStr = csvSplit.GetRange(2); uint32_t frameIndex; @@ -729,6 +740,14 @@ void Player::PrintStats() printf(" Total allocations created: %zu\n", m_Stats.GetAllocationCreationCount()); printf(" Total buffers created: %zu\n", m_Stats.GetBufferCreationCount()); printf(" Total images created: %zu\n", m_Stats.GetImageCreationCount()); + + float lastTime; + if(!m_LastLineTimeStr.empty() && StrRangeToFloat(StrRange(m_LastLineTimeStr), lastTime)) + { + std::string origTimeStr; + SecondsToFriendlyStr(lastTime, origTimeStr); + printf(" Original recording time: %s\n", origTimeStr.c_str()); + } } bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound) From 5765b8ac83e1d6e2a14544f35362fe352a269b33 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 13 Aug 2018 13:51:33 +0200 Subject: [PATCH 17/51] Added statistics about thread usage. --- src/VmaReplay/VmaReplay.cpp | 44 ++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index e74ba53..e3ef4a3 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -304,10 +304,15 @@ private: VkBuffer buffer; VkImage image; }; - std::unordered_map m_Pools; std::unordered_map m_Allocations; + struct Thread + { + uint32_t callCount; + }; + std::unordered_map m_Threads; + // Copy of column [1] from previously parsed line. std::string m_LastLineTimeStr; Statistics m_Stats; @@ -365,6 +370,31 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) if(csvSplit.GetCount() >= FIRST_PARAM_INDEX) { + // Check thread ID. + uint32_t threadId; + if(StrRangeToUint(csvSplit.GetRange(0), threadId)) + { + const auto it = m_Threads.find(threadId); + if(it != m_Threads.end()) + { + ++it->second.callCount; + } + else + { + Thread threadInfo{}; + threadInfo.callCount = 1; + m_Threads[threadId] = threadInfo; + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Incorrect thread ID.\n", lineNumber); + } + } + + // Save time. csvSplit.GetRange(1).to_str(m_LastLineTimeStr); // Update VMA current frame index. @@ -748,6 +778,18 @@ void Player::PrintStats() SecondsToFriendlyStr(lastTime, origTimeStr); printf(" Original recording time: %s\n", origTimeStr.c_str()); } + + // Thread statistics. + uint32_t threadCallCountMax = 0; + uint32_t threadCallCountSum = 0; + for(const auto& it : m_Threads) + { + threadCallCountMax = std::max(threadCallCountMax, it.second.callCount); + threadCallCountSum += it.second.callCount; + } + printf(" Threads making calls to VMA: %zu\n", m_Threads.size()); + printf(" %.2f%% calls from most active thread.\n", + (float)threadCallCountMax * 100.f / (float)threadCallCountSum); } bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound) From 385b716276c660ab0e5ab875ffffd0bea1d0f338 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 13 Aug 2018 14:06:17 +0200 Subject: [PATCH 18/51] Added statistics for number of VmaAllocator nad VmaPool objects. --- src/VmaReplay/VmaReplay.cpp | 92 +++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 15 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index e3ef4a3..2208e87 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -187,16 +187,39 @@ public: size_t GetImageCreationCount() const { return m_ImageCreationCount; } size_t GetBufferCreationCount() const { return m_BufferCreationCount; } size_t GetAllocationCreationCount() const { return m_AllocationCreationCount; } + size_t GetAllocatorCreationCount() const { return m_AllocatorCreationCount; } + size_t GetAllocatorPeakCount() const { return m_AllocatorPeakCount; } + size_t GetPoolCreationCount() const { return m_PoolCreationCount; } + void RegisterCreateAllocator(); + void RegisterDestroyAllocator(); void RegisterCreateImage(); void RegisterCreateBuffer(); + void RegisterCreatePool(); private: size_t m_ImageCreationCount = 0; size_t m_BufferCreationCount = 0; size_t m_AllocationCreationCount = 0; // Also includes buffers and images. + size_t m_AllocatorCreationCount = 0; + size_t m_AllocatorCurrCount = 0; + size_t m_AllocatorPeakCount = 0; + size_t m_PoolCreationCount = 0; }; +void Statistics::RegisterCreateAllocator() +{ + ++m_AllocatorCreationCount; + ++m_AllocatorCurrCount; + m_AllocatorPeakCount = std::max(m_AllocatorPeakCount, m_AllocatorCurrCount); +} + +void Statistics::RegisterDestroyAllocator() +{ + if(m_AllocatorCurrCount > 0) + --m_AllocatorCurrCount; +} + void Statistics::RegisterCreateImage() { ++m_ImageCreationCount; @@ -209,6 +232,11 @@ void Statistics::RegisterCreateBuffer() ++m_AllocationCreationCount; } +void Statistics::RegisterCreatePool() +{ + ++m_PoolCreationCount; +} + //////////////////////////////////////////////////////////////////////////////// // class Player @@ -352,6 +380,14 @@ int Player::Init() Player::~Player() { + if(m_Stats.GetAllocatorCreationCount() > 1) + { + printf("WARNING: %zu VmaAllocator objects were created. It is recommended to use just one.\n", + m_Stats.GetAllocatorCreationCount()); + printf(" At most %zu allocators existed simultaneously.\n", + m_Stats.GetAllocatorPeakCount()); + } + if(g_Verbosity > VERBOSITY::MINIMUM) { PrintStats(); @@ -422,14 +458,14 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 0, false)) { - // Nothing to do. + m_Stats.RegisterCreateAllocator(); } } else if(StrRangeEq(functionName, "vmaDestroyAllocator")) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 0, false)) { - // Nothing to do. + m_Stats.RegisterDestroyAllocator(); } } else if(StrRangeEq(functionName, "vmaCreatePool")) @@ -702,7 +738,7 @@ void Player::FinalizeVulkan() if(!m_Pools.empty()) { - printf("WARNING: Pools not destroyed: %zu.\n", m_Pools.size()); + printf("WARNING: Custom pools not destroyed: %zu.\n", m_Pools.size()); for(const auto it : m_Pools) vmaDestroyPool(m_Allocator, it.second.pool); @@ -767,9 +803,22 @@ void Player::RegisterDebugCallbacks() void Player::PrintStats() { printf("Statistics:\n"); - printf(" Total allocations created: %zu\n", m_Stats.GetAllocationCreationCount()); - printf(" Total buffers created: %zu\n", m_Stats.GetBufferCreationCount()); - printf(" Total images created: %zu\n", m_Stats.GetImageCreationCount()); + if(m_Stats.GetAllocationCreationCount() > 0) + { + printf(" Total allocations created: %zu\n", m_Stats.GetAllocationCreationCount()); + } + if(m_Stats.GetBufferCreationCount() > 0) + { + printf(" Total buffers created: %zu\n", m_Stats.GetBufferCreationCount()); + } + if(m_Stats.GetImageCreationCount() > 0) + { + printf(" Total images created: %zu\n", m_Stats.GetImageCreationCount()); + } + if(m_Stats.GetPoolCreationCount() > 0) + { + printf(" Total custom pools created: %zu\n", m_Stats.GetPoolCreationCount()); + } float lastTime; if(!m_LastLineTimeStr.empty() && StrRangeToFloat(StrRange(m_LastLineTimeStr), lastTime)) @@ -780,16 +829,24 @@ void Player::PrintStats() } // Thread statistics. - uint32_t threadCallCountMax = 0; - uint32_t threadCallCountSum = 0; - for(const auto& it : m_Threads) + const size_t threadCount = m_Threads.size(); + if(threadCount > 1) { - threadCallCountMax = std::max(threadCallCountMax, it.second.callCount); - threadCallCountSum += it.second.callCount; + uint32_t threadCallCountMax = 0; + uint32_t threadCallCountSum = 0; + for(const auto& it : m_Threads) + { + threadCallCountMax = std::max(threadCallCountMax, it.second.callCount); + threadCallCountSum += it.second.callCount; + } + printf(" Threads making calls to VMA: %zu\n", threadCount); + printf(" %.2f%% calls from most active thread.\n", + (float)threadCallCountMax * 100.f / (float)threadCallCountSum); + } + else + { + printf(" VMA used from only one thread.\n"); } - printf(" Threads making calls to VMA: %zu\n", m_Threads.size()); - printf(" %.2f%% calls from most active thread.\n", - (float)threadCallCountMax * 100.f / (float)threadCallCountSum); } bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound) @@ -826,6 +883,8 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), poolCreateInfo.frameInUseCount) && StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), origPtr)) { + m_Stats.RegisterCreatePool(); + Pool poolDesc = {}; VkResult res = vmaCreatePool(m_Allocator, &poolCreateInfo, &poolDesc.pool); if(res != VK_SUCCESS) @@ -1098,7 +1157,7 @@ static void PrintCommandLineSyntax() static int ProcessFile(const char* data, size_t numBytes) { // Begin stats. - if(g_Verbosity > VERBOSITY::MINIMUM) + if(g_Verbosity == VERBOSITY::MAXIMUM) { printf("File size: %zu B\n", numBytes); } @@ -1146,6 +1205,9 @@ static int ProcessFile(const char* data, size_t numBytes) printf("Done.\n"); printf("Playback took: %s\n", playDurationStr.c_str()); + } + if(g_Verbosity == VERBOSITY::MAXIMUM) + { printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); } } From ab8550340cd81439f463d4f478ffb03342e7e9b8 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 13 Aug 2018 15:15:36 +0200 Subject: [PATCH 19/51] Added more detailed statistics about created buffers and images based on human-friendly classification (see http://asawicki.info/news_1682_human-friendly_classification_of_vulkan_resources.html) --- src/VmaReplay/VmaReplay.cpp | 151 +++++++++++++++++++++++++++++++----- 1 file changed, 132 insertions(+), 19 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 2208e87..0da31e6 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -182,10 +182,14 @@ void CsvSplit::Set(const StrRange& line, size_t maxCount) class Statistics { public: + static uint32_t BufferUsageToClass(uint32_t usage); + static uint32_t ImageUsageToClass(uint32_t usage); + Statistics() { } - size_t GetImageCreationCount() const { return m_ImageCreationCount; } - size_t GetBufferCreationCount() const { return m_BufferCreationCount; } + size_t GetImageCreationCount(uint32_t imgClass) const { return m_ImageCreationCount[imgClass]; } + size_t GetLinearImageCreationCount() const { return m_LinearImageCreationCount; } + size_t GetBufferCreationCount(uint32_t bufClass) const { return m_BufferCreationCount[bufClass]; } size_t GetAllocationCreationCount() const { return m_AllocationCreationCount; } size_t GetAllocatorCreationCount() const { return m_AllocatorCreationCount; } size_t GetAllocatorPeakCount() const { return m_AllocatorPeakCount; } @@ -193,13 +197,14 @@ public: void RegisterCreateAllocator(); void RegisterDestroyAllocator(); - void RegisterCreateImage(); - void RegisterCreateBuffer(); + void RegisterCreateImage(uint32_t usage, uint32_t tiling); + void RegisterCreateBuffer(uint32_t usage); void RegisterCreatePool(); private: - size_t m_ImageCreationCount = 0; - size_t m_BufferCreationCount = 0; + size_t m_ImageCreationCount[4] = { }; + size_t m_LinearImageCreationCount = 0; + size_t m_BufferCreationCount[4] = { }; size_t m_AllocationCreationCount = 0; // Also includes buffers and images. size_t m_AllocatorCreationCount = 0; size_t m_AllocatorCurrCount = 0; @@ -207,6 +212,69 @@ private: size_t m_PoolCreationCount = 0; }; +uint32_t Statistics::BufferUsageToClass(uint32_t usage) +{ + // Buffer is used as source of data for fixed-function stage of graphics pipeline. + // It's indirect, vertex, or index buffer. + if ((usage & (VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | + VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDEX_BUFFER_BIT)) != 0) + { + return 0; + } + // Buffer is accessed by shaders for load/store/atomic. + // Aka "UAV" + else if ((usage & (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT)) != 0) + { + return 1; + } + // Buffer is accessed by shaders for reading uniform data. + // Aka "constant buffer" + else if ((usage & (VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | + VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT)) != 0) + { + return 2; + } + // Any other type of buffer. + // Notice that VK_BUFFER_USAGE_TRANSFER_SRC_BIT and VK_BUFFER_USAGE_TRANSFER_DST_BIT + // flags are intentionally ignored. + else + { + return 3; + } +} + +uint32_t Statistics::ImageUsageToClass(uint32_t usage) +{ + // Image is used as depth/stencil "texture/surface". + if ((usage & VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT) != 0) + { + return 0; + } + // Image is used as other type of attachment. + // Aka "render target" + else if ((usage & (VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT | + VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT | + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) != 0) + { + return 1; + } + // Image is accessed by shaders for sampling. + // Aka "texture" + else if ((usage & VK_IMAGE_USAGE_SAMPLED_BIT) != 0) + { + return 2; + } + // Any other type of image. + // Notice that VK_IMAGE_USAGE_TRANSFER_SRC_BIT and VK_IMAGE_USAGE_TRANSFER_DST_BIT + // flags are intentionally ignored. + else + { + return 3; + } +} + void Statistics::RegisterCreateAllocator() { ++m_AllocatorCreationCount; @@ -220,15 +288,24 @@ void Statistics::RegisterDestroyAllocator() --m_AllocatorCurrCount; } -void Statistics::RegisterCreateImage() +void Statistics::RegisterCreateImage(uint32_t usage, uint32_t tiling) { - ++m_ImageCreationCount; + if(tiling == VK_IMAGE_TILING_LINEAR) + ++m_LinearImageCreationCount; + else + { + const uint32_t imgClass = ImageUsageToClass(usage); + ++m_ImageCreationCount[imgClass]; + } + ++m_AllocationCreationCount; } -void Statistics::RegisterCreateBuffer() +void Statistics::RegisterCreateBuffer(uint32_t usage) { - ++m_BufferCreationCount; + const uint32_t bufClass = BufferUsageToClass(usage); + ++m_BufferCreationCount[bufClass]; + ++m_AllocationCreationCount; } @@ -807,14 +884,48 @@ void Player::PrintStats() { printf(" Total allocations created: %zu\n", m_Stats.GetAllocationCreationCount()); } - if(m_Stats.GetBufferCreationCount() > 0) + + // Buffers + const size_t bufferCreationCount = + m_Stats.GetBufferCreationCount(0) + + m_Stats.GetBufferCreationCount(1) + + m_Stats.GetBufferCreationCount(2) + + m_Stats.GetBufferCreationCount(3); + if(bufferCreationCount > 0) { - printf(" Total buffers created: %zu\n", m_Stats.GetBufferCreationCount()); + printf(" Total buffers created: %zu\n", bufferCreationCount); + if(g_Verbosity == VERBOSITY::MAXIMUM) + { + printf(" Class 0 (indirect/vertex/index): %zu\n", m_Stats.GetBufferCreationCount(0)); + printf(" Class 1 (storage): %zu\n", m_Stats.GetBufferCreationCount(1)); + printf(" Class 2 (uniform): %zu\n", m_Stats.GetBufferCreationCount(2)); + printf(" Class 3 (other): %zu\n", m_Stats.GetBufferCreationCount(3)); + } } - if(m_Stats.GetImageCreationCount() > 0) + + // Images + const size_t imageCreationCount = + m_Stats.GetImageCreationCount(0) + + m_Stats.GetImageCreationCount(1) + + m_Stats.GetImageCreationCount(2) + + m_Stats.GetImageCreationCount(3) + + m_Stats.GetLinearImageCreationCount(); + if(imageCreationCount > 0) { - printf(" Total images created: %zu\n", m_Stats.GetImageCreationCount()); + printf(" Total images created: %zu\n", imageCreationCount); + if(g_Verbosity == VERBOSITY::MAXIMUM) + { + printf(" Class 0 (depth/stencil): %zu\n", m_Stats.GetImageCreationCount(0)); + printf(" Class 1 (attachment): %zu\n", m_Stats.GetImageCreationCount(1)); + printf(" Class 2 (sampled): %zu\n", m_Stats.GetImageCreationCount(2)); + printf(" Class 3 (other): %zu\n", m_Stats.GetImageCreationCount(3)); + if(m_Stats.GetLinearImageCreationCount() > 0) + { + printf(" LINEAR tiling: %zu\n", m_Stats.GetLinearImageCreationCount()); + } + } } + if(m_Stats.GetPoolCreationCount() > 0) { printf(" Total custom pools created: %zu\n", m_Stats.GetPoolCreationCount()); @@ -883,11 +994,13 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), poolCreateInfo.frameInUseCount) && StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), origPtr)) { - m_Stats.RegisterCreatePool(); - Pool poolDesc = {}; VkResult res = vmaCreatePool(m_Allocator, &poolCreateInfo, &poolDesc.pool); - if(res != VK_SUCCESS) + if(res == VK_SUCCESS) + { + m_Stats.RegisterCreatePool(); + } + else { if(IssueWarning()) { @@ -990,7 +1103,7 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); if(res == VK_SUCCESS) { - m_Stats.RegisterCreateBuffer(); + m_Stats.RegisterCreateBuffer(bufCreateInfo.usage); } else { @@ -1106,7 +1219,7 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &allocDesc.image, &allocDesc.allocation, nullptr); if(res == VK_SUCCESS) { - m_Stats.RegisterCreateImage(); + m_Stats.RegisterCreateImage(imageCreateInfo.usage, imageCreateInfo.tiling); } else { From 87c818900253ee6846e06bdfd1daccfec3dc9b03 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 10:52:54 +0200 Subject: [PATCH 20/51] Foo --- src/VmaReplay/VmaReplay.cpp | 91 ++++++++++++++++++++++++++++++++++++- src/vk_mem_alloc.h | 17 +++++-- 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 0da31e6..8433610 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -44,6 +44,15 @@ static enum class VERBOSITY } g_Verbosity = VERBOSITY::DEFAULT; static std::string g_FilePath; +// Most significant 16 bits are major version, least significant 16 bits are minor version. +static uint32_t g_FileVersion; + +static bool ValidateFileVersion() +{ + const uint32_t major = g_FileVersion >> 16; + const uint32_t minor = g_FileVersion & 0xFFFF; + return major == 1 && minor <= 2; +} struct StrRange { @@ -176,6 +185,24 @@ void CsvSplit::Set(const StrRange& line, size_t maxCount) m_Count = rangeIndex; } +static bool ParseFileVersion(const StrRange& s) +{ + CsvSplit csvSplit; + csvSplit.Set(s, 2); + uint32_t major, minor; + if(csvSplit.GetCount() == 2 && + StrRangeToUint(csvSplit.GetRange(0), major) && + StrRangeToUint(csvSplit.GetRange(1), minor)) + { + g_FileVersion = (major << 16) | minor; + return true; + } + else + { + return false; + } +} + //////////////////////////////////////////////////////////////////////////////// // class Statistics @@ -442,6 +469,7 @@ private: void ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteDestroyImage(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } void ExecuteFreeMemory(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } + void ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvSplit); void DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit); }; @@ -566,6 +594,8 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) ExecuteDestroyImage(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaFreeMemory")) ExecuteFreeMemory(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaCreateLostAllocation")) + ExecuteCreateLostAllocation(lineNumber, csvSplit); else { if(IssueWarning()) @@ -1250,6 +1280,64 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) } } +void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origPtr = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + // TODO + if(origPool != 0) + { + const auto poolIt = m_Pools.find(origPool); + if(poolIt != m_Pools.end()) + allocCreateInfo.pool = poolIt->second.pool; + else + { + if(IssueWarning()) + { + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); + } + } + } + + Allocation allocDesc = {}; + VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); + if(res == VK_SUCCESS) + { + m_Stats.RegisterCreateBuffer(bufCreateInfo.usage); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: vmaCreateBuffer failed (%u).\n", lineNumber, res); + } + } + + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } + } + + m_Allocations[origPtr] = allocDesc; + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaCreateLostAllocation.\n", lineNumber); + } + } + } +} + //////////////////////////////////////////////////////////////////////////////// // Main functions @@ -1285,8 +1373,7 @@ static int ProcessFile(const char* data, size_t numBytes) return RESULT_ERROR_FORMAT; } - if(!lineSplit.GetNextLine(line) || - !(StrRangeEq(line, "1,0") || StrRangeEq(line, "1,1"))) + if(!lineSplit.GetNextLine(line) || !ParseFileVersion(line) || !ValidateFileVersion()) { printf("ERROR: Incorrect file format version.\n"); return RESULT_ERROR_FORMAT; diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 6250dd6..95aa1c8 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -2666,7 +2666,7 @@ void EnsureFile() { fopen_s(&g_File, "VMA_Usage_Dump", "wb"); fprintf(g_File, "%s\n", "Vulkan Memory Allocator,Calls recording"); - fprintf(g_File, "%s\n", "1,1"); + fprintf(g_File, "%s\n", "1,2"); QueryPerformanceFrequency(&g_Freq); QueryPerformanceCounter(&g_StartCounter); } @@ -10003,12 +10003,23 @@ void vmaCreateLostAllocation( VmaAllocator allocator, VmaAllocation* pAllocation) { - Crash(); VMA_ASSERT(allocator && pAllocation); VMA_DEBUG_GLOBAL_MUTEX_LOCK; allocator->CreateLostAllocation(pAllocation); + + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", threadId, time, frameIndex, + (*pAllocation)); + fflush(g_File); + } } VkResult vmaMapMemory( @@ -10089,7 +10100,6 @@ VkResult vmaBindBufferMemory( VmaAllocation allocation, VkBuffer buffer) { - Crash(); VMA_ASSERT(allocator && allocation && buffer); VMA_DEBUG_LOG("vmaBindBufferMemory"); @@ -10104,7 +10114,6 @@ VkResult vmaBindImageMemory( VmaAllocation allocation, VkImage image) { - Crash(); VMA_ASSERT(allocator && allocation && image); VMA_DEBUG_LOG("vmaBindImageMemory"); From 94e82065985cefd5f8791af50fcdc89b55afc0ac Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 11:12:51 +0200 Subject: [PATCH 21/51] Added new validation layer message to ignores, and documented it. "Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used." Issue #34 thanks @victorbush ! --- src/VulkanSample.cpp | 12 ++++++++++++ src/vk_mem_alloc.h | 27 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/VulkanSample.cpp b/src/VulkanSample.cpp index 91f2bf2..d33861e 100644 --- a/src/VulkanSample.cpp +++ b/src/VulkanSample.cpp @@ -190,6 +190,18 @@ VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( { return VK_FALSE; } + + /* + "Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used." + Ignoring because we map entire VkDeviceMemory blocks, where different types of + images and buffers may end up together, especially on GPUs with unified memory + like Intel. + */ + if(strstr(pMessage, "Mapping an image with layout") != nullptr && + strstr(pMessage, "can result in undefined behavior if this memory is used by the device") != nullptr) + { + return VK_FALSE; + } switch(flags) { diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 149445e..7e24916 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -323,6 +323,16 @@ memcpy(mappedData, &constantBufferData, sizeof(constantBufferData)); vmaUnmapMemory(allocator, constantBufferAllocation); \endcode +When mapping, you may see a warning from Vulkan validation layer similar to this one: + +Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used. + +It happens because the library maps entire `VkDeviceMemory` block, where different +types of images and buffers may end up together, especially on GPUs with unified memory like Intel. +You can safely ignore it if you are sure you access only memory of the intended +object that you wanted to map. + + \section memory_mapping_persistently_mapped_memory Persistently mapped memory Kepping your memory persistently mapped is generally OK in Vulkan. @@ -1150,6 +1160,23 @@ To learn more about this extension, see: threads at the same time if you pass the same #VmaAllocation object to these functions. +\section general_considerations_validation_layer_warnings Validation layer warnings + +When using this library, you can meet following types of warnings issued by +Vulkan validation layer. They don't necessarily indicate a bug, so you may need +to just ignore them. + +- *vkBindBufferMemory(): Binding memory to buffer 0xeb8e4 but vkGetBufferMemoryRequirements() has not been called on that buffer.* + - It happens when VK_KHR_dedicated_allocation extension is enabled. + `vkGetBufferMemoryRequirements2KHR` function is used instead, while validation layer seems to be unaware of it. +- *Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used.* + - It happens when you map a buffer or image, because the library maps entire + `VkDeviceMemory` block, where different types of images and buffers may end + up together, especially on GPUs with unified memory like Intel. +- *Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug.* + - It happens when you use lost allocations, and a new image or buffer is + created in place of an existing object that bacame lost. + \section general_considerations_allocation_algorithm Allocation algorithm The library uses following algorithm for allocation, in order: From 404d12e18ef54ef1b3a6b998190cf43df7ec779f Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 13:33:34 +0200 Subject: [PATCH 22/51] Format version 1,2. Added support for vmaAllocateMemory, vmaAllocateMemoryForBuffer, vmaAllocateMemoryForImage, vmaCreateLostAllocation. --- src/VmaReplay/VmaReplay.cpp | 193 ++++++++++++++++++++++++++++++++++-- src/vk_mem_alloc.h | 131 ++++++++++++++++++------ 2 files changed, 285 insertions(+), 39 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 8433610..8eeb51b 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -43,6 +43,8 @@ static enum class VERBOSITY COUNT, } g_Verbosity = VERBOSITY::DEFAULT; +enum class OBJECT_TYPE { BUFFER, IMAGE }; + static std::string g_FilePath; // Most significant 16 bits are major version, least significant 16 bits are minor version. static uint32_t g_FileVersion; @@ -99,6 +101,30 @@ static inline bool StrRangeToFloat(const StrRange& s, float& out) out = strtof(s.beg, &end); return end == s.end; } +static inline bool StrRangeToBool(const StrRange& s, bool& out) +{ + if(s.end - s.beg == 1) + { + if(*s.beg == '1') + { + out = true; + } + else if(*s.beg == '0') + { + out = false; + } + else + { + return false; + } + } + else + { + return false; + } + + return true; +} //////////////////////////////////////////////////////////////////////////////// // LineSplit class @@ -227,12 +253,13 @@ public: void RegisterCreateImage(uint32_t usage, uint32_t tiling); void RegisterCreateBuffer(uint32_t usage); void RegisterCreatePool(); + void RegisterCreateAllocation(); private: size_t m_ImageCreationCount[4] = { }; size_t m_LinearImageCreationCount = 0; size_t m_BufferCreationCount[4] = { }; - size_t m_AllocationCreationCount = 0; // Also includes buffers and images. + size_t m_AllocationCreationCount = 0; // Also includes buffers and images, and lost allocations. size_t m_AllocatorCreationCount = 0; size_t m_AllocatorCurrCount = 0; size_t m_AllocatorPeakCount = 0; @@ -341,6 +368,11 @@ void Statistics::RegisterCreatePool() ++m_PoolCreationCount; } +void Statistics::RegisterCreateAllocation() +{ + ++m_AllocationCreationCount; +} + //////////////////////////////////////////////////////////////////////////////// // class Player @@ -380,6 +412,18 @@ static VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( return VK_FALSE; } + /* + "Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used." + Ignoring because we map entire VkDeviceMemory blocks, where different types of + images and buffers may end up together, especially on GPUs with unified memory + like Intel. + */ + if(strstr(pMessage, "Mapping an image with layout") != nullptr && + strstr(pMessage, "can result in undefined behavior if this memory is used by the device") != nullptr) + { + return VK_FALSE; + } + printf("%s \xBA %s\n", pLayerPrefix, pMessage); return VK_FALSE; @@ -411,6 +455,7 @@ private: static const size_t MAX_WARNINGS_TO_SHOW = 64; size_t m_WarningCount = 0; + bool m_AllocateForBufferImageWarningIssued = false; VkInstance m_VulkanInstance = VK_NULL_HANDLE; VkPhysicalDevice m_PhysicalDevice = VK_NULL_HANDLE; @@ -470,6 +515,8 @@ private: void ExecuteDestroyImage(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } void ExecuteFreeMemory(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } void ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvSplit& csvSplit, OBJECT_TYPE objType); void DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit); }; @@ -596,6 +643,12 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) ExecuteFreeMemory(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaCreateLostAllocation")) ExecuteCreateLostAllocation(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaAllocateMemory")) + ExecuteAllocateMemory(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaAllocateMemoryForBuffer")) + ExecuteAllocateMemoryForBufferOrImage(lineNumber, csvSplit, OBJECT_TYPE::BUFFER); + else if(StrRangeEq(functionName, "vmaAllocateMemoryForImage")) + ExecuteAllocateMemoryForBufferOrImage(lineNumber, csvSplit, OBJECT_TYPE::IMAGE); else { if(IssueWarning()) @@ -1288,7 +1341,51 @@ void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvS if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) { - // TODO + Allocation allocDesc = {}; + vmaCreateLostAllocation(m_Allocator, &allocDesc.allocation); + m_Stats.RegisterCreateAllocation(); + + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } + } + + m_Allocations[origPtr] = allocDesc; + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaCreateLostAllocation.\n", lineNumber); + } + } + } +} + +void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 11, true)) + { + VkMemoryRequirements memReq = {}; + VmaAllocationCreateInfo allocCreateInfo = {}; + uint64_t origPool = 0; + uint64_t origPtr = 0; + + if(StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX), memReq.size) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), memReq.alignment) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), memReq.memoryTypeBits) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 3), allocCreateInfo.flags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 4), (uint32_t&)allocCreateInfo.usage) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), allocCreateInfo.requiredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), allocCreateInfo.preferredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 7), allocCreateInfo.memoryTypeBits) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 8), origPool) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 9), origPtr)) + { if(origPool != 0) { const auto poolIt = m_Pools.find(origPool); @@ -1304,16 +1401,16 @@ void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvS } Allocation allocDesc = {}; - VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); + VkResult res = vmaAllocateMemory(m_Allocator, &memReq, &allocCreateInfo, &allocDesc.allocation, nullptr); if(res == VK_SUCCESS) { - m_Stats.RegisterCreateBuffer(bufCreateInfo.usage); + m_Stats.RegisterCreateAllocation(); } else { if(IssueWarning()) { - printf("Line %zu: vmaCreateBuffer failed (%u).\n", lineNumber, res); + printf("Line %zu: vmaAllocateMemory failed (%u).\n", lineNumber, res); } } @@ -1332,7 +1429,91 @@ void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvS { if(IssueWarning()) { - printf("Line %zu: Invalid parameters for vmaCreateLostAllocation.\n", lineNumber); + printf("Line %zu: Invalid parameters for vmaAllocateMemory.\n", lineNumber); + } + } + } +} + +void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvSplit& csvSplit, OBJECT_TYPE objType) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 13, true)) + { + VkMemoryRequirements memReq = {}; + VmaAllocationCreateInfo allocCreateInfo = {}; + bool requiresDedicatedAllocation = false; + bool prefersDedicatedAllocation = false; + uint64_t origPool = 0; + uint64_t origPtr = 0; + + if(StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX), memReq.size) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), memReq.alignment) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), memReq.memoryTypeBits) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 3), allocCreateInfo.flags) && + StrRangeToBool(csvSplit.GetRange(FIRST_PARAM_INDEX + 4), requiresDedicatedAllocation) && + StrRangeToBool(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), prefersDedicatedAllocation) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), (uint32_t&)allocCreateInfo.usage) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 7), allocCreateInfo.requiredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 8), allocCreateInfo.preferredFlags) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 9), allocCreateInfo.memoryTypeBits) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 10), origPool) && + StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 11), origPtr)) + { + if(origPool != 0) + { + const auto poolIt = m_Pools.find(origPool); + if(poolIt != m_Pools.end()) + allocCreateInfo.pool = poolIt->second.pool; + else + { + if(IssueWarning()) + { + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); + } + } + } + + if(requiresDedicatedAllocation || prefersDedicatedAllocation) + { + allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + } + + if(!m_AllocateForBufferImageWarningIssued) + { + if(IssueWarning()) + { + printf("Line %zu: vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage cannot be replayed accurately. Using vmaCreateAllocation instead.\n", lineNumber); + } + m_AllocateForBufferImageWarningIssued = true; + } + + Allocation allocDesc = {}; + VkResult res = vmaAllocateMemory(m_Allocator, &memReq, &allocCreateInfo, &allocDesc.allocation, nullptr); + if(res == VK_SUCCESS) + { + m_Stats.RegisterCreateAllocation(); + } + else + { + printf("Line %zu: vmaAllocateMemory (called as vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage) failed (%u).\n", lineNumber, res); + } + + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } + } + + m_Allocations[origPtr] = allocDesc; + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage.\n", lineNumber); } } } diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index b8f4135..b3223d2 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -9353,32 +9353,6 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) #endif // #if VMA_STATS_STRING_ENABLED -static VkResult AllocateMemoryForImage( - VmaAllocator allocator, - VkImage image, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - VmaSuballocationType suballocType, - VmaAllocation* pAllocation) -{ - VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation); - - VkMemoryRequirements vkMemReq = {}; - bool requiresDedicatedAllocation = false; - bool prefersDedicatedAllocation = false; - allocator->GetImageMemoryRequirements(image, vkMemReq, - requiresDedicatedAllocation, prefersDedicatedAllocation); - - return allocator->AllocateMemory( - vkMemReq, - requiresDedicatedAllocation, - prefersDedicatedAllocation, - VK_NULL_HANDLE, // dedicatedBuffer - image, // dedicatedImage - *pAllocationCreateInfo, - suballocType, - pAllocation); -} - //////////////////////////////////////////////////////////////////////////////// // Public interface @@ -9857,7 +9831,6 @@ VkResult vmaAllocateMemory( VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { - Crash(); VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemory"); @@ -9874,6 +9847,28 @@ VkResult vmaAllocateMemory( VMA_SUBALLOCATION_TYPE_UNKNOWN, pAllocation); + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, + pVkMemoryRequirements->size, + pVkMemoryRequirements->alignment, + pVkMemoryRequirements->memoryTypeBits, + pCreateInfo->flags, + pCreateInfo->usage, + pCreateInfo->requiredFlags, + pCreateInfo->preferredFlags, + pCreateInfo->memoryTypeBits, + pCreateInfo->pool, + *pAllocation, + pCreateInfo->pUserData ? (const char*)pCreateInfo->pUserData : ""); + fflush(g_File); + } + if(pAllocationInfo && result == VK_SUCCESS) { allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); @@ -9889,7 +9884,6 @@ VkResult vmaAllocateMemoryForBuffer( VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { - Crash(); VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer"); @@ -9913,6 +9907,30 @@ VkResult vmaAllocateMemoryForBuffer( VMA_SUBALLOCATION_TYPE_BUFFER, pAllocation); + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, + vkMemReq.size, + vkMemReq.alignment, + vkMemReq.memoryTypeBits, + requiresDedicatedAllocation ? 1 : 0, + prefersDedicatedAllocation ? 1 : 0, + pCreateInfo->flags, + pCreateInfo->usage, + pCreateInfo->requiredFlags, + pCreateInfo->preferredFlags, + pCreateInfo->memoryTypeBits, + pCreateInfo->pool, + *pAllocation, + pCreateInfo->pUserData ? (const char*)pCreateInfo->pUserData : ""); + fflush(g_File); + } + if(pAllocationInfo && result == VK_SUCCESS) { allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); @@ -9935,13 +9953,46 @@ VkResult vmaAllocateMemoryForImage( VMA_DEBUG_GLOBAL_MUTEX_LOCK - VkResult result = AllocateMemoryForImage( - allocator, - image, - pCreateInfo, + VkMemoryRequirements vkMemReq = {}; + bool requiresDedicatedAllocation = false; + bool prefersDedicatedAllocation = false; + allocator->GetImageMemoryRequirements(image, vkMemReq, + requiresDedicatedAllocation, prefersDedicatedAllocation); + + VkResult result = allocator->AllocateMemory( + vkMemReq, + requiresDedicatedAllocation, + prefersDedicatedAllocation, + VK_NULL_HANDLE, // dedicatedBuffer + image, // dedicatedImage + *pCreateInfo, VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN, pAllocation); + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, + vkMemReq.size, + vkMemReq.alignment, + vkMemReq.memoryTypeBits, + requiresDedicatedAllocation ? 1 : 0, + prefersDedicatedAllocation ? 1 : 0, + pCreateInfo->flags, + pCreateInfo->usage, + pCreateInfo->requiredFlags, + pCreateInfo->preferredFlags, + pCreateInfo->memoryTypeBits, + pCreateInfo->pool, + *pAllocation, + pCreateInfo->pUserData ? (const char*)pCreateInfo->pUserData : ""); + fflush(g_File); + } + if(pAllocationInfo && result == VK_SUCCESS) { allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); @@ -10329,7 +10380,21 @@ VkResult vmaCreateImage( VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR; // 2. Allocate memory using allocator. - res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation); + VkMemoryRequirements vkMemReq = {}; + bool requiresDedicatedAllocation = false; + bool prefersDedicatedAllocation = false; + allocator->GetImageMemoryRequirements(*pImage, vkMemReq, + requiresDedicatedAllocation, prefersDedicatedAllocation); + + res = allocator->AllocateMemory( + vkMemReq, + requiresDedicatedAllocation, + prefersDedicatedAllocation, + VK_NULL_HANDLE, // dedicatedBuffer + *pImage, // dedicatedImage + *pAllocationCreateInfo, + suballocType, + pAllocation); if(res >= 0) { // 3. Bind image with memory. From 29785d1169e88bccd85dd4e058d86dd77d084ba5 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 13:42:13 +0200 Subject: [PATCH 23/51] VmaReplay.cpp, Player class: Small refactoring. --- src/VmaReplay/VmaReplay.cpp | 158 ++++++++++++------------------------ 1 file changed, 54 insertions(+), 104 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 8eeb51b..0bf3396 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -496,6 +496,13 @@ private: void Destroy(const Allocation& alloc); + // Finds VmaPool bu original pointer. + // If origPool = null, returns true and outPool = null. + // If failed, prints warning, returns false and outPool = null. + bool FindPool(size_t lineNumber, uint64_t origPool, VmaPool& outPool); + // If allocation with that origPtr already exists, prints warning and replaces it. + void AddAllocation(size_t lineNumber, uint64_t origPtr, Allocation&& allocDesc); + // Increments warning counter. Returns true if warning message should be printed. bool IssueWarning(); @@ -681,6 +688,44 @@ void Player::Destroy(const Allocation& alloc) vmaFreeMemory(m_Allocator, alloc.allocation); } +bool Player::FindPool(size_t lineNumber, uint64_t origPool, VmaPool& outPool) +{ + outPool = VK_NULL_HANDLE; + + if(origPool != 0) + { + const auto poolIt = m_Pools.find(origPool); + if(poolIt != m_Pools.end()) + { + outPool = poolIt->second.pool; + return true; + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); + } + } + } + + return true; +} + +void Player::AddAllocation(size_t lineNumber, uint64_t origPtr, Allocation&& allocDesc) +{ + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } + } + + m_Allocations[origPtr] = std::move(allocDesc); +} + bool Player::IssueWarning() { if(g_Verbosity < VERBOSITY::MAXIMUM) @@ -1168,19 +1213,7 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 9), origPool) && StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 10), origPtr)) { - if(origPool != 0) - { - const auto poolIt = m_Pools.find(origPool); - if(poolIt != m_Pools.end()) - allocCreateInfo.pool = poolIt->second.pool; - else - { - if(IssueWarning()) - { - printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); - } - } - } + FindPool(lineNumber, origPool, allocCreateInfo.pool); Allocation allocDesc = {}; VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); @@ -1196,16 +1229,7 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) } } - const auto existingIt = m_Allocations.find(origPtr); - if(existingIt != m_Allocations.end()) - { - if(IssueWarning()) - { - printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); - } - } - - m_Allocations[origPtr] = allocDesc; + AddAllocation(lineNumber, origPtr, std::move(allocDesc)); } else { @@ -1282,21 +1306,7 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 18), origPool) && StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 19), origPtr)) { - if(origPool != 0) - { - const auto poolIt = m_Pools.find(origPool); - if(poolIt != m_Pools.end()) - { - allocCreateInfo.pool = poolIt->second.pool; - } - else - { - if(IssueWarning()) - { - printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); - } - } - } + FindPool(lineNumber, origPool, allocCreateInfo.pool); Allocation allocDesc = {}; VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &allocDesc.image, &allocDesc.allocation, nullptr); @@ -1312,16 +1322,7 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) } } - const auto existingIt = m_Allocations.find(origPtr); - if(existingIt != m_Allocations.end()) - { - if(IssueWarning()) - { - printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); - } - } - - m_Allocations[origPtr] = allocDesc; + AddAllocation(lineNumber, origPtr, std::move(allocDesc)); } else { @@ -1345,16 +1346,7 @@ void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvS vmaCreateLostAllocation(m_Allocator, &allocDesc.allocation); m_Stats.RegisterCreateAllocation(); - const auto existingIt = m_Allocations.find(origPtr); - if(existingIt != m_Allocations.end()) - { - if(IssueWarning()) - { - printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); - } - } - - m_Allocations[origPtr] = allocDesc; + AddAllocation(lineNumber, origPtr, std::move(allocDesc)); } else { @@ -1386,19 +1378,7 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 8), origPool) && StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 9), origPtr)) { - if(origPool != 0) - { - const auto poolIt = m_Pools.find(origPool); - if(poolIt != m_Pools.end()) - allocCreateInfo.pool = poolIt->second.pool; - else - { - if(IssueWarning()) - { - printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); - } - } - } + FindPool(lineNumber, origPool, allocCreateInfo.pool); Allocation allocDesc = {}; VkResult res = vmaAllocateMemory(m_Allocator, &memReq, &allocCreateInfo, &allocDesc.allocation, nullptr); @@ -1414,16 +1394,7 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) } } - const auto existingIt = m_Allocations.find(origPtr); - if(existingIt != m_Allocations.end()) - { - if(IssueWarning()) - { - printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); - } - } - - m_Allocations[origPtr] = allocDesc; + AddAllocation(lineNumber, origPtr, std::move(allocDesc)); } else { @@ -1459,19 +1430,7 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 10), origPool) && StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 11), origPtr)) { - if(origPool != 0) - { - const auto poolIt = m_Pools.find(origPool); - if(poolIt != m_Pools.end()) - allocCreateInfo.pool = poolIt->second.pool; - else - { - if(IssueWarning()) - { - printf("Line %zu: Pool %llX not found.\n", lineNumber, origPool); - } - } - } + FindPool(lineNumber, origPool, allocCreateInfo.pool); if(requiresDedicatedAllocation || prefersDedicatedAllocation) { @@ -1498,16 +1457,7 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS printf("Line %zu: vmaAllocateMemory (called as vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage) failed (%u).\n", lineNumber, res); } - const auto existingIt = m_Allocations.find(origPtr); - if(existingIt != m_Allocations.end()) - { - if(IssueWarning()) - { - printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); - } - } - - m_Allocations[origPtr] = allocDesc; + AddAllocation(lineNumber, origPtr, std::move(allocDesc)); } else { From 5557b0ed63354931222e5de48494c9b3fb5e9e6d Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 13:54:54 +0200 Subject: [PATCH 24/51] Added support for vmaMapMemory, vmaUnmapMemory, vmaFlushAllocation, vmaInvalidateAllocation. --- src/VmaReplay/VmaReplay.cpp | 160 ++++++++++++++++++++++++++++++++++++ src/vk_mem_alloc.h | 52 ++++++++++++ 2 files changed, 212 insertions(+) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 0bf3396..7cf827a 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -524,6 +524,10 @@ private: void ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvSplit& csvSplit, OBJECT_TYPE objType); + void ExecuteMapMemory(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteUnmapMemory(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteFlushAllocation(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteInvalidateAllocation(size_t lineNumber, const CsvSplit& csvSplit); void DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit); }; @@ -656,6 +660,14 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) ExecuteAllocateMemoryForBufferOrImage(lineNumber, csvSplit, OBJECT_TYPE::BUFFER); else if(StrRangeEq(functionName, "vmaAllocateMemoryForImage")) ExecuteAllocateMemoryForBufferOrImage(lineNumber, csvSplit, OBJECT_TYPE::IMAGE); + else if(StrRangeEq(functionName, "vmaMapMemory")) + ExecuteMapMemory(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaUnmapMemory")) + ExecuteUnmapMemory(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaFlushAllocation")) + ExecuteFlushAllocation(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaInvalidateAllocation")) + ExecuteInvalidateAllocation(lineNumber, csvSplit); else { if(IssueWarning()) @@ -1469,6 +1481,154 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS } } +void Player::ExecuteMapMemory(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origPtr = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + if(origPtr != 0) + { + const auto it = m_Allocations.find(origPtr); + if(it != m_Allocations.end()) + { + void* pData; + VkResult res = vmaMapMemory(m_Allocator, it->second.allocation, &pData); + if(res != VK_SUCCESS) + { + printf("Line %zu: vmaMapMemory failed (%u)\n", lineNumber, res); + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origPtr); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaMapMemory.\n", lineNumber); + } + } + } +} + +void Player::ExecuteUnmapMemory(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origPtr = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + if(origPtr != 0) + { + const auto it = m_Allocations.find(origPtr); + if(it != m_Allocations.end()) + { + vmaUnmapMemory(m_Allocator, it->second.allocation); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origPtr); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaMapMemory.\n", lineNumber); + } + } + } +} + +void Player::ExecuteFlushAllocation(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 3, false)) + { + uint64_t origPtr = 0; + uint64_t offset = 0; + uint64_t size = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), offset) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), size)) + { + if(origPtr != 0) + { + const auto it = m_Allocations.find(origPtr); + if(it != m_Allocations.end()) + { + vmaFlushAllocation(m_Allocator, it->second.allocation, offset, size); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origPtr); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaFlushAllocation.\n", lineNumber); + } + } + } +} + +void Player::ExecuteInvalidateAllocation(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 3, false)) + { + uint64_t origPtr = 0; + uint64_t offset = 0; + uint64_t size = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 1), offset) && + StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 2), size)) + { + if(origPtr != 0) + { + const auto it = m_Allocations.find(origPtr); + if(it != m_Allocations.end()) + { + vmaInvalidateAllocation(m_Allocator, it->second.allocation, offset, size); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origPtr); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaInvalidateAllocation.\n", lineNumber); + } + } + } +} //////////////////////////////////////////////////////////////////////////////// // Main functions diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index b3223d2..1692147 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -10107,6 +10107,18 @@ VkResult vmaMapMemory( { VMA_ASSERT(allocator && allocation && ppData); + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaMapMemory,%p\n", threadId, time, frameIndex, + allocation); + fflush(g_File); + } + VMA_DEBUG_GLOBAL_MUTEX_LOCK return allocator->Map(allocation, ppData); @@ -10118,6 +10130,18 @@ void vmaUnmapMemory( { VMA_ASSERT(allocator && allocation); + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", threadId, time, frameIndex, + allocation); + fflush(g_File); + } + VMA_DEBUG_GLOBAL_MUTEX_LOCK allocator->Unmap(allocation); @@ -10129,6 +10153,20 @@ void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDevi VMA_DEBUG_LOG("vmaFlushAllocation"); + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", threadId, time, frameIndex, + allocation, + offset, + size); + fflush(g_File); + } + VMA_DEBUG_GLOBAL_MUTEX_LOCK allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH); @@ -10140,6 +10178,20 @@ void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, V VMA_DEBUG_LOG("vmaInvalidateAllocation"); + { + VmaMutexLock lock(g_FileMutex, true); + EnsureFile(); + LARGE_INTEGER counter; QueryPerformanceCounter(&counter); + const DWORD threadId = GetCurrentThreadId(); + const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; + const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); + fprintf(g_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", threadId, time, frameIndex, + allocation, + offset, + size); + fflush(g_File); + } + VMA_DEBUG_GLOBAL_MUTEX_LOCK allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE); From daec26e5eafe06102d30040985b4f47e3f5747fa Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 14:03:46 +0200 Subject: [PATCH 25/51] Minor fixes. VkResult is printed as signed int. --- src/VmaReplay/VmaReplay.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 7cf827a..cf06fd4 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -806,7 +806,7 @@ int Player::InitVulkan() res = vkCreateInstance(&instInfo, NULL, &m_VulkanInstance); if(res != VK_SUCCESS) { - printf("ERROR: vkCreateInstance failed (%u)\n", res); + printf("ERROR: vkCreateInstance failed (%d)\n", res); return RESULT_ERROR_VULKAN; } @@ -914,7 +914,7 @@ int Player::InitVulkan() res = vkCreateDevice(m_PhysicalDevice, &deviceCreateInfo, nullptr, &m_Device); if(res != VK_SUCCESS) { - printf("ERROR: vkCreateDevice failed (%u)\n", res); + printf("ERROR: vkCreateDevice failed (%d)\n", res); return RESULT_ERROR_VULKAN; } @@ -932,7 +932,7 @@ int Player::InitVulkan() res = vmaCreateAllocator(&allocatorInfo, &m_Allocator); if(res != VK_SUCCESS) { - printf("ERROR: vmaCreateAllocator failed (%u)\n", res); + printf("ERROR: vmaCreateAllocator failed (%d)\n", res); return RESULT_ERROR_VULKAN; } @@ -1144,7 +1144,7 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) { if(IssueWarning()) { - printf("Line %zu: vmaCreatePool failed (%u).\n", lineNumber, res); + printf("Line %zu: vmaCreatePool failed (%d).\n", lineNumber, res); } } @@ -1237,7 +1237,7 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) { if(IssueWarning()) { - printf("Line %zu: vmaCreateBuffer failed (%u).\n", lineNumber, res); + printf("Line %zu: vmaCreateBuffer failed (%d).\n", lineNumber, res); } } @@ -1330,7 +1330,7 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) { if(IssueWarning()) { - printf("Line %zu: vmaCreateImage failed (%u).\n", lineNumber, res); + printf("Line %zu: vmaCreateImage failed (%d).\n", lineNumber, res); } } @@ -1402,7 +1402,7 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) { if(IssueWarning()) { - printf("Line %zu: vmaAllocateMemory failed (%u).\n", lineNumber, res); + printf("Line %zu: vmaAllocateMemory failed (%d).\n", lineNumber, res); } } @@ -1466,7 +1466,7 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS } else { - printf("Line %zu: vmaAllocateMemory (called as vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage) failed (%u).\n", lineNumber, res); + printf("Line %zu: vmaAllocateMemory (called as vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage) failed (%d).\n", lineNumber, res); } AddAllocation(lineNumber, origPtr, std::move(allocDesc)); @@ -1498,7 +1498,7 @@ void Player::ExecuteMapMemory(size_t lineNumber, const CsvSplit& csvSplit) VkResult res = vmaMapMemory(m_Allocator, it->second.allocation, &pData); if(res != VK_SUCCESS) { - printf("Line %zu: vmaMapMemory failed (%u)\n", lineNumber, res); + printf("Line %zu: vmaMapMemory failed (%d)\n", lineNumber, res); } } else @@ -1670,6 +1670,11 @@ static int ProcessFile(const char* data, size_t numBytes) return RESULT_ERROR_FORMAT; } + if(g_Verbosity == VERBOSITY::MAXIMUM) + { + printf("Format version: %u,%u\n", g_FileVersion >> 16, g_FileVersion & 0xFFFF); + } + Player player; int result = player.Init(); if(result == 0) From 0e0f20fd5709b63bae572d46e90bda50aa352b19 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 15:34:55 +0200 Subject: [PATCH 26/51] Major refactoring: Defined clean API for recording, added VmaRecordFlagBits, VmaRecordFlags, VmaRecordSettings, VmaAllocatorCreateInfo::pRecordSettings. Created VmaRecorder class. --- src/vk_mem_alloc.h | 925 ++++++++++++++++++++++++++++++--------------- 1 file changed, 616 insertions(+), 309 deletions(-) diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 1692147..3167490 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -1317,6 +1317,19 @@ typedef struct VmaVulkanFunctions { #endif } VmaVulkanFunctions; +typedef enum VmaRecordFlagBits { + VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001, + VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF +} VmaRecordFlagBits; +typedef VkFlags VmaRecordFlags; + +/// TODO +typedef struct VmaRecordSettings +{ + VmaRecordFlags flags; + const char* pFilePath; +} VmaRecordSettings; + /// Description of a Allocator to be created. typedef struct VmaAllocatorCreateInfo { @@ -1387,6 +1400,9 @@ typedef struct VmaAllocatorCreateInfo e.g. fetched using `vkGetInstanceProcAddr()` and `vkGetDeviceProcAddr()`. */ const VmaVulkanFunctions* pVulkanFunctions; + /** TODO + */ + const VmaRecordSettings* pRecordSettings; } VmaAllocatorCreateInfo; /// Creates Allocator object. @@ -2676,29 +2692,6 @@ static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF; END OF CONFIGURATION */ -// TEMP ADDED - -void Crash() -{ - int* i = 0; *i = 0; -} - -FILE* g_File; -LARGE_INTEGER g_Freq, g_StartCounter; -VMA_MUTEX g_FileMutex; - -void EnsureFile() -{ - if(!g_File) - { - fopen_s(&g_File, "VMA_Usage_Dump", "wb"); - fprintf(g_File, "%s\n", "Vulkan Memory Allocator,Calls recording"); - fprintf(g_File, "%s\n", "1,2"); - QueryPerformanceFrequency(&g_Freq); - QueryPerformanceCounter(&g_StartCounter); - } -} - static VkAllocationCallbacks VmaEmptyAllocationCallbacks = { VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL }; @@ -2735,6 +2728,11 @@ inline T VmaRoundDiv(T x, T y) return (x + (y / (T)2)) / y; } +static inline bool VmaStrIsEmpty(const char* pStr) +{ + return pStr == VMA_NULL || *pStr == '\0'; +} + #ifndef VMA_SORT template @@ -4716,6 +4714,81 @@ public: uint32_t maxAllocationsToMove); }; +class VmaRecorder +{ +public: + VmaRecorder(); + VkResult Init(const VmaRecordSettings& settings, bool useMutex); + ~VmaRecorder(); + + void RecordCreateAllocator(uint32_t frameIndex); + void RecordDestroyAllocator(uint32_t frameIndex); + void RecordCreatePool(uint32_t frameIndex, + const VmaPoolCreateInfo& createInfo, + VmaPool pool); + void RecordDestroyPool(uint32_t frameIndex, VmaPool pool); + void RecordAllocateMemory(uint32_t frameIndex, + const VkMemoryRequirements& vkMemReq, + const VmaAllocationCreateInfo& createInfo, + VmaAllocation allocation); + void RecordAllocateMemoryForBuffer(uint32_t frameIndex, + const VkMemoryRequirements& vkMemReq, + bool requiresDedicatedAllocation, + bool prefersDedicatedAllocation, + const VmaAllocationCreateInfo& createInfo, + VmaAllocation allocation); + void RecordAllocateMemoryForImage(uint32_t frameIndex, + const VkMemoryRequirements& vkMemReq, + bool requiresDedicatedAllocation, + bool prefersDedicatedAllocation, + const VmaAllocationCreateInfo& createInfo, + VmaAllocation allocation); + void RecordFreeMemory(uint32_t frameIndex, + VmaAllocation allocation); + void RecordSetAllocationUserData(uint32_t frameIndex, + VmaAllocation allocation, + const void* pUserData); + void RecordCreateLostAllocation(uint32_t frameIndex, + VmaAllocation allocation); + void RecordMapMemory(uint32_t frameIndex, + VmaAllocation allocation); + void RecordUnmapMemory(uint32_t frameIndex, + VmaAllocation allocation); + void RecordFlushAllocation(uint32_t frameIndex, + VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size); + void RecordInvalidateAllocation(uint32_t frameIndex, + VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size); + void RecordCreateBuffer(uint32_t frameIndex, + const VkBufferCreateInfo& bufCreateInfo, + const VmaAllocationCreateInfo& allocCreateInfo, + VmaAllocation allocation); + void RecordCreateImage(uint32_t frameIndex, + const VkImageCreateInfo& imageCreateInfo, + const VmaAllocationCreateInfo& allocCreateInfo, + VmaAllocation allocation); + void RecordDestroyBuffer(uint32_t frameIndex, + VmaAllocation allocation); + void RecordDestroyImage(uint32_t frameIndex, + VmaAllocation allocation); + +private: + struct CallParams + { + uint32_t threadId; + double time; + }; + + bool m_UseMutex; + VmaRecordFlags m_Flags; + FILE* m_File; + VMA_MUTEX m_FileMutex; + int64_t m_Freq; + int64_t m_StartCounter; + + void GetBasicParams(CallParams& outParams); + void Flush(); +}; + // Main allocator object. struct VmaAllocator_T { @@ -4744,6 +4817,7 @@ public: VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES]; VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo); + VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo); ~VmaAllocator_T(); const VkAllocationCallbacks* GetAllocationCallbacks() const @@ -4789,6 +4863,8 @@ public: return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; } + VmaRecorder* GetRecorder() const { return m_pRecorder; } + void GetBufferMemoryRequirements( VkBuffer hBuffer, VkMemoryRequirements& memReq, @@ -4874,6 +4950,8 @@ private: VmaVulkanFunctions m_VulkanFunctions; + VmaRecorder* m_pRecorder; + void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions); VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex); @@ -7951,6 +8029,366 @@ bool VmaDefragmentator::MoveMakesSense( return false; } +//////////////////////////////////////////////////////////////////////////////// +// VmaRecorder + +VmaRecorder::VmaRecorder() : + m_UseMutex(true), + m_Flags(0), + m_File(VMA_NULL), + m_Freq(INT64_MAX), + m_StartCounter(INT64_MAX) +{ +} + +VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex) +{ + m_UseMutex = useMutex; + m_Flags = settings.flags; + + QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq); + QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter); + + // Open file for writing. + errno_t err = fopen_s(&m_File, settings.pFilePath, "wb"); + if(err != 0) + { + return VK_ERROR_INITIALIZATION_FAILED; + } + + // Write header. + fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording"); + fprintf(m_File, "%s\n", "1,2"); + + return VK_SUCCESS; +} + +VmaRecorder::~VmaRecorder() +{ + if(m_File != VMA_NULL) + { + fclose(m_File); + } +} + +void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex); + Flush(); +} + +void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex); + Flush(); +} + +void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex, + createInfo.memoryTypeIndex, + createInfo.flags, + createInfo.blockSize, + createInfo.minBlockCount, + createInfo.maxBlockCount, + createInfo.frameInUseCount, + pool); + Flush(); +} + +void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex, + pool); + Flush(); +} + +void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex, + const VkMemoryRequirements& vkMemReq, + const VmaAllocationCreateInfo& createInfo, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, + vkMemReq.size, + vkMemReq.alignment, + vkMemReq.memoryTypeBits, + createInfo.flags, + createInfo.usage, + createInfo.requiredFlags, + createInfo.preferredFlags, + createInfo.memoryTypeBits, + createInfo.pool, + allocation, + createInfo.pUserData ? (const char*)createInfo.pUserData : ""); + Flush(); +} + +void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex, + const VkMemoryRequirements& vkMemReq, + bool requiresDedicatedAllocation, + bool prefersDedicatedAllocation, + const VmaAllocationCreateInfo& createInfo, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, + vkMemReq.size, + vkMemReq.alignment, + vkMemReq.memoryTypeBits, + requiresDedicatedAllocation ? 1 : 0, + prefersDedicatedAllocation ? 1 : 0, + createInfo.flags, + createInfo.usage, + createInfo.requiredFlags, + createInfo.preferredFlags, + createInfo.memoryTypeBits, + createInfo.pool, + allocation, + createInfo.pUserData ? (const char*)createInfo.pUserData : ""); + Flush(); +} + +void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex, + const VkMemoryRequirements& vkMemReq, + bool requiresDedicatedAllocation, + bool prefersDedicatedAllocation, + const VmaAllocationCreateInfo& createInfo, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, + vkMemReq.size, + vkMemReq.alignment, + vkMemReq.memoryTypeBits, + requiresDedicatedAllocation ? 1 : 0, + prefersDedicatedAllocation ? 1 : 0, + createInfo.flags, + createInfo.usage, + createInfo.requiredFlags, + createInfo.preferredFlags, + createInfo.memoryTypeBits, + createInfo.pool, + allocation, + createInfo.pUserData ? (const char*)createInfo.pUserData : ""); + Flush(); +} + +void VmaRecorder::RecordFreeMemory(uint32_t frameIndex, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex, + allocation); + Flush(); +} + +void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex, + VmaAllocation allocation, + const void* pUserData) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex, + allocation, + pUserData ? (const char*)pUserData : ""); + Flush(); +} + +void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex, + allocation); + Flush(); +} + +void VmaRecorder::RecordMapMemory(uint32_t frameIndex, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex, + allocation); + Flush(); +} + +void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex, + allocation); + Flush(); +} + +void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex, + VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex, + allocation, + offset, + size); + Flush(); +} + +void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex, + VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex, + allocation, + offset, + size); + Flush(); +} + +void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex, + const VkBufferCreateInfo& bufCreateInfo, + const VmaAllocationCreateInfo& allocCreateInfo, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, + bufCreateInfo.flags, + bufCreateInfo.size, + bufCreateInfo.usage, + bufCreateInfo.sharingMode, + allocCreateInfo.flags, + allocCreateInfo.usage, + allocCreateInfo.requiredFlags, + allocCreateInfo.preferredFlags, + allocCreateInfo.memoryTypeBits, + allocCreateInfo.pool, + allocation, + allocCreateInfo.pUserData ? (const char*)allocCreateInfo.pUserData : ""); + Flush(); +} + +void VmaRecorder::RecordCreateImage(uint32_t frameIndex, + const VkImageCreateInfo& imageCreateInfo, + const VmaAllocationCreateInfo& allocCreateInfo, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, + imageCreateInfo.flags, + imageCreateInfo.imageType, + imageCreateInfo.format, + imageCreateInfo.extent.width, + imageCreateInfo.extent.height, + imageCreateInfo.extent.depth, + imageCreateInfo.mipLevels, + imageCreateInfo.arrayLayers, + imageCreateInfo.samples, + imageCreateInfo.tiling, + imageCreateInfo.usage, + imageCreateInfo.sharingMode, + imageCreateInfo.initialLayout, + allocCreateInfo.flags, + allocCreateInfo.usage, + allocCreateInfo.requiredFlags, + allocCreateInfo.preferredFlags, + allocCreateInfo.memoryTypeBits, + allocCreateInfo.pool, + allocation, + allocCreateInfo.pUserData ? (const char*)allocCreateInfo.pUserData : ""); + Flush(); +} + +void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex, + allocation); + Flush(); +} + +void VmaRecorder::RecordDestroyImage(uint32_t frameIndex, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex, + allocation); + Flush(); +} + +void VmaRecorder::GetBasicParams(CallParams& outParams) +{ + outParams.threadId = GetCurrentThreadId(); + + LARGE_INTEGER counter; + QueryPerformanceCounter(&counter); + outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq; +} + +void VmaRecorder::Flush() +{ + if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0) + { + fflush(m_File); + } +} + //////////////////////////////////////////////////////////////////////////////// // VmaAllocator_T @@ -7965,7 +8403,8 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : m_PhysicalDevice(pCreateInfo->physicalDevice), m_CurrentFrameIndex(0), m_Pools(VmaStlAllocator(GetAllocationCallbacks())), - m_NextPoolId(0) + m_NextPoolId(0), + m_pRecorder(VMA_NULL) { if(VMA_DEBUG_DETECT_CORRUPTION) { @@ -8044,8 +8483,37 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : } } +VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo) +{ + VkResult res = VK_SUCCESS; + + if(pCreateInfo->pRecordSettings != VMA_NULL && + !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath)) + { + m_pRecorder = vma_new(this, VmaRecorder)(); + res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex); + if(res != VK_SUCCESS) + { + return res; + } + } + + if(m_pRecorder != VMA_NULL) + { + m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex()); + } + + return res; +} + VmaAllocator_T::~VmaAllocator_T() { + if(m_pRecorder != VMA_NULL) + { + m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex()); + vma_delete(this, m_pRecorder); + } + VMA_ASSERT(m_Pools.empty()); for(size_t i = GetMemoryTypeCount(); i--; ) @@ -9363,19 +9831,7 @@ VkResult vmaCreateAllocator( VMA_ASSERT(pCreateInfo && pAllocator); VMA_DEBUG_LOG("vmaCreateAllocator"); *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo); - - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = (*pAllocator)->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaCreateAllocator\n", threadId, time, frameIndex); - fflush(g_File); - } - - return VK_SUCCESS; + return (*pAllocator)->Init(pCreateInfo); } void vmaDestroyAllocator( @@ -9383,17 +9839,6 @@ void vmaDestroyAllocator( { if(allocator != VK_NULL_HANDLE) { - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaDestroyAllocator\n", threadId, time, frameIndex); - fflush(g_File); - } - VMA_DEBUG_LOG("vmaDestroyAllocator"); VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks; vma_delete(&allocationCallbacks, allocator); @@ -9731,31 +10176,18 @@ VkResult vmaCreatePool( VmaPool* pPool) { VMA_ASSERT(allocator && pCreateInfo && pPool); - + VMA_DEBUG_LOG("vmaCreatePool"); - + VMA_DEBUG_GLOBAL_MUTEX_LOCK - + VkResult res = allocator->CreatePool(pCreateInfo, pPool); - + + if(res == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", threadId, time, frameIndex, - pCreateInfo->memoryTypeIndex, - pCreateInfo->flags, - pCreateInfo->blockSize, - pCreateInfo->minBlockCount, - pCreateInfo->maxBlockCount, - pCreateInfo->frameInUseCount, - (*pPool)); - fflush(g_File); + allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool); } - + return res; } @@ -9764,27 +10196,20 @@ void vmaDestroyPool( VmaPool pool) { VMA_ASSERT(allocator); - + if(pool == VK_NULL_HANDLE) { return; } - - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", threadId, time, frameIndex, - pool); - fflush(g_File); - } - + VMA_DEBUG_LOG("vmaDestroyPool"); - + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool); + } allocator->DestroyPool(pool); } @@ -9847,29 +10272,16 @@ VkResult vmaAllocateMemory( VMA_SUBALLOCATION_TYPE_UNKNOWN, pAllocation); + if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, - pVkMemoryRequirements->size, - pVkMemoryRequirements->alignment, - pVkMemoryRequirements->memoryTypeBits, - pCreateInfo->flags, - pCreateInfo->usage, - pCreateInfo->requiredFlags, - pCreateInfo->preferredFlags, - pCreateInfo->memoryTypeBits, - pCreateInfo->pool, - *pAllocation, - pCreateInfo->pUserData ? (const char*)pCreateInfo->pUserData : ""); - fflush(g_File); + allocator->GetRecorder()->RecordAllocateMemory( + allocator->GetCurrentFrameIndex(), + *pVkMemoryRequirements, + *pCreateInfo, + *pAllocation); } - - if(pAllocationInfo && result == VK_SUCCESS) + + if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS) { allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } @@ -9907,28 +10319,15 @@ VkResult vmaAllocateMemoryForBuffer( VMA_SUBALLOCATION_TYPE_BUFFER, pAllocation); + if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, - vkMemReq.size, - vkMemReq.alignment, - vkMemReq.memoryTypeBits, - requiresDedicatedAllocation ? 1 : 0, - prefersDedicatedAllocation ? 1 : 0, - pCreateInfo->flags, - pCreateInfo->usage, - pCreateInfo->requiredFlags, - pCreateInfo->preferredFlags, - pCreateInfo->memoryTypeBits, - pCreateInfo->pool, - *pAllocation, - pCreateInfo->pUserData ? (const char*)pCreateInfo->pUserData : ""); - fflush(g_File); + allocator->GetRecorder()->RecordAllocateMemoryForBuffer( + allocator->GetCurrentFrameIndex(), + vkMemReq, + requiresDedicatedAllocation, + prefersDedicatedAllocation, + *pCreateInfo, + *pAllocation); } if(pAllocationInfo && result == VK_SUCCESS) @@ -9946,7 +10345,6 @@ VkResult vmaAllocateMemoryForImage( VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { - Crash(); VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemoryForImage"); @@ -9969,28 +10367,15 @@ VkResult vmaAllocateMemoryForImage( VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN, pAllocation); + if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, - vkMemReq.size, - vkMemReq.alignment, - vkMemReq.memoryTypeBits, - requiresDedicatedAllocation ? 1 : 0, - prefersDedicatedAllocation ? 1 : 0, - pCreateInfo->flags, - pCreateInfo->usage, - pCreateInfo->requiredFlags, - pCreateInfo->preferredFlags, - pCreateInfo->memoryTypeBits, - pCreateInfo->pool, - *pAllocation, - pCreateInfo->pUserData ? (const char*)pCreateInfo->pUserData : ""); - fflush(g_File); + allocator->GetRecorder()->RecordAllocateMemoryForImage( + allocator->GetCurrentFrameIndex(), + vkMemReq, + requiresDedicatedAllocation, + prefersDedicatedAllocation, + *pCreateInfo, + *pAllocation); } if(pAllocationInfo && result == VK_SUCCESS) @@ -10006,26 +10391,23 @@ void vmaFreeMemory( VmaAllocation allocation) { VMA_ASSERT(allocator); - + if(allocation == VK_NULL_HANDLE) { return; } - - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", threadId, time, frameIndex, - allocation); - fflush(g_File); - } - + VMA_DEBUG_LOG("vmaFreeMemory"); + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordFreeMemory( + allocator->GetCurrentFrameIndex(), + allocation); + } + allocator->FreeMemory(allocation); } @@ -10061,20 +10443,15 @@ void vmaSetAllocationUserData( VMA_DEBUG_GLOBAL_MUTEX_LOCK - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", threadId, time, frameIndex, - allocation, - (const char*)pUserData); - fflush(g_File); - } - allocation->SetUserData(allocator, pUserData); + + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordSetAllocationUserData( + allocator->GetCurrentFrameIndex(), + allocation, + pUserData); + } } void vmaCreateLostAllocation( @@ -10087,16 +10464,11 @@ void vmaCreateLostAllocation( allocator->CreateLostAllocation(pAllocation); + if(allocator->GetRecorder() != VMA_NULL) { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", threadId, time, frameIndex, - (*pAllocation)); - fflush(g_File); + allocator->GetRecorder()->RecordCreateLostAllocation( + allocator->GetCurrentFrameIndex(), + *pAllocation); } } @@ -10107,21 +10479,18 @@ VkResult vmaMapMemory( { VMA_ASSERT(allocator && allocation && ppData); - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaMapMemory,%p\n", threadId, time, frameIndex, - allocation); - fflush(g_File); - } - VMA_DEBUG_GLOBAL_MUTEX_LOCK - return allocator->Map(allocation, ppData); + VkResult res = allocator->Map(allocation, ppData); + + if(res == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordMapMemory( + allocator->GetCurrentFrameIndex(), + allocation); + } + + return res; } void vmaUnmapMemory( @@ -10130,20 +10499,15 @@ void vmaUnmapMemory( { VMA_ASSERT(allocator && allocation); - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", threadId, time, frameIndex, - allocation); - fflush(g_File); - } - VMA_DEBUG_GLOBAL_MUTEX_LOCK + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordUnmapMemory( + allocator->GetCurrentFrameIndex(), + allocation); + } + allocator->Unmap(allocation); } @@ -10153,23 +10517,16 @@ void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDevi VMA_DEBUG_LOG("vmaFlushAllocation"); - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", threadId, time, frameIndex, - allocation, - offset, - size); - fflush(g_File); - } - VMA_DEBUG_GLOBAL_MUTEX_LOCK allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH); + + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordFlushAllocation( + allocator->GetCurrentFrameIndex(), + allocation, offset, size); + } } void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) @@ -10178,23 +10535,16 @@ void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, V VMA_DEBUG_LOG("vmaInvalidateAllocation"); - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", threadId, time, frameIndex, - allocation, - offset, - size); - fflush(g_File); - } - VMA_DEBUG_GLOBAL_MUTEX_LOCK allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE); + + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordInvalidateAllocation( + allocator->GetCurrentFrameIndex(), + allocation, offset, size); + } } VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits) @@ -10328,28 +10678,14 @@ VkResult vmaCreateBuffer( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, - pBufferCreateInfo->flags, - pBufferCreateInfo->size, - pBufferCreateInfo->usage, - pBufferCreateInfo->sharingMode, - pAllocationCreateInfo->flags, - pAllocationCreateInfo->usage, - pAllocationCreateInfo->requiredFlags, - pAllocationCreateInfo->preferredFlags, - pAllocationCreateInfo->memoryTypeBits, - pAllocationCreateInfo->pool, - (*pAllocation), - pAllocationCreateInfo->pUserData ? (const char*)pAllocationCreateInfo->pUserData : ""); - fflush(g_File); - } + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordCreateBuffer( + allocator->GetCurrentFrameIndex(), + *pBufferCreateInfo, + *pAllocationCreateInfo, + *pAllocation); + } return VK_SUCCESS; } @@ -10378,24 +10714,22 @@ void vmaDestroyBuffer( return; } + VMA_DEBUG_LOG("vmaDestroyBuffer"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + if(allocator->GetRecorder() != VMA_NULL) { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", threadId, time, frameIndex, + allocator->GetRecorder()->RecordDestroyBuffer( + allocator->GetCurrentFrameIndex(), allocation); - fflush(g_File); } - VMA_DEBUG_LOG("vmaDestroyBuffer"); - VMA_DEBUG_GLOBAL_MUTEX_LOCK if(buffer != VK_NULL_HANDLE) { (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks()); } + if(allocation != VK_NULL_HANDLE) { allocator->FreeMemory(allocation); @@ -10462,37 +10796,14 @@ VkResult vmaCreateImage( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } - { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", threadId, time, frameIndex, - pImageCreateInfo->flags, - pImageCreateInfo->imageType, - pImageCreateInfo->format, - pImageCreateInfo->extent.width, - pImageCreateInfo->extent.height, - pImageCreateInfo->extent.depth, - pImageCreateInfo->mipLevels, - pImageCreateInfo->arrayLayers, - pImageCreateInfo->samples, - pImageCreateInfo->tiling, - pImageCreateInfo->usage, - pImageCreateInfo->sharingMode, - pImageCreateInfo->initialLayout, - pAllocationCreateInfo->flags, - pAllocationCreateInfo->usage, - pAllocationCreateInfo->requiredFlags, - pAllocationCreateInfo->preferredFlags, - pAllocationCreateInfo->memoryTypeBits, - pAllocationCreateInfo->pool, - (*pAllocation), - pAllocationCreateInfo->pUserData ? (const char*)pAllocationCreateInfo->pUserData : ""); - fflush(g_File); - } + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordCreateImage( + allocator->GetCurrentFrameIndex(), + *pImageCreateInfo, + *pAllocationCreateInfo, + *pAllocation); + } return VK_SUCCESS; } @@ -10521,21 +10832,17 @@ void vmaDestroyImage( return; } + VMA_DEBUG_LOG("vmaDestroyImage"); + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + if(allocator->GetRecorder() != VMA_NULL) { - VmaMutexLock lock(g_FileMutex, true); - EnsureFile(); - LARGE_INTEGER counter; QueryPerformanceCounter(&counter); - const DWORD threadId = GetCurrentThreadId(); - const double time = (double)(counter.QuadPart - g_StartCounter.QuadPart) / (double)g_Freq.QuadPart; - const uint32_t frameIndex = allocator->GetCurrentFrameIndex(); - fprintf(g_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", threadId, time, frameIndex, + allocator->GetRecorder()->RecordDestroyImage( + allocator->GetCurrentFrameIndex(), allocation); - fflush(g_File); } - VMA_DEBUG_LOG("vmaDestroyImage"); - VMA_DEBUG_GLOBAL_MUTEX_LOCK if(image != VK_NULL_HANDLE) { (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks()); From e5d9b01a3acf14d637be43379f662b36c1f7f3fb Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 15:50:03 +0200 Subject: [PATCH 27/51] Added macro VMA_RECORDING_ENABLED, enabled only on Windows for now. --- src/VmaUsage.h | 1 + src/vk_mem_alloc.h | 72 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/VmaUsage.h b/src/VmaUsage.h index 79daaf8..e788a30 100644 --- a/src/VmaUsage.h +++ b/src/VmaUsage.h @@ -25,6 +25,7 @@ include all public interface declarations. Example: //#define VMA_DEBUG_MARGIN 16 //#define VMA_DEBUG_DETECT_CORRUPTION 1 //#define VMA_DEBUG_INITIALIZE_ALLOCATIONS 1 +//#define VMA_RECORDING_ENABLED 0 #pragma warning(push, 4) #pragma warning(disable: 4127) // conditional expression is constant diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 3167490..7dfdd81 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -1323,6 +1323,18 @@ typedef enum VmaRecordFlagBits { } VmaRecordFlagBits; typedef VkFlags VmaRecordFlags; +/* +Define this macro to 0/1 to disable/enable support for recording functionality, +available through VmaAllocatorCreateInfo::pRecordSettings. +*/ +#ifndef VMA_RECORDING_ENABLED + #ifdef _WIN32 + #define VMA_RECORDING_ENABLED 1 + #else + #define VMA_RECORDING_ENABLED 0 + #endif +#endif + /// TODO typedef struct VmaRecordSettings { @@ -4714,6 +4726,8 @@ public: uint32_t maxAllocationsToMove); }; +#if VMA_RECORDING_ENABLED + class VmaRecorder { public: @@ -4789,6 +4803,8 @@ private: void Flush(); }; +#endif // #if VMA_RECORDING_ENABLED + // Main allocator object. struct VmaAllocator_T { @@ -4863,7 +4879,9 @@ public: return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; } +#if VMA_RECORDING_ENABLED VmaRecorder* GetRecorder() const { return m_pRecorder; } +#endif void GetBufferMemoryRequirements( VkBuffer hBuffer, @@ -4950,7 +4968,9 @@ private: VmaVulkanFunctions m_VulkanFunctions; +#if VMA_RECORDING_ENABLED VmaRecorder* m_pRecorder; +#endif void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions); @@ -8032,6 +8052,8 @@ bool VmaDefragmentator::MoveMakesSense( //////////////////////////////////////////////////////////////////////////////// // VmaRecorder +#if VMA_RECORDING_ENABLED + VmaRecorder::VmaRecorder() : m_UseMutex(true), m_Flags(0), @@ -8389,6 +8411,8 @@ void VmaRecorder::Flush() } } +#endif // #if VMA_RECORDING_ENABLED + //////////////////////////////////////////////////////////////////////////////// // VmaAllocator_T @@ -8403,8 +8427,10 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : m_PhysicalDevice(pCreateInfo->physicalDevice), m_CurrentFrameIndex(0), m_Pools(VmaStlAllocator(GetAllocationCallbacks())), - m_NextPoolId(0), - m_pRecorder(VMA_NULL) + m_NextPoolId(0) +#if VMA_RECORDING_ENABLED + ,m_pRecorder(VMA_NULL) +#endif { if(VMA_DEBUG_DETECT_CORRUPTION) { @@ -8490,17 +8516,17 @@ VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo) if(pCreateInfo->pRecordSettings != VMA_NULL && !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath)) { +#if VMA_RECORDING_ENABLED m_pRecorder = vma_new(this, VmaRecorder)(); res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex); if(res != VK_SUCCESS) { return res; } - } - - if(m_pRecorder != VMA_NULL) - { m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex()); +#else + return VK_ERROR_FEATURE_NOT_PRESENT; +#endif } return res; @@ -8508,11 +8534,13 @@ VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo) VmaAllocator_T::~VmaAllocator_T() { +#if VMA_RECORDING_ENABLED if(m_pRecorder != VMA_NULL) { m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex()); vma_delete(this, m_pRecorder); } +#endif VMA_ASSERT(m_Pools.empty()); @@ -10183,10 +10211,12 @@ VkResult vmaCreatePool( VkResult res = allocator->CreatePool(pCreateInfo, pPool); +#if VMA_RECORDING_ENABLED if(res == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool); } +#endif return res; } @@ -10206,10 +10236,12 @@ void vmaDestroyPool( VMA_DEBUG_GLOBAL_MUTEX_LOCK +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool); } +#endif allocator->DestroyPool(pool); } @@ -10272,6 +10304,7 @@ VkResult vmaAllocateMemory( VMA_SUBALLOCATION_TYPE_UNKNOWN, pAllocation); +#if VMA_RECORDING_ENABLED if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordAllocateMemory( @@ -10280,6 +10313,7 @@ VkResult vmaAllocateMemory( *pCreateInfo, *pAllocation); } +#endif if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS) { @@ -10319,6 +10353,7 @@ VkResult vmaAllocateMemoryForBuffer( VMA_SUBALLOCATION_TYPE_BUFFER, pAllocation); +#if VMA_RECORDING_ENABLED if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordAllocateMemoryForBuffer( @@ -10329,6 +10364,7 @@ VkResult vmaAllocateMemoryForBuffer( *pCreateInfo, *pAllocation); } +#endif if(pAllocationInfo && result == VK_SUCCESS) { @@ -10367,6 +10403,7 @@ VkResult vmaAllocateMemoryForImage( VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN, pAllocation); +#if VMA_RECORDING_ENABLED if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordAllocateMemoryForImage( @@ -10377,6 +10414,7 @@ VkResult vmaAllocateMemoryForImage( *pCreateInfo, *pAllocation); } +#endif if(pAllocationInfo && result == VK_SUCCESS) { @@ -10401,12 +10439,14 @@ void vmaFreeMemory( VMA_DEBUG_GLOBAL_MUTEX_LOCK +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordFreeMemory( allocator->GetCurrentFrameIndex(), allocation); } +#endif allocator->FreeMemory(allocation); } @@ -10445,6 +10485,7 @@ void vmaSetAllocationUserData( allocation->SetUserData(allocator, pUserData); +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordSetAllocationUserData( @@ -10452,6 +10493,7 @@ void vmaSetAllocationUserData( allocation, pUserData); } +#endif } void vmaCreateLostAllocation( @@ -10464,12 +10506,14 @@ void vmaCreateLostAllocation( allocator->CreateLostAllocation(pAllocation); +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordCreateLostAllocation( allocator->GetCurrentFrameIndex(), *pAllocation); } +#endif } VkResult vmaMapMemory( @@ -10483,12 +10527,14 @@ VkResult vmaMapMemory( VkResult res = allocator->Map(allocation, ppData); +#if VMA_RECORDING_ENABLED if(res == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordMapMemory( allocator->GetCurrentFrameIndex(), allocation); } +#endif return res; } @@ -10501,12 +10547,14 @@ void vmaUnmapMemory( VMA_DEBUG_GLOBAL_MUTEX_LOCK +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordUnmapMemory( allocator->GetCurrentFrameIndex(), allocation); } +#endif allocator->Unmap(allocation); } @@ -10521,12 +10569,14 @@ void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDevi allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH); +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordFlushAllocation( allocator->GetCurrentFrameIndex(), allocation, offset, size); } +#endif } void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) @@ -10539,12 +10589,14 @@ void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, V allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE); +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordInvalidateAllocation( allocator->GetCurrentFrameIndex(), allocation, offset, size); } +#endif } VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits) @@ -10678,6 +10730,7 @@ VkResult vmaCreateBuffer( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordCreateBuffer( @@ -10686,6 +10739,7 @@ VkResult vmaCreateBuffer( *pAllocationCreateInfo, *pAllocation); } +#endif return VK_SUCCESS; } @@ -10718,12 +10772,14 @@ void vmaDestroyBuffer( VMA_DEBUG_GLOBAL_MUTEX_LOCK +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordDestroyBuffer( allocator->GetCurrentFrameIndex(), allocation); } +#endif if(buffer != VK_NULL_HANDLE) { @@ -10796,6 +10852,7 @@ VkResult vmaCreateImage( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordCreateImage( @@ -10804,6 +10861,7 @@ VkResult vmaCreateImage( *pAllocationCreateInfo, *pAllocation); } +#endif return VK_SUCCESS; } @@ -10836,12 +10894,14 @@ void vmaDestroyImage( VMA_DEBUG_GLOBAL_MUTEX_LOCK +#if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordDestroyImage( allocator->GetCurrentFrameIndex(), allocation); } +#endif if(image != VK_NULL_HANDLE) { From 5c49bebe101a2d15961e171bd2d131739baaf768 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 16:10:11 +0200 Subject: [PATCH 28/51] Added support for recording and replaying object creation also when it originally failed. --- src/VmaReplay/VmaReplay.cpp | 180 +++++++++++++++++++++--------------- src/vk_mem_alloc.h | 61 ++++++------ 2 files changed, 139 insertions(+), 102 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index cf06fd4..d3a991f 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -501,7 +501,7 @@ private: // If failed, prints warning, returns false and outPool = null. bool FindPool(size_t lineNumber, uint64_t origPool, VmaPool& outPool); // If allocation with that origPtr already exists, prints warning and replaces it. - void AddAllocation(size_t lineNumber, uint64_t origPtr, Allocation&& allocDesc); + void AddAllocation(size_t lineNumber, uint64_t origPtr, VkResult res, const char* functionName, Allocation&& allocDesc); // Increments warning counter. Returns true if warning message should be printed. bool IssueWarning(); @@ -724,18 +724,58 @@ bool Player::FindPool(size_t lineNumber, uint64_t origPool, VmaPool& outPool) return true; } -void Player::AddAllocation(size_t lineNumber, uint64_t origPtr, Allocation&& allocDesc) +void Player::AddAllocation(size_t lineNumber, uint64_t origPtr, VkResult res, const char* functionName, Allocation&& allocDesc) { - const auto existingIt = m_Allocations.find(origPtr); - if(existingIt != m_Allocations.end()) + if(origPtr) { - if(IssueWarning()) + if(res == VK_SUCCESS) { - printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + // Originally succeeded, currently succeeded. + // Just save pointer (done below). + } + else + { + // Originally succeeded, currently failed. + // Print warning. Save null pointer. + if(IssueWarning()) + { + printf("Line %zu: %s failed (%d), while originally succeeded.\n", lineNumber, functionName, res); + } + } + + const auto existingIt = m_Allocations.find(origPtr); + if(existingIt != m_Allocations.end()) + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX already exists.\n", lineNumber, origPtr); + } + } + m_Allocations[origPtr] = std::move(allocDesc); + } + else + { + if(res == VK_SUCCESS) + { + // Originally failed, currently succeeded. + // Print warning, destroy the object. + if(IssueWarning()) + { + printf("Line %zu: %s succeeded, originally failed.\n", lineNumber, functionName); + } + + Destroy(allocDesc); + } + else + { + // Originally failed, currently failed. + // Print warning. + if(IssueWarning()) + { + printf("Line %zu: %s failed (%d), originally also failed.\n", lineNumber, functionName, res); + } } } - - m_Allocations[origPtr] = std::move(allocDesc); } bool Player::IssueWarning() @@ -1134,30 +1174,61 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) StrRangeToUint(csvSplit.GetRange(FIRST_PARAM_INDEX + 5), poolCreateInfo.frameInUseCount) && StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX + 6), origPtr)) { + m_Stats.RegisterCreatePool(); + Pool poolDesc = {}; VkResult res = vmaCreatePool(m_Allocator, &poolCreateInfo, &poolDesc.pool); - if(res == VK_SUCCESS) + + if(origPtr) { - m_Stats.RegisterCreatePool(); + if(res == VK_SUCCESS) + { + // Originally succeeded, currently succeeded. + // Just save pointer (done below). + } + else + { + // Originally succeeded, currently failed. + // Print warning. Save null pointer. + if(IssueWarning()) + { + printf("Line %zu: vmaCreatePool failed (%d), while originally succeeded.\n", lineNumber, res); + } + } + + const auto existingIt = m_Pools.find(origPtr); + if(existingIt != m_Pools.end()) + { + if(IssueWarning()) + { + printf("Line %zu: Pool %llX already exists.\n", lineNumber, origPtr); + } + } + m_Pools[origPtr] = poolDesc; } else { - if(IssueWarning()) + if(res == VK_SUCCESS) { - printf("Line %zu: vmaCreatePool failed (%d).\n", lineNumber, res); + // Originally failed, currently succeeded. + // Print warning, destroy the pool. + if(IssueWarning()) + { + printf("Line %zu: vmaCreatePool succeeded, originally failed.\n", lineNumber); + } + + vmaDestroyPool(m_Allocator, poolDesc.pool); + } + else + { + // Originally failed, currently failed. + // Print warning. + if(IssueWarning()) + { + printf("Line %zu: vmaCreatePool failed (%d), originally also failed.\n", lineNumber, res); + } } } - - const auto existingIt = m_Pools.find(origPtr); - if(existingIt != m_Pools.end()) - { - if(IssueWarning()) - { - printf("Line %zu: Pool %llX already exists.\n", lineNumber, origPtr); - } - } - - m_Pools[origPtr] = poolDesc; } else { @@ -1227,21 +1298,11 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) { FindPool(lineNumber, origPool, allocCreateInfo.pool); + m_Stats.RegisterCreateBuffer(bufCreateInfo.usage); + Allocation allocDesc = {}; VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); - if(res == VK_SUCCESS) - { - m_Stats.RegisterCreateBuffer(bufCreateInfo.usage); - } - else - { - if(IssueWarning()) - { - printf("Line %zu: vmaCreateBuffer failed (%d).\n", lineNumber, res); - } - } - - AddAllocation(lineNumber, origPtr, std::move(allocDesc)); + AddAllocation(lineNumber, origPtr, res, "vmaCreateBuffer", std::move(allocDesc)); } else { @@ -1320,21 +1381,11 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) { FindPool(lineNumber, origPool, allocCreateInfo.pool); + m_Stats.RegisterCreateImage(imageCreateInfo.usage, imageCreateInfo.tiling); + Allocation allocDesc = {}; VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &allocDesc.image, &allocDesc.allocation, nullptr); - if(res == VK_SUCCESS) - { - m_Stats.RegisterCreateImage(imageCreateInfo.usage, imageCreateInfo.tiling); - } - else - { - if(IssueWarning()) - { - printf("Line %zu: vmaCreateImage failed (%d).\n", lineNumber, res); - } - } - - AddAllocation(lineNumber, origPtr, std::move(allocDesc)); + AddAllocation(lineNumber, origPtr, res, "vmaCreateImage", std::move(allocDesc)); } else { @@ -1358,7 +1409,7 @@ void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvS vmaCreateLostAllocation(m_Allocator, &allocDesc.allocation); m_Stats.RegisterCreateAllocation(); - AddAllocation(lineNumber, origPtr, std::move(allocDesc)); + AddAllocation(lineNumber, origPtr, VK_SUCCESS, "vmaCreateLostAllocation", std::move(allocDesc)); } else { @@ -1392,21 +1443,11 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) { FindPool(lineNumber, origPool, allocCreateInfo.pool); + m_Stats.RegisterCreateAllocation(); + Allocation allocDesc = {}; VkResult res = vmaAllocateMemory(m_Allocator, &memReq, &allocCreateInfo, &allocDesc.allocation, nullptr); - if(res == VK_SUCCESS) - { - m_Stats.RegisterCreateAllocation(); - } - else - { - if(IssueWarning()) - { - printf("Line %zu: vmaAllocateMemory failed (%d).\n", lineNumber, res); - } - } - - AddAllocation(lineNumber, origPtr, std::move(allocDesc)); + AddAllocation(lineNumber, origPtr, res, "vmaAllocateMemory", std::move(allocDesc)); } else { @@ -1458,18 +1499,11 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS m_AllocateForBufferImageWarningIssued = true; } + m_Stats.RegisterCreateAllocation(); + Allocation allocDesc = {}; VkResult res = vmaAllocateMemory(m_Allocator, &memReq, &allocCreateInfo, &allocDesc.allocation, nullptr); - if(res == VK_SUCCESS) - { - m_Stats.RegisterCreateAllocation(); - } - else - { - printf("Line %zu: vmaAllocateMemory (called as vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage) failed (%d).\n", lineNumber, res); - } - - AddAllocation(lineNumber, origPtr, std::move(allocDesc)); + AddAllocation(lineNumber, origPtr, res, "vmaAllocateMemory (called as vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage)", std::move(allocDesc)); } else { diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 7dfdd81..591c5c7 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -8525,6 +8525,7 @@ VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo) } m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex()); #else + VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1."); return VK_ERROR_FEATURE_NOT_PRESENT; #endif } @@ -10212,7 +10213,7 @@ VkResult vmaCreatePool( VkResult res = allocator->CreatePool(pCreateInfo, pPool); #if VMA_RECORDING_ENABLED - if(res == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) + if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool); } @@ -10305,7 +10306,7 @@ VkResult vmaAllocateMemory( pAllocation); #if VMA_RECORDING_ENABLED - if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) + if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordAllocateMemory( allocator->GetCurrentFrameIndex(), @@ -10354,7 +10355,7 @@ VkResult vmaAllocateMemoryForBuffer( pAllocation); #if VMA_RECORDING_ENABLED - if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) + if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordAllocateMemoryForBuffer( allocator->GetCurrentFrameIndex(), @@ -10404,7 +10405,7 @@ VkResult vmaAllocateMemoryForImage( pAllocation); #if VMA_RECORDING_ENABLED - if(result == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) + if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordAllocateMemoryForImage( allocator->GetCurrentFrameIndex(), @@ -10528,7 +10529,7 @@ VkResult vmaMapMemory( VkResult res = allocator->Map(allocation, ppData); #if VMA_RECORDING_ENABLED - if(res == VK_SUCCESS && allocator->GetRecorder() != VMA_NULL) + if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordMapMemory( allocator->GetCurrentFrameIndex(), @@ -10687,8 +10688,8 @@ VkResult vmaCreateBuffer( allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq, requiresDedicatedAllocation, prefersDedicatedAllocation); - // Make sure alignment requirements for specific buffer usages reported - // in Physical Device Properties are included in alignment reported by memory requirements. + // Make sure alignment requirements for specific buffer usages reported + // in Physical Device Properties are included in alignment reported by memory requirements. if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0) { VMA_ASSERT(vkMemReq.alignment % @@ -10715,6 +10716,18 @@ VkResult vmaCreateBuffer( *pAllocationCreateInfo, VMA_SUBALLOCATION_TYPE_BUFFER, pAllocation); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordCreateBuffer( + allocator->GetCurrentFrameIndex(), + *pBufferCreateInfo, + *pAllocationCreateInfo, + *pAllocation); + } +#endif + if(res >= 0) { // 3. Bind buffer with memory. @@ -10730,17 +10743,6 @@ VkResult vmaCreateBuffer( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } -#if VMA_RECORDING_ENABLED - if(allocator->GetRecorder() != VMA_NULL) - { - allocator->GetRecorder()->RecordCreateBuffer( - allocator->GetCurrentFrameIndex(), - *pBufferCreateInfo, - *pAllocationCreateInfo, - *pAllocation); - } -#endif - return VK_SUCCESS; } allocator->FreeMemory(*pAllocation); @@ -10837,6 +10839,18 @@ VkResult vmaCreateImage( *pAllocationCreateInfo, suballocType, pAllocation); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordCreateImage( + allocator->GetCurrentFrameIndex(), + *pImageCreateInfo, + *pAllocationCreateInfo, + *pAllocation); + } +#endif + if(res >= 0) { // 3. Bind image with memory. @@ -10852,17 +10866,6 @@ VkResult vmaCreateImage( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } -#if VMA_RECORDING_ENABLED - if(allocator->GetRecorder() != VMA_NULL) - { - allocator->GetRecorder()->RecordCreateImage( - allocator->GetCurrentFrameIndex(), - *pImageCreateInfo, - *pAllocationCreateInfo, - *pAllocation); - } -#endif - return VK_SUCCESS; } allocator->FreeMemory(*pAllocation); From c49eb6282fafedbadb304167ab3b47cac733d4b5 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 16:36:07 +0200 Subject: [PATCH 29/51] Deleted calculation of number of vmaCreateAllocator calls, as there is always at most one. --- src/VmaReplay/VmaReplay.cpp | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index d3a991f..db0db95 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -244,12 +244,8 @@ public: size_t GetLinearImageCreationCount() const { return m_LinearImageCreationCount; } size_t GetBufferCreationCount(uint32_t bufClass) const { return m_BufferCreationCount[bufClass]; } size_t GetAllocationCreationCount() const { return m_AllocationCreationCount; } - size_t GetAllocatorCreationCount() const { return m_AllocatorCreationCount; } - size_t GetAllocatorPeakCount() const { return m_AllocatorPeakCount; } size_t GetPoolCreationCount() const { return m_PoolCreationCount; } - void RegisterCreateAllocator(); - void RegisterDestroyAllocator(); void RegisterCreateImage(uint32_t usage, uint32_t tiling); void RegisterCreateBuffer(uint32_t usage); void RegisterCreatePool(); @@ -260,9 +256,6 @@ private: size_t m_LinearImageCreationCount = 0; size_t m_BufferCreationCount[4] = { }; size_t m_AllocationCreationCount = 0; // Also includes buffers and images, and lost allocations. - size_t m_AllocatorCreationCount = 0; - size_t m_AllocatorCurrCount = 0; - size_t m_AllocatorPeakCount = 0; size_t m_PoolCreationCount = 0; }; @@ -329,19 +322,6 @@ uint32_t Statistics::ImageUsageToClass(uint32_t usage) } } -void Statistics::RegisterCreateAllocator() -{ - ++m_AllocatorCreationCount; - ++m_AllocatorCurrCount; - m_AllocatorPeakCount = std::max(m_AllocatorPeakCount, m_AllocatorCurrCount); -} - -void Statistics::RegisterDestroyAllocator() -{ - if(m_AllocatorCurrCount > 0) - --m_AllocatorCurrCount; -} - void Statistics::RegisterCreateImage(uint32_t usage, uint32_t tiling) { if(tiling == VK_IMAGE_TILING_LINEAR) @@ -543,14 +523,6 @@ int Player::Init() Player::~Player() { - if(m_Stats.GetAllocatorCreationCount() > 1) - { - printf("WARNING: %zu VmaAllocator objects were created. It is recommended to use just one.\n", - m_Stats.GetAllocatorCreationCount()); - printf(" At most %zu allocators existed simultaneously.\n", - m_Stats.GetAllocatorPeakCount()); - } - if(g_Verbosity > VERBOSITY::MINIMUM) { PrintStats(); @@ -621,14 +593,14 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 0, false)) { - m_Stats.RegisterCreateAllocator(); + // Nothing. } } else if(StrRangeEq(functionName, "vmaDestroyAllocator")) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 0, false)) { - m_Stats.RegisterDestroyAllocator(); + // Nothing. } } else if(StrRangeEq(functionName, "vmaCreatePool")) From dd715fbf9fdd003d50e3621729272fbabdd13b58 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 17:12:24 +0200 Subject: [PATCH 30/51] Fixed support for pUserData in object creation functions, as well as vmaSetAllocationUserData, whether it's null, custom pointer, or character string. --- src/VmaReplay/VmaReplay.cpp | 144 +++++++++++++++++++++++++++++++++--- src/vk_mem_alloc.h | 51 +++++++++++-- 2 files changed, 177 insertions(+), 18 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index db0db95..d3dc579 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -178,16 +178,18 @@ public: void Set(const StrRange& line, size_t maxCount = RANGE_COUNT_MAX); + const StrRange& GetLine() const { return m_Line; } + size_t GetCount() const { return m_Count; } StrRange GetRange(size_t index) const { return StrRange { - m_Str + m_Ranges[index * 2], - m_Str + m_Ranges[index * 2 + 1] }; + m_Line.beg + m_Ranges[index * 2], + m_Line.beg + m_Ranges[index * 2 + 1] }; } private: - const char* m_Str = nullptr; + StrRange m_Line = { nullptr, nullptr }; size_t m_Count = 0; size_t m_Ranges[RANGE_COUNT_MAX * 2]; // Pairs of begin-end. }; @@ -195,14 +197,14 @@ private: void CsvSplit::Set(const StrRange& line, size_t maxCount) { assert(maxCount <= RANGE_COUNT_MAX); - m_Str = line.beg; + m_Line = line; const size_t strLen = line.length(); size_t rangeIndex = 0; size_t charIndex = 0; while(charIndex < strLen && rangeIndex < maxCount) { m_Ranges[rangeIndex * 2] = charIndex; - while(charIndex < strLen && (rangeIndex + 1 == maxCount || m_Str[charIndex] != ',')) + while(charIndex < strLen && (rangeIndex + 1 == maxCount || m_Line.beg[charIndex] != ',')) ++charIndex; m_Ranges[rangeIndex * 2 + 1] = charIndex; ++rangeIndex; @@ -457,6 +459,7 @@ private: }; struct Allocation { + uint32_t allocationFlags; VmaAllocation allocation; VkBuffer buffer; VkImage image; @@ -474,6 +477,8 @@ private: std::string m_LastLineTimeStr; Statistics m_Stats; + std::vector m_UserDataTmpStr; + void Destroy(const Allocation& alloc); // Finds VmaPool bu original pointer. @@ -494,8 +499,12 @@ private: // If parmeter count doesn't match, issues warning and returns false. bool ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound); + // If failed, prints warning, returns false, and sets allocCreateInfo.pUserData to null. + bool PrepareUserData(size_t lineNumber, uint32_t allocCreateFlags, const StrRange& userDataColumn, const StrRange& wholeLine, void*& outUserData); + void ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteSetAllocationUserData(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteDestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } void ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit); @@ -608,12 +617,7 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) else if(StrRangeEq(functionName, "vmaDestroyPool")) ExecuteDestroyPool(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaSetAllocationUserData")) - { - if(ValidateFunctionParameterCount(lineNumber, csvSplit, 2, true)) - { - // Ignore for now. - } - } + ExecuteSetAllocationUserData(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaCreateBuffer")) ExecuteCreateBuffer(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaDestroyBuffer")) @@ -1131,6 +1135,37 @@ bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& c return ok; } +bool Player::PrepareUserData(size_t lineNumber, uint32_t allocCreateFlags, const StrRange& userDataColumn, const StrRange& wholeLine, void*& outUserData) +{ + // String + if((allocCreateFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0) + { + const size_t len = wholeLine.end - userDataColumn.beg; + m_UserDataTmpStr.resize(len + 1); + memcpy(m_UserDataTmpStr.data(), userDataColumn.beg, len); + m_UserDataTmpStr[len] = '\0'; + outUserData = m_UserDataTmpStr.data(); + return true; + } + // Pointer + else + { + uint64_t pUserData = 0; + if(StrRangeToPtr(userDataColumn, pUserData)) + { + outUserData = (void*)(uintptr_t)pUserData; + return true; + } + } + + if(IssueWarning()) + { + printf("Line %zu: Invalid pUserData.\n", lineNumber); + } + outUserData = 0; + return false; +} + void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 7, false)) @@ -1247,6 +1282,47 @@ void Player::ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit) } } +void Player::ExecuteSetAllocationUserData(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 2, true)) + { + uint64_t origPtr = 0; + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + const auto it = m_Allocations.find(origPtr); + if(it != m_Allocations.end()) + { + void* pUserData = nullptr; + if(csvSplit.GetCount() > FIRST_PARAM_INDEX + 1) + { + PrepareUserData( + lineNumber, + it->second.allocationFlags, + csvSplit.GetRange(FIRST_PARAM_INDEX + 1), + csvSplit.GetLine(), + pUserData); + } + + vmaSetAllocationUserData(m_Allocator, it->second.allocation, pUserData); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origPtr); + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaSetAllocationUserData.\n", lineNumber); + } + } + } +} + void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) { if(ValidateFunctionParameterCount(lineNumber, csvSplit, 12, true)) @@ -1270,9 +1346,20 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) { FindPool(lineNumber, origPool, allocCreateInfo.pool); + if(csvSplit.GetCount() > FIRST_PARAM_INDEX + 11) + { + PrepareUserData( + lineNumber, + allocCreateInfo.flags, + csvSplit.GetRange(FIRST_PARAM_INDEX + 11), + csvSplit.GetLine(), + allocCreateInfo.pUserData); + } + m_Stats.RegisterCreateBuffer(bufCreateInfo.usage); - Allocation allocDesc = {}; + Allocation allocDesc = { }; + allocDesc.allocationFlags = allocCreateInfo.flags; VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); AddAllocation(lineNumber, origPtr, res, "vmaCreateBuffer", std::move(allocDesc)); } @@ -1353,9 +1440,20 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) { FindPool(lineNumber, origPool, allocCreateInfo.pool); + if(csvSplit.GetCount() > FIRST_PARAM_INDEX + 20) + { + PrepareUserData( + lineNumber, + allocCreateInfo.flags, + csvSplit.GetRange(FIRST_PARAM_INDEX + 20), + csvSplit.GetLine(), + allocCreateInfo.pUserData); + } + m_Stats.RegisterCreateImage(imageCreateInfo.usage, imageCreateInfo.tiling); Allocation allocDesc = {}; + allocDesc.allocationFlags = allocCreateInfo.flags; VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &allocDesc.image, &allocDesc.allocation, nullptr); AddAllocation(lineNumber, origPtr, res, "vmaCreateImage", std::move(allocDesc)); } @@ -1415,9 +1513,20 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) { FindPool(lineNumber, origPool, allocCreateInfo.pool); + if(csvSplit.GetCount() > FIRST_PARAM_INDEX + 10) + { + PrepareUserData( + lineNumber, + allocCreateInfo.flags, + csvSplit.GetRange(FIRST_PARAM_INDEX + 10), + csvSplit.GetLine(), + allocCreateInfo.pUserData); + } + m_Stats.RegisterCreateAllocation(); Allocation allocDesc = {}; + allocDesc.allocationFlags = allocCreateInfo.flags; VkResult res = vmaAllocateMemory(m_Allocator, &memReq, &allocCreateInfo, &allocDesc.allocation, nullptr); AddAllocation(lineNumber, origPtr, res, "vmaAllocateMemory", std::move(allocDesc)); } @@ -1457,6 +1566,16 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS { FindPool(lineNumber, origPool, allocCreateInfo.pool); + if(csvSplit.GetCount() > FIRST_PARAM_INDEX + 12) + { + PrepareUserData( + lineNumber, + allocCreateInfo.flags, + csvSplit.GetRange(FIRST_PARAM_INDEX + 12), + csvSplit.GetLine(), + allocCreateInfo.pUserData); + } + if(requiresDedicatedAllocation || prefersDedicatedAllocation) { allocCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; @@ -1474,6 +1593,7 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS m_Stats.RegisterCreateAllocation(); Allocation allocDesc = {}; + allocDesc.allocationFlags = allocCreateInfo.flags; VkResult res = vmaAllocateMemory(m_Allocator, &memReq, &allocCreateInfo, &allocDesc.allocation, nullptr); AddAllocation(lineNumber, origPtr, res, "vmaAllocateMemory (called as vmaAllocateMemoryForBuffer or vmaAllocateMemoryForImage)", std::move(allocDesc)); } diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 591c5c7..d9a8029 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -4792,6 +4792,17 @@ private: double time; }; + class UserDataString + { + public: + UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData); + const char* GetString() const { return m_Str; } + + private: + char m_PtrStr[17]; + const char* m_Str; + }; + bool m_UseMutex; VmaRecordFlags m_Flags; FILE* m_File; @@ -8150,6 +8161,7 @@ void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex, GetBasicParams(callParams); VmaMutexLock lock(m_FileMutex, m_UseMutex); + UserDataString userDataStr(createInfo.flags, createInfo.pUserData); fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, vkMemReq.size, vkMemReq.alignment, @@ -8161,7 +8173,7 @@ void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex, createInfo.memoryTypeBits, createInfo.pool, allocation, - createInfo.pUserData ? (const char*)createInfo.pUserData : ""); + userDataStr.GetString()); Flush(); } @@ -8176,6 +8188,7 @@ void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex, GetBasicParams(callParams); VmaMutexLock lock(m_FileMutex, m_UseMutex); + UserDataString userDataStr(createInfo.flags, createInfo.pUserData); fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, vkMemReq.size, vkMemReq.alignment, @@ -8189,7 +8202,7 @@ void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex, createInfo.memoryTypeBits, createInfo.pool, allocation, - createInfo.pUserData ? (const char*)createInfo.pUserData : ""); + userDataStr.GetString()); Flush(); } @@ -8204,6 +8217,7 @@ void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex, GetBasicParams(callParams); VmaMutexLock lock(m_FileMutex, m_UseMutex); + UserDataString userDataStr(createInfo.flags, createInfo.pUserData); fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, vkMemReq.size, vkMemReq.alignment, @@ -8217,7 +8231,7 @@ void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex, createInfo.memoryTypeBits, createInfo.pool, allocation, - createInfo.pUserData ? (const char*)createInfo.pUserData : ""); + userDataStr.GetString()); Flush(); } @@ -8241,9 +8255,12 @@ void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex, GetBasicParams(callParams); VmaMutexLock lock(m_FileMutex, m_UseMutex); + UserDataString userDataStr( + allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0, + pUserData); fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex, allocation, - pUserData ? (const char*)pUserData : ""); + userDataStr.GetString()); Flush(); } @@ -8320,6 +8337,7 @@ void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex, GetBasicParams(callParams); VmaMutexLock lock(m_FileMutex, m_UseMutex); + UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData); fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, bufCreateInfo.flags, bufCreateInfo.size, @@ -8332,7 +8350,7 @@ void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex, allocCreateInfo.memoryTypeBits, allocCreateInfo.pool, allocation, - allocCreateInfo.pUserData ? (const char*)allocCreateInfo.pUserData : ""); + userDataStr.GetString()); Flush(); } @@ -8345,6 +8363,7 @@ void VmaRecorder::RecordCreateImage(uint32_t frameIndex, GetBasicParams(callParams); VmaMutexLock lock(m_FileMutex, m_UseMutex); + UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData); fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex, imageCreateInfo.flags, imageCreateInfo.imageType, @@ -8366,7 +8385,7 @@ void VmaRecorder::RecordCreateImage(uint32_t frameIndex, allocCreateInfo.memoryTypeBits, allocCreateInfo.pool, allocation, - allocCreateInfo.pUserData ? (const char*)allocCreateInfo.pUserData : ""); + userDataStr.GetString()); Flush(); } @@ -8394,6 +8413,26 @@ void VmaRecorder::RecordDestroyImage(uint32_t frameIndex, Flush(); } +VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData) +{ + if(pUserData != VMA_NULL) + { + if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0) + { + m_Str = (const char*)pUserData; + } + else + { + sprintf_s(m_PtrStr, "%p", pUserData); + m_Str = m_PtrStr; + } + } + else + { + m_Str = ""; + } +} + void VmaRecorder::GetBasicParams(CallParams& outParams) { outParams.threadId = GetCurrentThreadId(); From 3eb870d6d6c548b706a7b001dcee155d8b79114d Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 20 Aug 2018 17:53:02 +0200 Subject: [PATCH 31/51] Added documentation for new code elements related to recording. Added "Recording file format.md" file. --- docs/Recording file format.md | 176 ++++++++++++++ docs/html/annotated.html | 7 +- docs/html/classes.html | 8 +- docs/html/functions.html | 7 + docs/html/functions_vars.html | 7 + docs/html/general_considerations.html | 17 ++ docs/html/globals.html | 18 ++ docs/html/globals_defs.html | 3 + docs/html/globals_enum.html | 3 + docs/html/globals_eval.html | 6 + docs/html/globals_type.html | 9 + docs/html/memory_mapping.html | 7 +- docs/html/search/all_4.js | 2 +- docs/html/search/all_9.js | 2 + docs/html/search/all_f.js | 6 + docs/html/search/classes_0.js | 1 + docs/html/search/defines_0.js | 1 + docs/html/search/enums_0.js | 3 +- docs/html/search/enumvalues_0.js | 4 +- docs/html/search/typedefs_1.js | 3 + docs/html/search/variables_3.js | 2 +- docs/html/search/variables_6.js | 2 + ...uct_vma_allocator_create_info-members.html | 5 +- .../struct_vma_allocator_create_info.html | 20 ++ .../struct_vma_record_settings-members.html | 78 ++++++ docs/html/struct_vma_record_settings.html | 130 ++++++++++ docs/html/vk__mem__alloc_8h.html | 100 ++++++++ docs/html/vk__mem__alloc_8h_source.html | 228 +++++++++--------- src/vk_mem_alloc.h | 23 +- 29 files changed, 752 insertions(+), 126 deletions(-) create mode 100644 docs/Recording file format.md create mode 100644 docs/html/struct_vma_record_settings-members.html create mode 100644 docs/html/struct_vma_record_settings.html diff --git a/docs/Recording file format.md b/docs/Recording file format.md new file mode 100644 index 0000000..882a294 --- /dev/null +++ b/docs/Recording file format.md @@ -0,0 +1,176 @@ +This is an official documentation for file format used by Vulkan Memory Allocator library +to record and replay sequence of calls to its functions. +This feature can be enabled by using `VmaAllocatorCreateInfo::pRecordSettings` structure members. +For details, see main documentation of the library. +Playback can be launched using **VmaReplay** console application. + +Recording is a text file. +Line endings: Unix `'\n'`. +Character encoding: single-byte (can be ASCII or UTF-8, whaterver you use in custom strings). +Suggested file extension: **csv**. File can be processed sequentially - no random access is needed. +Each line forms a separate entry. +Each line consists of a set of values (also called columns), separated by comma `','` (hence "CSV" format - Comma Separated Values). +Number of columns is different in different lines. + +# Header + +First line identifies file format. It must always be: + + Vulkan Memory Allocator,Calls recording + +Second line identifies format version, where first column is major version and second column is minor version. +Formats with only minor version incremented are backward compatible. +VmaReplay application supports all older versions. +Current version is: + + 1,2 + +# Function calls + +Remaining lines contain recorded calls to VMA functions. First columns are always: + +- Thread ID : uint32 +- Time since first call : float, in seconds +- VMA frame index : uint32 +- Function name : string + +Remaining columns are function parameters and output, depending on function name, which can be: + +**vmaCreateAllocator, vmaDestroyAllocator** + +No parameters. + +**vmaCreatePool** + +- memoryTypeIndex : uint32 +- flags : uint32 +- blockSize : uint64 +- minBlockCount : uint64 +- maxBlockCount : uint64 +- frameInUseCount : uint32 +- pool (output) : pointer + +**vmaDestroyPool** + +- pool : pointer + +**vmaSetAllocationUserData** + +- allocation : pointer +- pUserData : string (may contain additional commas) + +**vmaCreateBuffer** + +- bufferCreateInfo.flags : uint32 +- bufferCreateInfo.size : uint64 +- bufferCreateInfo.usage : uint32 +- bufferCreateInfo.sharingMode : uint32 +- allocationCreateInfo.flags : uint32 +- allocationCreateInfo.usage : uint32 +- allocationCreateInfo.requiredFlags : uint32 +- allocationCreateInfo.preferredFlags : uint32 +- allocationCreateInfo.memoryTypeBits : uint32 +- allocationCreateInfo.pool : pointer +- allocation (output) : pointer +- allocationCreateInfo.pUserData : string (may contain additional commas) + +**vmaDestroyBuffer** + +- allocation : pointer + +**vmaCreateImage** + +- imageCreateInfo.flags : uint32 +- imageCreateInfo.imageType : uint32 +- imageCreateInfo.format : uint32 +- imageCreateInfo.extent.width : uint32 +- imageCreateInfo.extent.height : uint32 +- imageCreateInfo.extent.depth : uint32 +- imageCreateInfo.mipLevels : uint32 +- imageCreateInfo.arrayLayers : uint32 +- imageCreateInfo.samples : uint32 +- imageCreateInfo.tiling : uint32 +- imageCreateInfo.usage : uint32 +- imageCreateInfo.sharingMode : uint32 +- imageCreateInfo.initialLayout : uint32 +- allocationCreateInfo.flags : uint32 +- allocationCreateInfo.usage : uint32 +- allocationCreateInfo.requiredFlags : uint32 +- allocationCreateInfo.preferredFlags : uint32 +- allocationCreateInfo.memoryTypeBits : uint32 +- allocationCreateInfo.pool : pointer +- allocation (output) : pointer +- allocationCreateInfo.pUserData : string (may contain additional commas) + +**vmaDestroyImage** + +- allocation : pointer + +**vmaFreeMemory** (min format version 1.1) + +- allocation : pointer + +**vmaCreateLostAllocation** (min format version 1.2) + +- allocation (output) : pointer + +**vmaAllocateMemory** (min format version 1.2) + +- vkMemoryRequirements.size : uint64 +- vkMemoryRequirements.alignment : uint64 +- vkMemoryRequirements.memoryTypeBits : uint32 +- allocationCreateInfo.flags : uint32 +- allocationCreateInfo.usage : uint32 +- allocationCreateInfo.requiredFlags : uint32 +- allocationCreateInfo.preferredFlags : uint32 +- allocationCreateInfo.memoryTypeBits : uint32 +- allocationCreateInfo.pool : pointer +- allocation (output) : pointer +- allocationCreateInfo.pUserData : string (may contain additional commas) + +**vmaAllocateMemoryForBuffer, vmaAllocateMemoryForImage** (min format version 1.2) + +- vkMemoryRequirements.size : uint64 +- vkMemoryRequirements.alignment : uint64 +- vkMemoryRequirements.memoryTypeBits : uint32 +- requiresDedicatedAllocation : bool +- prefersDedicatedAllocation : bool +- allocationCreateInfo.flags : uint32 +- allocationCreateInfo.usage : uint32 +- allocationCreateInfo.requiredFlags : uint32 +- allocationCreateInfo.preferredFlags : uint32 +- allocationCreateInfo.memoryTypeBits : uint32 +- allocationCreateInfo.pool : pointer +- allocation (output) : pointer +- allocationCreateInfo.pUserData : string (may contain additional commas) + +**vmaMapMemory, vmaUnmapMemory** (min format version 1.2) + +- allocation : pointer + +**vmaFlushAllocation, vmaInvalidateAllocation** (min format version 1.2) + +- allocation : pointer +- offset : uint64 +- size : uint64 + +## Data types + +**bool** + +Encoded as `0` for false or `1` for true. + +**uint32, uint64** + +Encoded in decimal format. + +**pointer** + +Encoded in hexadecimal format. + +**pUserData** + +If `pUserData` was a pointer, it is encoded as hexadecimal string. +If `VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT` was used with the allocation, the string is written as-is. +It may contain additional commas. +It should not contain end-of-line characters - results are then undefined. diff --git a/docs/html/annotated.html b/docs/html/annotated.html index 18ddbd5..37ef74d 100644 --- a/docs/html/annotated.html +++ b/docs/html/annotated.html @@ -75,9 +75,10 @@ $(function() {  CVmaPoolRepresents custom memory pool  CVmaPoolCreateInfoDescribes parameter of created VmaPool  CVmaPoolStatsDescribes parameter of existing VmaPool - CVmaStatInfoCalculated statistics of memory usage in entire allocator - CVmaStatsGeneral statistics from current state of Allocator - CVmaVulkanFunctionsPointers to some Vulkan functions - a subset used by the library + CVmaRecordSettingsParameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings + CVmaStatInfoCalculated statistics of memory usage in entire allocator + CVmaStatsGeneral statistics from current state of Allocator + CVmaVulkanFunctionsPointers to some Vulkan functions - a subset used by the library diff --git a/docs/html/classes.html b/docs/html/classes.html index b900f2a..099f7e3 100644 --- a/docs/html/classes.html +++ b/docs/html/classes.html @@ -65,10 +65,10 @@ $(function() {
v
- - - + + + +
  v  
-		VmaAllocationInfo   VmaDefragmentationStats   VmaPoolStats   
VmaAllocator   VmaDeviceMemoryCallbacks   VmaStatInfo   
VmaAllocation   VmaAllocatorCreateInfo   VmaPool   VmaStats   
VmaAllocationCreateInfo   VmaDefragmentationInfo   VmaPoolCreateInfo   VmaVulkanFunctions   
VmaAllocationInfo   VmaDefragmentationStats   VmaPoolStats   VmaVulkanFunctions   
VmaAllocator   VmaDeviceMemoryCallbacks   VmaRecordSettings   
VmaAllocation   VmaAllocatorCreateInfo   VmaPool   VmaStatInfo   
VmaAllocationCreateInfo   VmaDefragmentationInfo   VmaPoolCreateInfo   VmaStats   
v
diff --git a/docs/html/functions.html b/docs/html/functions.html index 0751d69..c8ce62d 100644 --- a/docs/html/functions.html +++ b/docs/html/functions.html @@ -114,6 +114,7 @@ $(function() { : VmaAllocationCreateInfo , VmaAllocatorCreateInfo , VmaPoolCreateInfo +, VmaRecordSettings
  • frameInUseCount : VmaAllocatorCreateInfo @@ -165,6 +166,9 @@ $(function() {
  • pDeviceMemoryCallbacks : VmaAllocatorCreateInfo
    • +
  • pFilePath +: VmaRecordSettings +
  • pfnAllocate : VmaDeviceMemoryCallbacks
    • @@ -183,6 +187,9 @@ $(function() {
  • pool : VmaAllocationCreateInfo
    • +
  • pRecordSettings +: VmaAllocatorCreateInfo +
  • preferredFlags : VmaAllocationCreateInfo
    • diff --git a/docs/html/functions_vars.html b/docs/html/functions_vars.html index 6ee6243..de9e9a0 100644 --- a/docs/html/functions_vars.html +++ b/docs/html/functions_vars.html @@ -114,6 +114,7 @@ $(function() { : VmaAllocationCreateInfo , VmaAllocatorCreateInfo , VmaPoolCreateInfo +, VmaRecordSettings
  • frameInUseCount : VmaAllocatorCreateInfo @@ -165,6 +166,9 @@ $(function() {
  • pDeviceMemoryCallbacks : VmaAllocatorCreateInfo
    • +
  • pFilePath +: VmaRecordSettings +
  • pfnAllocate : VmaDeviceMemoryCallbacks
    • @@ -183,6 +187,9 @@ $(function() {
  • pool : VmaAllocationCreateInfo
    • +
  • pRecordSettings +: VmaAllocatorCreateInfo +
  • preferredFlags : VmaAllocationCreateInfo
    • diff --git a/docs/html/general_considerations.html b/docs/html/general_considerations.html index 5d4362d..300ab1b 100644 --- a/docs/html/general_considerations.html +++ b/docs/html/general_considerations.html @@ -74,6 +74,23 @@ Thread safety
  • When the allocator is created with VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT flag, calls to functions that take such VmaAllocator object must be synchronized externally.
  • Access to a VmaAllocation object must be externally synchronized. For example, you must not call vmaGetAllocationInfo() and vmaMapMemory() from different threads at the same time if you pass the same VmaAllocation object to these functions.
    • +

    +Validation layer warnings

    +

    When using this library, you can meet following types of warnings issued by Vulkan validation layer. They don't necessarily indicate a bug, so you may need to just ignore them.

    +
      +
    • vkBindBufferMemory(): Binding memory to buffer 0xeb8e4 but vkGetBufferMemoryRequirements() has not been called on that buffer.
        +
      • It happens when VK_KHR_dedicated_allocation extension is enabled. vkGetBufferMemoryRequirements2KHR function is used instead, while validation layer seems to be unaware of it.
        • +
      +
      • +
    • Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used.
        +
      • It happens when you map a buffer or image, because the library maps entire VkDeviceMemory block, where different types of images and buffers may end up together, especially on GPUs with unified memory like Intel.
        • +
      +
      • +
    • Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug.
        +
      • It happens when you use lost allocations, and a new image or buffer is created in place of an existing object that bacame lost.
        • +
      +
      • +

    Allocation algorithm

    The library uses following algorithm for allocation, in order:

    diff --git a/docs/html/globals.html b/docs/html/globals.html index 08a3076..2f1f2fb 100644 --- a/docs/html/globals.html +++ b/docs/html/globals.html @@ -128,6 +128,15 @@ $(function() {
  • VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT : vk_mem_alloc.h
    • +
  • VMA_RECORD_FLAG_BITS_MAX_ENUM +: vk_mem_alloc.h +
    • +
  • VMA_RECORD_FLUSH_AFTER_CALL_BIT +: vk_mem_alloc.h +
    • +
  • VMA_RECORDING_ENABLED +: vk_mem_alloc.h +
  • VMA_STATS_STRING_ENABLED : vk_mem_alloc.h
    • @@ -275,6 +284,15 @@ $(function() {
  • VmaPoolStats : vk_mem_alloc.h
    • +
  • VmaRecordFlagBits +: vk_mem_alloc.h +
    • +
  • VmaRecordFlags +: vk_mem_alloc.h +
    • +
  • VmaRecordSettings +: vk_mem_alloc.h +
  • vmaSetAllocationUserData() : vk_mem_alloc.h
    • diff --git a/docs/html/globals_defs.html b/docs/html/globals_defs.html index aa8115b..ca5de3f 100644 --- a/docs/html/globals_defs.html +++ b/docs/html/globals_defs.html @@ -62,6 +62,9 @@ $(function() {
  • VMA_DEDICATED_ALLOCATION : vk_mem_alloc.h
    • +
  • VMA_RECORDING_ENABLED +: vk_mem_alloc.h +
  • VMA_STATS_STRING_ENABLED : vk_mem_alloc.h
    • diff --git a/docs/html/globals_enum.html b/docs/html/globals_enum.html index 1df06f1..d295ccd 100644 --- a/docs/html/globals_enum.html +++ b/docs/html/globals_enum.html @@ -71,6 +71,9 @@ $(function() {
  • VmaPoolCreateFlagBits : vk_mem_alloc.h
    • +
  • VmaRecordFlagBits +: vk_mem_alloc.h +
    • diff --git a/docs/html/globals_eval.html b/docs/html/globals_eval.html index 2035414..091f76b 100644 --- a/docs/html/globals_eval.html +++ b/docs/html/globals_eval.html @@ -113,6 +113,12 @@ $(function() {
  • VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT : vk_mem_alloc.h
    • +
  • VMA_RECORD_FLAG_BITS_MAX_ENUM +: vk_mem_alloc.h +
    • +
  • VMA_RECORD_FLUSH_AFTER_CALL_BIT +: vk_mem_alloc.h +
    • diff --git a/docs/html/globals_type.html b/docs/html/globals_type.html index 0b3156b..68106a1 100644 --- a/docs/html/globals_type.html +++ b/docs/html/globals_type.html @@ -110,6 +110,15 @@ $(function() {
  • VmaPoolStats : vk_mem_alloc.h
    • +
  • VmaRecordFlagBits +: vk_mem_alloc.h +
    • +
  • VmaRecordFlags +: vk_mem_alloc.h +
    • +
  • VmaRecordSettings +: vk_mem_alloc.h +
  • VmaStatInfo : vk_mem_alloc.h
    • diff --git a/docs/html/memory_mapping.html b/docs/html/memory_mapping.html index fa97ad7..7e2c263 100644 --- a/docs/html/memory_mapping.html +++ b/docs/html/memory_mapping.html @@ -69,8 +69,11 @@ $(function() {

    To "map memory" in Vulkan means to obtain a CPU pointer to VkDeviceMemory, to be able to read from it or write to it in CPU code. Mapping is possible only of memory allocated from a memory type that has VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT flag. Functions vkMapMemory(), vkUnmapMemory() are designed for this purpose. You can use them directly with memory allocated by this library, but it is not recommended because of following issue: Mapping the same VkDeviceMemory block multiple times is illegal - only one mapping at a time is allowed. This includes mapping disjoint regions. Mapping is not reference-counted internally by Vulkan. Because of this, Vulkan Memory Allocator provides following facilities:

    Mapping functions

    -

    The library provides following functions for mapping of a specific VmaAllocation: vmaMapMemory(), vmaUnmapMemory(). They are safer and more convenient to use than standard Vulkan functions. You can map an allocation multiple times simultaneously - mapping is reference-counted internally. You can also map different allocations simultaneously regardless of whether they use the same VkDeviceMemory block. They way it's implemented is that the library always maps entire memory block, not just region of the allocation. For further details, see description of vmaMapMemory() function. Example:

    -
    // Having these objects initialized:
    struct ConstantBuffer
    {
    ...
    };
    ConstantBuffer constantBufferData;
    VmaAllocator allocator;
    VmaBuffer constantBuffer;
    VmaAllocation constantBufferAllocation;
    // You can map and fill your buffer using following code:
    void* mappedData;
    vmaMapMemory(allocator, constantBufferAllocation, &mappedData);
    memcpy(mappedData, &constantBufferData, sizeof(constantBufferData));
    vmaUnmapMemory(allocator, constantBufferAllocation);

    +

    The library provides following functions for mapping of a specific VmaAllocation: vmaMapMemory(), vmaUnmapMemory(). They are safer and more convenient to use than standard Vulkan functions. You can map an allocation multiple times simultaneously - mapping is reference-counted internally. You can also map different allocations simultaneously regardless of whether they use the same VkDeviceMemory block. The way it's implemented is that the library always maps entire memory block, not just region of the allocation. For further details, see description of vmaMapMemory() function. Example:

    +
    // Having these objects initialized:
    struct ConstantBuffer
    {
    ...
    };
    ConstantBuffer constantBufferData;
    VmaAllocator allocator;
    VkBuffer constantBuffer;
    VmaAllocation constantBufferAllocation;
    // You can map and fill your buffer using following code:
    void* mappedData;
    vmaMapMemory(allocator, constantBufferAllocation, &mappedData);
    memcpy(mappedData, &constantBufferData, sizeof(constantBufferData));
    vmaUnmapMemory(allocator, constantBufferAllocation);

    When mapping, you may see a warning from Vulkan validation layer similar to this one:

    +

    Mapping an image with layout VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL can result in undefined behavior if this memory is used by the device. Only GENERAL or PREINITIALIZED should be used.

    +

    It happens because the library maps entire VkDeviceMemory block, where different types of images and buffers may end up together, especially on GPUs with unified memory like Intel. You can safely ignore it if you are sure you access only memory of the intended object that you wanted to map.

    +

    Persistently mapped memory

    Kepping your memory persistently mapped is generally OK in Vulkan. You don't need to unmap it before using its data on the GPU. The library provides a special feature designed for that: Allocations made with VMA_ALLOCATION_CREATE_MAPPED_BIT flag set in VmaAllocationCreateInfo::flags stay mapped all the time, so you can just access CPU pointer to it any time without a need to call any "map" or "unmap" function. Example:

    VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
    bufCreateInfo.size = sizeof(ConstantBuffer);
    bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
    VmaAllocationCreateInfo allocCreateInfo = {};
    allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY;
    allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
    VkBuffer buf;
    VmaAllocation alloc;
    VmaAllocationInfo allocInfo;
    vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
    // Buffer is already mapped. You can access its memory.
    memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));

    There are some exceptions though, when you should consider mapping memory only for a short period of time:

    diff --git a/docs/html/search/all_4.js b/docs/html/search/all_4.js index 7bacfcc..347c9f3 100644 --- a/docs/html/search/all_4.js +++ b/docs/html/search/all_4.js @@ -1,5 +1,5 @@ var searchData= [ - ['flags',['flags',['../struct_vma_allocator_create_info.html#a392ea2ecbaff93f91a7c49f735ad4346',1,'VmaAllocatorCreateInfo::flags()'],['../struct_vma_allocation_create_info.html#add09658ac14fe290ace25470ddd6d41b',1,'VmaAllocationCreateInfo::flags()'],['../struct_vma_pool_create_info.html#a8405139f63d078340ae74513a59f5446',1,'VmaPoolCreateInfo::flags()']]], + ['flags',['flags',['../struct_vma_record_settings.html#ad8fdcc92119ae7a8c08c1a564c01d63a',1,'VmaRecordSettings::flags()'],['../struct_vma_allocator_create_info.html#a392ea2ecbaff93f91a7c49f735ad4346',1,'VmaAllocatorCreateInfo::flags()'],['../struct_vma_allocation_create_info.html#add09658ac14fe290ace25470ddd6d41b',1,'VmaAllocationCreateInfo::flags()'],['../struct_vma_pool_create_info.html#a8405139f63d078340ae74513a59f5446',1,'VmaPoolCreateInfo::flags()']]], ['frameinusecount',['frameInUseCount',['../struct_vma_allocator_create_info.html#a21ea188dd212b8171cb9ecbed4a2a3a7',1,'VmaAllocatorCreateInfo::frameInUseCount()'],['../struct_vma_pool_create_info.html#a9437e43ffbb644dbbf7fc4e50cfad6aa',1,'VmaPoolCreateInfo::frameInUseCount()']]] ]; diff --git a/docs/html/search/all_9.js b/docs/html/search/all_9.js index 590465a..33e01da 100644 --- a/docs/html/search/all_9.js +++ b/docs/html/search/all_9.js @@ -2,6 +2,7 @@ var searchData= [ ['pallocationcallbacks',['pAllocationCallbacks',['../struct_vma_allocator_create_info.html#a6e409087e3be55400d0e4ccbe43c608d',1,'VmaAllocatorCreateInfo']]], ['pdevicememorycallbacks',['pDeviceMemoryCallbacks',['../struct_vma_allocator_create_info.html#af1380969b5e1ea4c3184a877892d260e',1,'VmaAllocatorCreateInfo']]], + ['pfilepath',['pFilePath',['../struct_vma_record_settings.html#a6cb1fdbf6bcb610b68f2010dd629e89d',1,'VmaRecordSettings']]], ['pfn_5fvmaallocatedevicememoryfunction',['PFN_vmaAllocateDeviceMemoryFunction',['../vk__mem__alloc_8h.html#ab6a6477cda1ce775b30bde96d766203b',1,'vk_mem_alloc.h']]], ['pfn_5fvmafreedevicememoryfunction',['PFN_vmaFreeDeviceMemoryFunction',['../vk__mem__alloc_8h.html#aef2545dc2e9dd4f29ab9ba6ac6fe2f49',1,'vk_mem_alloc.h']]], ['pfnallocate',['pfnAllocate',['../struct_vma_device_memory_callbacks.html#a4f17f7b255101e733b44d5633aceabfb',1,'VmaDeviceMemoryCallbacks']]], @@ -10,6 +11,7 @@ var searchData= ['physicaldevice',['physicalDevice',['../struct_vma_allocator_create_info.html#a08230f04ae6ccf8a78150a9e829a7156',1,'VmaAllocatorCreateInfo']]], ['pmappeddata',['pMappedData',['../struct_vma_allocation_info.html#a5eeffbe2d2f30f53370ff14aefbadbe2',1,'VmaAllocationInfo']]], ['pool',['pool',['../struct_vma_allocation_create_info.html#a6272c0555cfd1fe28bff1afeb6190150',1,'VmaAllocationCreateInfo']]], + ['precordsettings',['pRecordSettings',['../struct_vma_allocator_create_info.html#ace2aa4877b16a42b0b7673d4e26000ee',1,'VmaAllocatorCreateInfo']]], ['preferredflags',['preferredFlags',['../struct_vma_allocation_create_info.html#a7fe8d81a1ad10b2a2faacacee5b15d6d',1,'VmaAllocationCreateInfo']]], ['preferredlargeheapblocksize',['preferredLargeHeapBlockSize',['../struct_vma_allocator_create_info.html#a8e4714298e3121cdd8b214a1ae7a637a',1,'VmaAllocatorCreateInfo']]], ['puserdata',['pUserData',['../struct_vma_allocation_create_info.html#a8259e85c272683434f4abb4ddddffe19',1,'VmaAllocationCreateInfo::pUserData()'],['../struct_vma_allocation_info.html#adc507656149c04de7ed95d0042ba2a13',1,'VmaAllocationInfo::pUserData()']]], diff --git a/docs/html/search/all_f.js b/docs/html/search/all_f.js index a80478e..60dbc10 100644 --- a/docs/html/search/all_f.js +++ b/docs/html/search/all_f.js @@ -38,6 +38,9 @@ var searchData= ['vma_5fmemory_5fusage_5funknown',['VMA_MEMORY_USAGE_UNKNOWN',['../vk__mem__alloc_8h.html#aa5846affa1e9da3800e3e78fae2305ccaf50d27e34e0925cf3a63db8c839121dd',1,'vk_mem_alloc.h']]], ['vma_5fpool_5fcreate_5fflag_5fbits_5fmax_5fenum',['VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM',['../vk__mem__alloc_8h.html#a9a7c45f9c863695d98c83fa5ac940fe7a1c7312bea9ea246846b9054fd6bd6aec',1,'vk_mem_alloc.h']]], ['vma_5fpool_5fcreate_5fignore_5fbuffer_5fimage_5fgranularity_5fbit',['VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT',['../vk__mem__alloc_8h.html#a9a7c45f9c863695d98c83fa5ac940fe7a9f1a499508a8edb4e8ba40aa0290a3d2',1,'vk_mem_alloc.h']]], + ['vma_5frecord_5fflag_5fbits_5fmax_5fenum',['VMA_RECORD_FLAG_BITS_MAX_ENUM',['../vk__mem__alloc_8h.html#a4dd2c44642312a147a4e93373a6e64d2a20dd17d69966dbffa054739d6090b85e',1,'vk_mem_alloc.h']]], + ['vma_5frecord_5fflush_5fafter_5fcall_5fbit',['VMA_RECORD_FLUSH_AFTER_CALL_BIT',['../vk__mem__alloc_8h.html#a4dd2c44642312a147a4e93373a6e64d2a8e7ab322e8732654be627c4ea8f36cc7',1,'vk_mem_alloc.h']]], + ['vma_5frecording_5fenabled',['VMA_RECORDING_ENABLED',['../vk__mem__alloc_8h.html#a1f0c126759fc96ccb6e2d23c101d770c',1,'vk_mem_alloc.h']]], ['vma_5fstats_5fstring_5fenabled',['VMA_STATS_STRING_ENABLED',['../vk__mem__alloc_8h.html#ae25f0d55fd91cb166f002b63244800e1',1,'vk_mem_alloc.h']]], ['vmaallocatememory',['vmaAllocateMemory',['../vk__mem__alloc_8h.html#abf28077dbf82d0908b8acbe8ee8dd9b8',1,'vk_mem_alloc.h']]], ['vmaallocatememoryforbuffer',['vmaAllocateMemoryForBuffer',['../vk__mem__alloc_8h.html#a7fdf64415b6c3d83c454f28d2c53df7b',1,'vk_mem_alloc.h']]], @@ -90,6 +93,9 @@ var searchData= ['vmapoolcreateflags',['VmaPoolCreateFlags',['../vk__mem__alloc_8h.html#a2770e325ea42e087c1b91fdf46d0292a',1,'vk_mem_alloc.h']]], ['vmapoolcreateinfo',['VmaPoolCreateInfo',['../struct_vma_pool_create_info.html',1,'VmaPoolCreateInfo'],['../vk__mem__alloc_8h.html#a211706e9348dcee25a843ed4ea69bce7',1,'VmaPoolCreateInfo(): vk_mem_alloc.h']]], ['vmapoolstats',['VmaPoolStats',['../struct_vma_pool_stats.html',1,'VmaPoolStats'],['../vk__mem__alloc_8h.html#a2e5612d871d64c5624087b837a338c34',1,'VmaPoolStats(): vk_mem_alloc.h']]], + ['vmarecordflagbits',['VmaRecordFlagBits',['../vk__mem__alloc_8h.html#a4dd2c44642312a147a4e93373a6e64d2',1,'VmaRecordFlagBits(): vk_mem_alloc.h'],['../vk__mem__alloc_8h.html#ade20b626a6635ce1bf30ea53dea774e4',1,'VmaRecordFlagBits(): vk_mem_alloc.h']]], + ['vmarecordflags',['VmaRecordFlags',['../vk__mem__alloc_8h.html#af3929a1a4547c592fc0b0e55ef452828',1,'vk_mem_alloc.h']]], + ['vmarecordsettings',['VmaRecordSettings',['../struct_vma_record_settings.html',1,'VmaRecordSettings'],['../vk__mem__alloc_8h.html#a0ab61e87ff6365f1d59915eadc37a9f0',1,'VmaRecordSettings(): vk_mem_alloc.h']]], ['vmasetallocationuserdata',['vmaSetAllocationUserData',['../vk__mem__alloc_8h.html#af9147d31ffc11d62fc187bde283ed14f',1,'vk_mem_alloc.h']]], ['vmasetcurrentframeindex',['vmaSetCurrentFrameIndex',['../vk__mem__alloc_8h.html#ade56bf8dc9f5a5eaddf5f119ed525236',1,'vk_mem_alloc.h']]], ['vmastatinfo',['VmaStatInfo',['../struct_vma_stat_info.html',1,'VmaStatInfo'],['../vk__mem__alloc_8h.html#a810b009a788ee8aac72a25b42ffbe31c',1,'VmaStatInfo(): vk_mem_alloc.h']]], diff --git a/docs/html/search/classes_0.js b/docs/html/search/classes_0.js index 152fb2b..7fe2da1 100644 --- a/docs/html/search/classes_0.js +++ b/docs/html/search/classes_0.js @@ -11,6 +11,7 @@ var searchData= ['vmapool',['VmaPool',['../struct_vma_pool.html',1,'']]], ['vmapoolcreateinfo',['VmaPoolCreateInfo',['../struct_vma_pool_create_info.html',1,'']]], ['vmapoolstats',['VmaPoolStats',['../struct_vma_pool_stats.html',1,'']]], + ['vmarecordsettings',['VmaRecordSettings',['../struct_vma_record_settings.html',1,'']]], ['vmastatinfo',['VmaStatInfo',['../struct_vma_stat_info.html',1,'']]], ['vmastats',['VmaStats',['../struct_vma_stats.html',1,'']]], ['vmavulkanfunctions',['VmaVulkanFunctions',['../struct_vma_vulkan_functions.html',1,'']]] diff --git a/docs/html/search/defines_0.js b/docs/html/search/defines_0.js index 336effe..bf18592 100644 --- a/docs/html/search/defines_0.js +++ b/docs/html/search/defines_0.js @@ -1,5 +1,6 @@ var searchData= [ ['vma_5fdedicated_5fallocation',['VMA_DEDICATED_ALLOCATION',['../vk__mem__alloc_8h.html#af7b860e63b96d11e44ae8587ba06bbf4',1,'vk_mem_alloc.h']]], + ['vma_5frecording_5fenabled',['VMA_RECORDING_ENABLED',['../vk__mem__alloc_8h.html#a1f0c126759fc96ccb6e2d23c101d770c',1,'vk_mem_alloc.h']]], ['vma_5fstats_5fstring_5fenabled',['VMA_STATS_STRING_ENABLED',['../vk__mem__alloc_8h.html#ae25f0d55fd91cb166f002b63244800e1',1,'vk_mem_alloc.h']]] ]; diff --git a/docs/html/search/enums_0.js b/docs/html/search/enums_0.js index 9bd6e39..c5ed539 100644 --- a/docs/html/search/enums_0.js +++ b/docs/html/search/enums_0.js @@ -3,5 +3,6 @@ var searchData= ['vmaallocationcreateflagbits',['VmaAllocationCreateFlagBits',['../vk__mem__alloc_8h.html#ad9889c10c798b040d59c92f257cae597',1,'vk_mem_alloc.h']]], ['vmaallocatorcreateflagbits',['VmaAllocatorCreateFlagBits',['../vk__mem__alloc_8h.html#a4f87c9100d154a65a4ad495f7763cf7c',1,'vk_mem_alloc.h']]], ['vmamemoryusage',['VmaMemoryUsage',['../vk__mem__alloc_8h.html#aa5846affa1e9da3800e3e78fae2305cc',1,'vk_mem_alloc.h']]], - ['vmapoolcreateflagbits',['VmaPoolCreateFlagBits',['../vk__mem__alloc_8h.html#a9a7c45f9c863695d98c83fa5ac940fe7',1,'vk_mem_alloc.h']]] + ['vmapoolcreateflagbits',['VmaPoolCreateFlagBits',['../vk__mem__alloc_8h.html#a9a7c45f9c863695d98c83fa5ac940fe7',1,'vk_mem_alloc.h']]], + ['vmarecordflagbits',['VmaRecordFlagBits',['../vk__mem__alloc_8h.html#a4dd2c44642312a147a4e93373a6e64d2',1,'vk_mem_alloc.h']]] ]; diff --git a/docs/html/search/enumvalues_0.js b/docs/html/search/enumvalues_0.js index ea47966..9f462fc 100644 --- a/docs/html/search/enumvalues_0.js +++ b/docs/html/search/enumvalues_0.js @@ -17,5 +17,7 @@ var searchData= ['vma_5fmemory_5fusage_5fmax_5fenum',['VMA_MEMORY_USAGE_MAX_ENUM',['../vk__mem__alloc_8h.html#aa5846affa1e9da3800e3e78fae2305cca091e69437ef693e8d0d287f1c719ba6e',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5funknown',['VMA_MEMORY_USAGE_UNKNOWN',['../vk__mem__alloc_8h.html#aa5846affa1e9da3800e3e78fae2305ccaf50d27e34e0925cf3a63db8c839121dd',1,'vk_mem_alloc.h']]], ['vma_5fpool_5fcreate_5fflag_5fbits_5fmax_5fenum',['VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM',['../vk__mem__alloc_8h.html#a9a7c45f9c863695d98c83fa5ac940fe7a1c7312bea9ea246846b9054fd6bd6aec',1,'vk_mem_alloc.h']]], - ['vma_5fpool_5fcreate_5fignore_5fbuffer_5fimage_5fgranularity_5fbit',['VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT',['../vk__mem__alloc_8h.html#a9a7c45f9c863695d98c83fa5ac940fe7a9f1a499508a8edb4e8ba40aa0290a3d2',1,'vk_mem_alloc.h']]] + ['vma_5fpool_5fcreate_5fignore_5fbuffer_5fimage_5fgranularity_5fbit',['VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT',['../vk__mem__alloc_8h.html#a9a7c45f9c863695d98c83fa5ac940fe7a9f1a499508a8edb4e8ba40aa0290a3d2',1,'vk_mem_alloc.h']]], + ['vma_5frecord_5fflag_5fbits_5fmax_5fenum',['VMA_RECORD_FLAG_BITS_MAX_ENUM',['../vk__mem__alloc_8h.html#a4dd2c44642312a147a4e93373a6e64d2a20dd17d69966dbffa054739d6090b85e',1,'vk_mem_alloc.h']]], + ['vma_5frecord_5fflush_5fafter_5fcall_5fbit',['VMA_RECORD_FLUSH_AFTER_CALL_BIT',['../vk__mem__alloc_8h.html#a4dd2c44642312a147a4e93373a6e64d2a8e7ab322e8732654be627c4ea8f36cc7',1,'vk_mem_alloc.h']]] ]; diff --git a/docs/html/search/typedefs_1.js b/docs/html/search/typedefs_1.js index 716f5e3..5d53a90 100644 --- a/docs/html/search/typedefs_1.js +++ b/docs/html/search/typedefs_1.js @@ -15,6 +15,9 @@ var searchData= ['vmapoolcreateflags',['VmaPoolCreateFlags',['../vk__mem__alloc_8h.html#a2770e325ea42e087c1b91fdf46d0292a',1,'vk_mem_alloc.h']]], ['vmapoolcreateinfo',['VmaPoolCreateInfo',['../vk__mem__alloc_8h.html#a211706e9348dcee25a843ed4ea69bce7',1,'vk_mem_alloc.h']]], ['vmapoolstats',['VmaPoolStats',['../vk__mem__alloc_8h.html#a2e5612d871d64c5624087b837a338c34',1,'vk_mem_alloc.h']]], + ['vmarecordflagbits',['VmaRecordFlagBits',['../vk__mem__alloc_8h.html#ade20b626a6635ce1bf30ea53dea774e4',1,'vk_mem_alloc.h']]], + ['vmarecordflags',['VmaRecordFlags',['../vk__mem__alloc_8h.html#af3929a1a4547c592fc0b0e55ef452828',1,'vk_mem_alloc.h']]], + ['vmarecordsettings',['VmaRecordSettings',['../vk__mem__alloc_8h.html#a0ab61e87ff6365f1d59915eadc37a9f0',1,'vk_mem_alloc.h']]], ['vmastatinfo',['VmaStatInfo',['../vk__mem__alloc_8h.html#a810b009a788ee8aac72a25b42ffbe31c',1,'vk_mem_alloc.h']]], ['vmastats',['VmaStats',['../vk__mem__alloc_8h.html#a732be855fb4a7c248e6853d928a729af',1,'vk_mem_alloc.h']]], ['vmavulkanfunctions',['VmaVulkanFunctions',['../vk__mem__alloc_8h.html#a97064a1a271b0061ebfc3a079862d0c5',1,'vk_mem_alloc.h']]] diff --git a/docs/html/search/variables_3.js b/docs/html/search/variables_3.js index 7bacfcc..347c9f3 100644 --- a/docs/html/search/variables_3.js +++ b/docs/html/search/variables_3.js @@ -1,5 +1,5 @@ var searchData= [ - ['flags',['flags',['../struct_vma_allocator_create_info.html#a392ea2ecbaff93f91a7c49f735ad4346',1,'VmaAllocatorCreateInfo::flags()'],['../struct_vma_allocation_create_info.html#add09658ac14fe290ace25470ddd6d41b',1,'VmaAllocationCreateInfo::flags()'],['../struct_vma_pool_create_info.html#a8405139f63d078340ae74513a59f5446',1,'VmaPoolCreateInfo::flags()']]], + ['flags',['flags',['../struct_vma_record_settings.html#ad8fdcc92119ae7a8c08c1a564c01d63a',1,'VmaRecordSettings::flags()'],['../struct_vma_allocator_create_info.html#a392ea2ecbaff93f91a7c49f735ad4346',1,'VmaAllocatorCreateInfo::flags()'],['../struct_vma_allocation_create_info.html#add09658ac14fe290ace25470ddd6d41b',1,'VmaAllocationCreateInfo::flags()'],['../struct_vma_pool_create_info.html#a8405139f63d078340ae74513a59f5446',1,'VmaPoolCreateInfo::flags()']]], ['frameinusecount',['frameInUseCount',['../struct_vma_allocator_create_info.html#a21ea188dd212b8171cb9ecbed4a2a3a7',1,'VmaAllocatorCreateInfo::frameInUseCount()'],['../struct_vma_pool_create_info.html#a9437e43ffbb644dbbf7fc4e50cfad6aa',1,'VmaPoolCreateInfo::frameInUseCount()']]] ]; diff --git a/docs/html/search/variables_6.js b/docs/html/search/variables_6.js index 731ce38..46cab85 100644 --- a/docs/html/search/variables_6.js +++ b/docs/html/search/variables_6.js @@ -2,12 +2,14 @@ var searchData= [ ['pallocationcallbacks',['pAllocationCallbacks',['../struct_vma_allocator_create_info.html#a6e409087e3be55400d0e4ccbe43c608d',1,'VmaAllocatorCreateInfo']]], ['pdevicememorycallbacks',['pDeviceMemoryCallbacks',['../struct_vma_allocator_create_info.html#af1380969b5e1ea4c3184a877892d260e',1,'VmaAllocatorCreateInfo']]], + ['pfilepath',['pFilePath',['../struct_vma_record_settings.html#a6cb1fdbf6bcb610b68f2010dd629e89d',1,'VmaRecordSettings']]], ['pfnallocate',['pfnAllocate',['../struct_vma_device_memory_callbacks.html#a4f17f7b255101e733b44d5633aceabfb',1,'VmaDeviceMemoryCallbacks']]], ['pfnfree',['pfnFree',['../struct_vma_device_memory_callbacks.html#abe8a3328bbc916f6f712fdb6b299444c',1,'VmaDeviceMemoryCallbacks']]], ['pheapsizelimit',['pHeapSizeLimit',['../struct_vma_allocator_create_info.html#a31c192aa6cbffa33279f6d9f0c47c44b',1,'VmaAllocatorCreateInfo']]], ['physicaldevice',['physicalDevice',['../struct_vma_allocator_create_info.html#a08230f04ae6ccf8a78150a9e829a7156',1,'VmaAllocatorCreateInfo']]], ['pmappeddata',['pMappedData',['../struct_vma_allocation_info.html#a5eeffbe2d2f30f53370ff14aefbadbe2',1,'VmaAllocationInfo']]], ['pool',['pool',['../struct_vma_allocation_create_info.html#a6272c0555cfd1fe28bff1afeb6190150',1,'VmaAllocationCreateInfo']]], + ['precordsettings',['pRecordSettings',['../struct_vma_allocator_create_info.html#ace2aa4877b16a42b0b7673d4e26000ee',1,'VmaAllocatorCreateInfo']]], ['preferredflags',['preferredFlags',['../struct_vma_allocation_create_info.html#a7fe8d81a1ad10b2a2faacacee5b15d6d',1,'VmaAllocationCreateInfo']]], ['preferredlargeheapblocksize',['preferredLargeHeapBlockSize',['../struct_vma_allocator_create_info.html#a8e4714298e3121cdd8b214a1ae7a637a',1,'VmaAllocatorCreateInfo']]], ['puserdata',['pUserData',['../struct_vma_allocation_create_info.html#a8259e85c272683434f4abb4ddddffe19',1,'VmaAllocationCreateInfo::pUserData()'],['../struct_vma_allocation_info.html#adc507656149c04de7ed95d0042ba2a13',1,'VmaAllocationInfo::pUserData()']]], diff --git a/docs/html/struct_vma_allocator_create_info-members.html b/docs/html/struct_vma_allocator_create_info-members.html index 9340920..598ecde 100644 --- a/docs/html/struct_vma_allocator_create_info-members.html +++ b/docs/html/struct_vma_allocator_create_info-members.html @@ -72,8 +72,9 @@ $(function() { pDeviceMemoryCallbacksVmaAllocatorCreateInfo pHeapSizeLimitVmaAllocatorCreateInfo physicalDeviceVmaAllocatorCreateInfo - preferredLargeHeapBlockSizeVmaAllocatorCreateInfo - pVulkanFunctionsVmaAllocatorCreateInfo + pRecordSettingsVmaAllocatorCreateInfo + preferredLargeHeapBlockSizeVmaAllocatorCreateInfo + pVulkanFunctionsVmaAllocatorCreateInfo
    diff --git a/docs/html/struct_vma_allocator_create_info.html b/docs/html/struct_vma_allocator_create_info.html index 35e8d05..f9735fb 100644 --- a/docs/html/struct_vma_allocator_create_info.html +++ b/docs/html/struct_vma_allocator_create_info.html @@ -100,6 +100,9 @@ Public Attributes const VmaVulkanFunctions * pVulkanFunctions  Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1. More...
      +const VmaRecordSettings * pRecordSettings + Parameters for recording of VMA calls. Can be null. More...
    + 

    Detailed Description

    Description of a Allocator to be created.

    @@ -229,6 +232,23 @@ Public Attributes

    Vulkan physical device.

    It must be valid throughout whole lifetime of created allocator.

    +
    + + +

    ◆ pRecordSettings

    + +
    +
    + + + + +
    const VmaRecordSettings* VmaAllocatorCreateInfo::pRecordSettings
    +
    + +

    Parameters for recording of VMA calls. Can be null.

    +

    If not null, it enables recording of calls to VMA functions to a file. If support for recording is not enabled using VMA_RECORDING_ENABLED macro, creation of the allocator object fails with VK_ERROR_FEATURE_NOT_PRESENT.

    +
    diff --git a/docs/html/struct_vma_record_settings-members.html b/docs/html/struct_vma_record_settings-members.html new file mode 100644 index 0000000..8079ef8 --- /dev/null +++ b/docs/html/struct_vma_record_settings-members.html @@ -0,0 +1,78 @@ + + + + + + + +Vulkan Memory Allocator: Member List + + + + + + + + + +
    +
    + + + + + + +
    +
    Vulkan Memory Allocator +
    +
    +
    + + + + + + + + +
    +
    + + +
    + +
    + +
    +
    +
    +
    VmaRecordSettings Member List
    +
    +
    + +

    This is the complete list of members for VmaRecordSettings, including all inherited members.

    + + + +
    flagsVmaRecordSettings
    pFilePathVmaRecordSettings
    + +
    +Generated by   +doxygen + 1.8.13 +
    + + diff --git a/docs/html/struct_vma_record_settings.html b/docs/html/struct_vma_record_settings.html new file mode 100644 index 0000000..fe9f186 --- /dev/null +++ b/docs/html/struct_vma_record_settings.html @@ -0,0 +1,130 @@ + + + + + + + +Vulkan Memory Allocator: VmaRecordSettings Struct Reference + + + + + + + + + +
    +
    + + + + + + +
    +
    Vulkan Memory Allocator +
    +
    +
    + + + + + + + + +
    +
    + + +
    + +
    + +
    +
    +
    +Public Attributes | +List of all members
    +
    +
    VmaRecordSettings Struct Reference
    +
    +
    + +

    Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings. + More...

    + +

    #include <vk_mem_alloc.h>

    + + + + + + + + +

    +Public Attributes

    VmaRecordFlags flags
     Flags for recording. Use VmaRecordFlagBits enum. More...
     
    const char * pFilePath
     Path to the file that should be written by the recording. More...
     
    +

    Detailed Description

    +

    Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings.

    +

    Member Data Documentation

    + +

    ◆ flags

    + +
    +
    + + + + +
    VmaRecordFlags VmaRecordSettings::flags
    +
    + +

    Flags for recording. Use VmaRecordFlagBits enum.

    + +
    +
    + +

    ◆ pFilePath

    + +
    +
    + + + + +
    const char* VmaRecordSettings::pFilePath
    +
    + +

    Path to the file that should be written by the recording.

    +

    Suggested extension: "csv". If the file already exists, it will be overwritten. It will be opened for the whole time VmaAllocator object is alive. If opening this file fails, creation of the whole allocator object fails.

    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    + +
    +Generated by   +doxygen + 1.8.13 +
    + + diff --git a/docs/html/vk__mem__alloc_8h.html b/docs/html/vk__mem__alloc_8h.html index 8a066b6..4c2a48f 100644 --- a/docs/html/vk__mem__alloc_8h.html +++ b/docs/html/vk__mem__alloc_8h.html @@ -80,6 +80,9 @@ Classes struct  VmaVulkanFunctions  Pointers to some Vulkan functions - a subset used by the library. More...
      +struct  VmaRecordSettings + Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings. More...
    +  struct  VmaAllocatorCreateInfo  Description of a Allocator to be created. More...
      @@ -111,6 +114,8 @@ Classes Macros #define VMA_DEDICATED_ALLOCATION   0   +#define VMA_RECORDING_ENABLED   0 +  #define VMA_STATS_STRING_ENABLED   1   @@ -133,6 +138,14 @@ Typedefs + + + + + + + + @@ -180,6 +193,11 @@ Enumerations } + + +
    typedef struct VmaVulkanFunctions VmaVulkanFunctions
     Pointers to some Vulkan functions - a subset used by the library. More...
     
    typedef enum VmaRecordFlagBits VmaRecordFlagBits
     Flags to be used in VmaRecordSettings::flags. More...
     
    typedef VkFlags VmaRecordFlags
     
    typedef struct VmaRecordSettings VmaRecordSettings
     Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings. More...
     
    typedef struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
     Description of a Allocator to be created. More...
     
     Flags for created VmaAllocator. More...
     
    enum  VmaRecordFlagBits { VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001, +VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF + }
     Flags to be used in VmaRecordSettings::flags. More...
     
    enum  VmaMemoryUsage {
      VMA_MEMORY_USAGE_UNKNOWN = 0, VMA_MEMORY_USAGE_GPU_ONLY = 1, @@ -332,6 +350,20 @@ Functions
    +
    + + +

    ◆ VMA_RECORDING_ENABLED

    + +
    +
    + + + + +
    #define VMA_RECORDING_ENABLED   0
    +
    +
    @@ -611,6 +643,52 @@ Functions

    Describes parameter of existing VmaPool.

    + + + +

    ◆ VmaRecordFlagBits

    + +
    +
    + + + + +
    typedef enum VmaRecordFlagBits VmaRecordFlagBits
    +
    + +

    Flags to be used in VmaRecordSettings::flags.

    + +
    +
    + +

    ◆ VmaRecordFlags

    + +
    +
    + + + + +
    typedef VkFlags VmaRecordFlags
    +
    + +
    +
    + +

    ◆ VmaRecordSettings

    + +
    +
    + + + + +
    typedef struct VmaRecordSettings VmaRecordSettings
    +
    + +

    Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings.

    +
    @@ -803,6 +881,28 @@ Functions VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM  + + + +

    ◆ VmaRecordFlagBits

    + +
    +
    + + + + +
    enum VmaRecordFlagBits
    +
    + +

    Flags to be used in VmaRecordSettings::flags.

    + + + +
    Enumerator
    VMA_RECORD_FLUSH_AFTER_CALL_BIT 

    Enables flush after recording every function call.

    +

    Enable it if you expect your application to crash, which may leave recording file truncated. It may degrade performance though.

    +
    VMA_RECORD_FLAG_BITS_MAX_ENUM 
    +

    Function Documentation

    diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 8354ec9..d5b79bf 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,166 +62,176 @@ $(function() {
    vk_mem_alloc.h
    -Go to the documentation of this file.
    1 //
    2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
    3 //
    4 // Permission is hereby granted, free of charge, to any person obtaining a copy
    5 // of this software and associated documentation files (the "Software"), to deal
    6 // in the Software without restriction, including without limitation the rights
    7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    8 // copies of the Software, and to permit persons to whom the Software is
    9 // furnished to do so, subject to the following conditions:
    10 //
    11 // The above copyright notice and this permission notice shall be included in
    12 // all copies or substantial portions of the Software.
    13 //
    14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    20 // THE SOFTWARE.
    21 //
    22 
    23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
    24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
    25 
    26 #ifdef __cplusplus
    27 extern "C" {
    28 #endif
    29 
    1184 #include <vulkan/vulkan.h>
    1185 
    1186 #if !defined(VMA_DEDICATED_ALLOCATION)
    1187  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
    1188  #define VMA_DEDICATED_ALLOCATION 1
    1189  #else
    1190  #define VMA_DEDICATED_ALLOCATION 0
    1191  #endif
    1192 #endif
    1193 
    1203 VK_DEFINE_HANDLE(VmaAllocator)
    1204 
    1205 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
    1207  VmaAllocator allocator,
    1208  uint32_t memoryType,
    1209  VkDeviceMemory memory,
    1210  VkDeviceSize size);
    1212 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
    1213  VmaAllocator allocator,
    1214  uint32_t memoryType,
    1215  VkDeviceMemory memory,
    1216  VkDeviceSize size);
    1217 
    1225 typedef struct VmaDeviceMemoryCallbacks {
    1227  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
    1229  PFN_vmaFreeDeviceMemoryFunction pfnFree;
    1230 } VmaDeviceMemoryCallbacks;
    1231 
    1233 typedef enum VmaAllocatorCreateFlagBits {
    1238  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
    1260  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
    1261 
    1262  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1263 } VmaAllocatorCreateFlagBits;
    1264 typedef VkFlags VmaAllocatorCreateFlags;
    1265 
    1270 typedef struct VmaVulkanFunctions {
    1271  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
    1272  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
    1273  PFN_vkAllocateMemory vkAllocateMemory;
    1274  PFN_vkFreeMemory vkFreeMemory;
    1275  PFN_vkMapMemory vkMapMemory;
    1276  PFN_vkUnmapMemory vkUnmapMemory;
    1277  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
    1278  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
    1279  PFN_vkBindBufferMemory vkBindBufferMemory;
    1280  PFN_vkBindImageMemory vkBindImageMemory;
    1281  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
    1282  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
    1283  PFN_vkCreateBuffer vkCreateBuffer;
    1284  PFN_vkDestroyBuffer vkDestroyBuffer;
    1285  PFN_vkCreateImage vkCreateImage;
    1286  PFN_vkDestroyImage vkDestroyImage;
    1287 #if VMA_DEDICATED_ALLOCATION
    1288  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
    1289  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
    1290 #endif
    1291 } VmaVulkanFunctions;
    1292 
    1294 typedef struct VmaAllocatorCreateInfo
    1295 {
    1297  VmaAllocatorCreateFlags flags;
    1299 
    1300  VkPhysicalDevice physicalDevice;
    1302 
    1303  VkDevice device;
    1305 
    1306  VkDeviceSize preferredLargeHeapBlockSize;
    1308 
    1309  const VkAllocationCallbacks* pAllocationCallbacks;
    1311 
    1312  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
    1326  uint32_t frameInUseCount;
    1350  const VkDeviceSize* pHeapSizeLimit;
    1362  const VmaVulkanFunctions* pVulkanFunctions;
    1363 } VmaAllocatorCreateInfo;
    1364 
    1366 VkResult vmaCreateAllocator(
    1367  const VmaAllocatorCreateInfo* pCreateInfo,
    1368  VmaAllocator* pAllocator);
    1369 
    1371 void vmaDestroyAllocator(
    1372  VmaAllocator allocator);
    1373 
    1378 void vmaGetPhysicalDeviceProperties(
    1379  VmaAllocator allocator,
    1380  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
    1381 
    1386 void vmaGetMemoryProperties(
    1387  VmaAllocator allocator,
    1388  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
    1389 
    1396 void vmaGetMemoryTypeProperties(
    1397  VmaAllocator allocator,
    1398  uint32_t memoryTypeIndex,
    1399  VkMemoryPropertyFlags* pFlags);
    1400 
    1409 void vmaSetCurrentFrameIndex(
    1410  VmaAllocator allocator,
    1411  uint32_t frameIndex);
    1412 
    1415 typedef struct VmaStatInfo
    1416 {
    1418  uint32_t blockCount;
    1420  uint32_t allocationCount;
    1422  uint32_t unusedRangeCount;
    1424  VkDeviceSize usedBytes;
    1426  VkDeviceSize unusedBytes;
    1427  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
    1428  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
    1429 } VmaStatInfo;
    1430 
    1432 typedef struct VmaStats
    1433 {
    1434  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
    1435  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
    1436  VmaStatInfo total;
    1437 } VmaStats;
    1438 
    1440 void vmaCalculateStats(
    1441  VmaAllocator allocator,
    1442  VmaStats* pStats);
    1443 
    1444 #define VMA_STATS_STRING_ENABLED 1
    1445 
    1446 #if VMA_STATS_STRING_ENABLED
    1447 
    1449 
    1451 void vmaBuildStatsString(
    1452  VmaAllocator allocator,
    1453  char** ppStatsString,
    1454  VkBool32 detailedMap);
    1455 
    1456 void vmaFreeStatsString(
    1457  VmaAllocator allocator,
    1458  char* pStatsString);
    1459 
    1460 #endif // #if VMA_STATS_STRING_ENABLED
    1461 
    1470 VK_DEFINE_HANDLE(VmaPool)
    1471 
    1472 typedef enum VmaMemoryUsage
    1473 {
    1477  VMA_MEMORY_USAGE_UNKNOWN = 0,
    1494  VMA_MEMORY_USAGE_GPU_ONLY = 1,
    1504  VMA_MEMORY_USAGE_CPU_ONLY = 2,
    1511  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
    1520  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
    1521  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
    1522 } VmaMemoryUsage;
    1523 
    1525 typedef enum VmaAllocationCreateFlagBits {
    1537  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
    1538 
    1548  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
    1561  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
    1574  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
    1581  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
    1587  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
    1588 
    1589  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1590 } VmaAllocationCreateFlagBits;
    1591 typedef VkFlags VmaAllocationCreateFlags;
    1592 
    1593 typedef struct VmaAllocationCreateInfo
    1594 {
    1596  VmaAllocationCreateFlags flags;
    1602  VmaMemoryUsage usage;
    1607  VkMemoryPropertyFlags requiredFlags;
    1612  VkMemoryPropertyFlags preferredFlags;
    1620  uint32_t memoryTypeBits;
    1626  VmaPool pool;
    1633  void* pUserData;
    1634 } VmaAllocationCreateInfo;
    1635 
    1652 VkResult vmaFindMemoryTypeIndex(
    1653  VmaAllocator allocator,
    1654  uint32_t memoryTypeBits,
    1655  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1656  uint32_t* pMemoryTypeIndex);
    1657 
    1670 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    1671  VmaAllocator allocator,
    1672  const VkBufferCreateInfo* pBufferCreateInfo,
    1673  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1674  uint32_t* pMemoryTypeIndex);
    1675 
    1688 VkResult vmaFindMemoryTypeIndexForImageInfo(
    1689  VmaAllocator allocator,
    1690  const VkImageCreateInfo* pImageCreateInfo,
    1691  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1692  uint32_t* pMemoryTypeIndex);
    1693 
    1695 typedef enum VmaPoolCreateFlagBits {
    1713  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
    1714 
    1715  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1716 } VmaPoolCreateFlagBits;
    1717 typedef VkFlags VmaPoolCreateFlags;
    1718 
    1721 typedef struct VmaPoolCreateInfo {
    1724  uint32_t memoryTypeIndex;
    1727  VmaPoolCreateFlags flags;
    1732  VkDeviceSize blockSize;
    1737  size_t minBlockCount;
    1745  size_t maxBlockCount;
    1759  uint32_t frameInUseCount;
    1760 } VmaPoolCreateInfo;
    1761 
    1764 typedef struct VmaPoolStats {
    1767  VkDeviceSize size;
    1770  VkDeviceSize unusedSize;
    1773  size_t allocationCount;
    1776  size_t unusedRangeCount;
    1783  VkDeviceSize unusedRangeSizeMax;
    1784 } VmaPoolStats;
    1785 
    1792 VkResult vmaCreatePool(
    1793  VmaAllocator allocator,
    1794  const VmaPoolCreateInfo* pCreateInfo,
    1795  VmaPool* pPool);
    1796 
    1799 void vmaDestroyPool(
    1800  VmaAllocator allocator,
    1801  VmaPool pool);
    1802 
    1809 void vmaGetPoolStats(
    1810  VmaAllocator allocator,
    1811  VmaPool pool,
    1812  VmaPoolStats* pPoolStats);
    1813 
    1820 void vmaMakePoolAllocationsLost(
    1821  VmaAllocator allocator,
    1822  VmaPool pool,
    1823  size_t* pLostAllocationCount);
    1824 
    1839 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
    1840 
    1865 VK_DEFINE_HANDLE(VmaAllocation)
    1866 
    1867 
    1869 typedef struct VmaAllocationInfo {
    1874  uint32_t memoryType;
    1883  VkDeviceMemory deviceMemory;
    1888  VkDeviceSize offset;
    1893  VkDeviceSize size;
    1902  void* pMappedData;
    1907  void* pUserData;
    1908 } VmaAllocationInfo;
    1909 
    1920 VkResult vmaAllocateMemory(
    1921  VmaAllocator allocator,
    1922  const VkMemoryRequirements* pVkMemoryRequirements,
    1923  const VmaAllocationCreateInfo* pCreateInfo,
    1924  VmaAllocation* pAllocation,
    1925  VmaAllocationInfo* pAllocationInfo);
    1926 
    1933 VkResult vmaAllocateMemoryForBuffer(
    1934  VmaAllocator allocator,
    1935  VkBuffer buffer,
    1936  const VmaAllocationCreateInfo* pCreateInfo,
    1937  VmaAllocation* pAllocation,
    1938  VmaAllocationInfo* pAllocationInfo);
    1939 
    1941 VkResult vmaAllocateMemoryForImage(
    1942  VmaAllocator allocator,
    1943  VkImage image,
    1944  const VmaAllocationCreateInfo* pCreateInfo,
    1945  VmaAllocation* pAllocation,
    1946  VmaAllocationInfo* pAllocationInfo);
    1947 
    1949 void vmaFreeMemory(
    1950  VmaAllocator allocator,
    1951  VmaAllocation allocation);
    1952 
    1969 void vmaGetAllocationInfo(
    1970  VmaAllocator allocator,
    1971  VmaAllocation allocation,
    1972  VmaAllocationInfo* pAllocationInfo);
    1973 
    1988 VkBool32 vmaTouchAllocation(
    1989  VmaAllocator allocator,
    1990  VmaAllocation allocation);
    1991 
    2005 void vmaSetAllocationUserData(
    2006  VmaAllocator allocator,
    2007  VmaAllocation allocation,
    2008  void* pUserData);
    2009 
    2020 void vmaCreateLostAllocation(
    2021  VmaAllocator allocator,
    2022  VmaAllocation* pAllocation);
    2023 
    2058 VkResult vmaMapMemory(
    2059  VmaAllocator allocator,
    2060  VmaAllocation allocation,
    2061  void** ppData);
    2062 
    2067 void vmaUnmapMemory(
    2068  VmaAllocator allocator,
    2069  VmaAllocation allocation);
    2070 
    2083 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2084 
    2097 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2098 
    2115 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
    2116 
    2118 typedef struct VmaDefragmentationInfo {
    2123  VkDeviceSize maxBytesToMove;
    2128  uint32_t maxAllocationsToMove;
    2129 } VmaDefragmentationInfo;
    2130 
    2132 typedef struct VmaDefragmentationStats {
    2134  VkDeviceSize bytesMoved;
    2136  VkDeviceSize bytesFreed;
    2138  uint32_t allocationsMoved;
    2140  uint32_t deviceMemoryBlocksFreed;
    2141 } VmaDefragmentationStats;
    2142 
    2225 VkResult vmaDefragment(
    2226  VmaAllocator allocator,
    2227  VmaAllocation* pAllocations,
    2228  size_t allocationCount,
    2229  VkBool32* pAllocationsChanged,
    2230  const VmaDefragmentationInfo *pDefragmentationInfo,
    2231  VmaDefragmentationStats* pDefragmentationStats);
    2232 
    2245 VkResult vmaBindBufferMemory(
    2246  VmaAllocator allocator,
    2247  VmaAllocation allocation,
    2248  VkBuffer buffer);
    2249 
    2262 VkResult vmaBindImageMemory(
    2263  VmaAllocator allocator,
    2264  VmaAllocation allocation,
    2265  VkImage image);
    2266 
    2293 VkResult vmaCreateBuffer(
    2294  VmaAllocator allocator,
    2295  const VkBufferCreateInfo* pBufferCreateInfo,
    2296  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2297  VkBuffer* pBuffer,
    2298  VmaAllocation* pAllocation,
    2299  VmaAllocationInfo* pAllocationInfo);
    2300 
    2312 void vmaDestroyBuffer(
    2313  VmaAllocator allocator,
    2314  VkBuffer buffer,
    2315  VmaAllocation allocation);
    2316 
    2318 VkResult vmaCreateImage(
    2319  VmaAllocator allocator,
    2320  const VkImageCreateInfo* pImageCreateInfo,
    2321  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2322  VkImage* pImage,
    2323  VmaAllocation* pAllocation,
    2324  VmaAllocationInfo* pAllocationInfo);
    2325 
    2337 void vmaDestroyImage(
    2338  VmaAllocator allocator,
    2339  VkImage image,
    2340  VmaAllocation allocation);
    2341 
    2342 #ifdef __cplusplus
    2343 }
    2344 #endif
    2345 
    2346 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
    2347 
    2348 // For Visual Studio IntelliSense.
    2349 #if defined(__cplusplus) && defined(__INTELLISENSE__)
    2350 #define VMA_IMPLEMENTATION
    2351 #endif
    2352 
    2353 #ifdef VMA_IMPLEMENTATION
    2354 #undef VMA_IMPLEMENTATION
    2355 
    2356 #include <cstdint>
    2357 #include <cstdlib>
    2358 #include <cstring>
    2359 
    2360 /*******************************************************************************
    2361 CONFIGURATION SECTION
    2362 
    2363 Define some of these macros before each #include of this header or change them
    2364 here if you need other then default behavior depending on your environment.
    2365 */
    2366 
    2367 /*
    2368 Define this macro to 1 to make the library fetch pointers to Vulkan functions
    2369 internally, like:
    2370 
    2371  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    2372 
    2373 Define to 0 if you are going to provide you own pointers to Vulkan functions via
    2374 VmaAllocatorCreateInfo::pVulkanFunctions.
    2375 */
    2376 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
    2377 #define VMA_STATIC_VULKAN_FUNCTIONS 1
    2378 #endif
    2379 
    2380 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
    2381 //#define VMA_USE_STL_CONTAINERS 1
    2382 
    2383 /* Set this macro to 1 to make the library including and using STL containers:
    2384 std::pair, std::vector, std::list, std::unordered_map.
    2385 
    2386 Set it to 0 or undefined to make the library using its own implementation of
    2387 the containers.
    2388 */
    2389 #if VMA_USE_STL_CONTAINERS
    2390  #define VMA_USE_STL_VECTOR 1
    2391  #define VMA_USE_STL_UNORDERED_MAP 1
    2392  #define VMA_USE_STL_LIST 1
    2393 #endif
    2394 
    2395 #if VMA_USE_STL_VECTOR
    2396  #include <vector>
    2397 #endif
    2398 
    2399 #if VMA_USE_STL_UNORDERED_MAP
    2400  #include <unordered_map>
    2401 #endif
    2402 
    2403 #if VMA_USE_STL_LIST
    2404  #include <list>
    2405 #endif
    2406 
    2407 /*
    2408 Following headers are used in this CONFIGURATION section only, so feel free to
    2409 remove them if not needed.
    2410 */
    2411 #include <cassert> // for assert
    2412 #include <algorithm> // for min, max
    2413 #include <mutex> // for std::mutex
    2414 #include <atomic> // for std::atomic
    2415 
    2416 #ifndef VMA_NULL
    2417  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
    2418  #define VMA_NULL nullptr
    2419 #endif
    2420 
    2421 #if defined(__APPLE__) || defined(__ANDROID__)
    2422 #include <cstdlib>
    2423 void *aligned_alloc(size_t alignment, size_t size)
    2424 {
    2425  // alignment must be >= sizeof(void*)
    2426  if(alignment < sizeof(void*))
    2427  {
    2428  alignment = sizeof(void*);
    2429  }
    2430 
    2431  void *pointer;
    2432  if(posix_memalign(&pointer, alignment, size) == 0)
    2433  return pointer;
    2434  return VMA_NULL;
    2435 }
    2436 #endif
    2437 
    2438 // If your compiler is not compatible with C++11 and definition of
    2439 // aligned_alloc() function is missing, uncommeting following line may help:
    2440 
    2441 //#include <malloc.h>
    2442 
    2443 // Normal assert to check for programmer's errors, especially in Debug configuration.
    2444 #ifndef VMA_ASSERT
    2445  #ifdef _DEBUG
    2446  #define VMA_ASSERT(expr) assert(expr)
    2447  #else
    2448  #define VMA_ASSERT(expr)
    2449  #endif
    2450 #endif
    2451 
    2452 // Assert that will be called very often, like inside data structures e.g. operator[].
    2453 // Making it non-empty can make program slow.
    2454 #ifndef VMA_HEAVY_ASSERT
    2455  #ifdef _DEBUG
    2456  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
    2457  #else
    2458  #define VMA_HEAVY_ASSERT(expr)
    2459  #endif
    2460 #endif
    2461 
    2462 #ifndef VMA_ALIGN_OF
    2463  #define VMA_ALIGN_OF(type) (__alignof(type))
    2464 #endif
    2465 
    2466 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
    2467  #if defined(_WIN32)
    2468  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
    2469  #else
    2470  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
    2471  #endif
    2472 #endif
    2473 
    2474 #ifndef VMA_SYSTEM_FREE
    2475  #if defined(_WIN32)
    2476  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
    2477  #else
    2478  #define VMA_SYSTEM_FREE(ptr) free(ptr)
    2479  #endif
    2480 #endif
    2481 
    2482 #ifndef VMA_MIN
    2483  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
    2484 #endif
    2485 
    2486 #ifndef VMA_MAX
    2487  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
    2488 #endif
    2489 
    2490 #ifndef VMA_SWAP
    2491  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
    2492 #endif
    2493 
    2494 #ifndef VMA_SORT
    2495  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
    2496 #endif
    2497 
    2498 #ifndef VMA_DEBUG_LOG
    2499  #define VMA_DEBUG_LOG(format, ...)
    2500  /*
    2501  #define VMA_DEBUG_LOG(format, ...) do { \
    2502  printf(format, __VA_ARGS__); \
    2503  printf("\n"); \
    2504  } while(false)
    2505  */
    2506 #endif
    2507 
    2508 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
    2509 #if VMA_STATS_STRING_ENABLED
    2510  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
    2511  {
    2512  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
    2513  }
    2514  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
    2515  {
    2516  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
    2517  }
    2518  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
    2519  {
    2520  snprintf(outStr, strLen, "%p", ptr);
    2521  }
    2522 #endif
    2523 
    2524 #ifndef VMA_MUTEX
    2525  class VmaMutex
    2526  {
    2527  public:
    2528  VmaMutex() { }
    2529  ~VmaMutex() { }
    2530  void Lock() { m_Mutex.lock(); }
    2531  void Unlock() { m_Mutex.unlock(); }
    2532  private:
    2533  std::mutex m_Mutex;
    2534  };
    2535  #define VMA_MUTEX VmaMutex
    2536 #endif
    2537 
    2538 /*
    2539 If providing your own implementation, you need to implement a subset of std::atomic:
    2540 
    2541 - Constructor(uint32_t desired)
    2542 - uint32_t load() const
    2543 - void store(uint32_t desired)
    2544 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
    2545 */
    2546 #ifndef VMA_ATOMIC_UINT32
    2547  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
    2548 #endif
    2549 
    2550 #ifndef VMA_BEST_FIT
    2551 
    2563  #define VMA_BEST_FIT (1)
    2564 #endif
    2565 
    2566 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
    2567 
    2571  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
    2572 #endif
    2573 
    2574 #ifndef VMA_DEBUG_ALIGNMENT
    2575 
    2579  #define VMA_DEBUG_ALIGNMENT (1)
    2580 #endif
    2581 
    2582 #ifndef VMA_DEBUG_MARGIN
    2583 
    2587  #define VMA_DEBUG_MARGIN (0)
    2588 #endif
    2589 
    2590 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
    2591 
    2595  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
    2596 #endif
    2597 
    2598 #ifndef VMA_DEBUG_DETECT_CORRUPTION
    2599 
    2604  #define VMA_DEBUG_DETECT_CORRUPTION (0)
    2605 #endif
    2606 
    2607 #ifndef VMA_DEBUG_GLOBAL_MUTEX
    2608 
    2612  #define VMA_DEBUG_GLOBAL_MUTEX (0)
    2613 #endif
    2614 
    2615 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
    2616 
    2620  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
    2621 #endif
    2622 
    2623 #ifndef VMA_SMALL_HEAP_MAX_SIZE
    2624  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
    2626 #endif
    2627 
    2628 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
    2629  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
    2631 #endif
    2632 
    2633 #ifndef VMA_CLASS_NO_COPY
    2634  #define VMA_CLASS_NO_COPY(className) \
    2635  private: \
    2636  className(const className&) = delete; \
    2637  className& operator=(const className&) = delete;
    2638 #endif
    2639 
    2640 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
    2641 
    2642 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
    2643 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
    2644 
    2645 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
    2646 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
    2647 
    2648 /*******************************************************************************
    2649 END OF CONFIGURATION
    2650 */
    2651 
    2652 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
    2653  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
    2654 
    2655 // Returns number of bits set to 1 in (v).
    2656 static inline uint32_t VmaCountBitsSet(uint32_t v)
    2657 {
    2658  uint32_t c = v - ((v >> 1) & 0x55555555);
    2659  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
    2660  c = ((c >> 4) + c) & 0x0F0F0F0F;
    2661  c = ((c >> 8) + c) & 0x00FF00FF;
    2662  c = ((c >> 16) + c) & 0x0000FFFF;
    2663  return c;
    2664 }
    2665 
    2666 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
    2667 // Use types like uint32_t, uint64_t as T.
    2668 template <typename T>
    2669 static inline T VmaAlignUp(T val, T align)
    2670 {
    2671  return (val + align - 1) / align * align;
    2672 }
    2673 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
    2674 // Use types like uint32_t, uint64_t as T.
    2675 template <typename T>
    2676 static inline T VmaAlignDown(T val, T align)
    2677 {
    2678  return val / align * align;
    2679 }
    2680 
    2681 // Division with mathematical rounding to nearest number.
    2682 template <typename T>
    2683 inline T VmaRoundDiv(T x, T y)
    2684 {
    2685  return (x + (y / (T)2)) / y;
    2686 }
    2687 
    2688 #ifndef VMA_SORT
    2689 
    2690 template<typename Iterator, typename Compare>
    2691 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
    2692 {
    2693  Iterator centerValue = end; --centerValue;
    2694  Iterator insertIndex = beg;
    2695  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
    2696  {
    2697  if(cmp(*memTypeIndex, *centerValue))
    2698  {
    2699  if(insertIndex != memTypeIndex)
    2700  {
    2701  VMA_SWAP(*memTypeIndex, *insertIndex);
    2702  }
    2703  ++insertIndex;
    2704  }
    2705  }
    2706  if(insertIndex != centerValue)
    2707  {
    2708  VMA_SWAP(*insertIndex, *centerValue);
    2709  }
    2710  return insertIndex;
    2711 }
    2712 
    2713 template<typename Iterator, typename Compare>
    2714 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
    2715 {
    2716  if(beg < end)
    2717  {
    2718  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
    2719  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
    2720  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
    2721  }
    2722 }
    2723 
    2724 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
    2725 
    2726 #endif // #ifndef VMA_SORT
    2727 
    2728 /*
    2729 Returns true if two memory blocks occupy overlapping pages.
    2730 ResourceA must be in less memory offset than ResourceB.
    2731 
    2732 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
    2733 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
    2734 */
    2735 static inline bool VmaBlocksOnSamePage(
    2736  VkDeviceSize resourceAOffset,
    2737  VkDeviceSize resourceASize,
    2738  VkDeviceSize resourceBOffset,
    2739  VkDeviceSize pageSize)
    2740 {
    2741  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
    2742  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
    2743  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
    2744  VkDeviceSize resourceBStart = resourceBOffset;
    2745  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
    2746  return resourceAEndPage == resourceBStartPage;
    2747 }
    2748 
    2749 enum VmaSuballocationType
    2750 {
    2751  VMA_SUBALLOCATION_TYPE_FREE = 0,
    2752  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
    2753  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
    2754  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
    2755  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
    2756  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
    2757  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
    2758 };
    2759 
    2760 /*
    2761 Returns true if given suballocation types could conflict and must respect
    2762 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
    2763 or linear image and another one is optimal image. If type is unknown, behave
    2764 conservatively.
    2765 */
    2766 static inline bool VmaIsBufferImageGranularityConflict(
    2767  VmaSuballocationType suballocType1,
    2768  VmaSuballocationType suballocType2)
    2769 {
    2770  if(suballocType1 > suballocType2)
    2771  {
    2772  VMA_SWAP(suballocType1, suballocType2);
    2773  }
    2774 
    2775  switch(suballocType1)
    2776  {
    2777  case VMA_SUBALLOCATION_TYPE_FREE:
    2778  return false;
    2779  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
    2780  return true;
    2781  case VMA_SUBALLOCATION_TYPE_BUFFER:
    2782  return
    2783  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2784  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2785  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
    2786  return
    2787  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2788  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
    2789  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2790  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
    2791  return
    2792  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2793  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
    2794  return false;
    2795  default:
    2796  VMA_ASSERT(0);
    2797  return true;
    2798  }
    2799 }
    2800 
    2801 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
    2802 {
    2803  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
    2804  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2805  for(size_t i = 0; i < numberCount; ++i, ++pDst)
    2806  {
    2807  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
    2808  }
    2809 }
    2810 
    2811 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
    2812 {
    2813  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
    2814  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2815  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
    2816  {
    2817  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
    2818  {
    2819  return false;
    2820  }
    2821  }
    2822  return true;
    2823 }
    2824 
    2825 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
    2826 struct VmaMutexLock
    2827 {
    2828  VMA_CLASS_NO_COPY(VmaMutexLock)
    2829 public:
    2830  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
    2831  m_pMutex(useMutex ? &mutex : VMA_NULL)
    2832  {
    2833  if(m_pMutex)
    2834  {
    2835  m_pMutex->Lock();
    2836  }
    2837  }
    2838 
    2839  ~VmaMutexLock()
    2840  {
    2841  if(m_pMutex)
    2842  {
    2843  m_pMutex->Unlock();
    2844  }
    2845  }
    2846 
    2847 private:
    2848  VMA_MUTEX* m_pMutex;
    2849 };
    2850 
    2851 #if VMA_DEBUG_GLOBAL_MUTEX
    2852  static VMA_MUTEX gDebugGlobalMutex;
    2853  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
    2854 #else
    2855  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
    2856 #endif
    2857 
    2858 // Minimum size of a free suballocation to register it in the free suballocation collection.
    2859 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
    2860 
    2861 /*
    2862 Performs binary search and returns iterator to first element that is greater or
    2863 equal to (key), according to comparison (cmp).
    2864 
    2865 Cmp should return true if first argument is less than second argument.
    2866 
    2867 Returned value is the found element, if present in the collection or place where
    2868 new element with value (key) should be inserted.
    2869 */
    2870 template <typename IterT, typename KeyT, typename CmpT>
    2871 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
    2872 {
    2873  size_t down = 0, up = (end - beg);
    2874  while(down < up)
    2875  {
    2876  const size_t mid = (down + up) / 2;
    2877  if(cmp(*(beg+mid), key))
    2878  {
    2879  down = mid + 1;
    2880  }
    2881  else
    2882  {
    2883  up = mid;
    2884  }
    2885  }
    2886  return beg + down;
    2887 }
    2888 
    2890 // Memory allocation
    2891 
    2892 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
    2893 {
    2894  if((pAllocationCallbacks != VMA_NULL) &&
    2895  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
    2896  {
    2897  return (*pAllocationCallbacks->pfnAllocation)(
    2898  pAllocationCallbacks->pUserData,
    2899  size,
    2900  alignment,
    2901  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    2902  }
    2903  else
    2904  {
    2905  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
    2906  }
    2907 }
    2908 
    2909 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
    2910 {
    2911  if((pAllocationCallbacks != VMA_NULL) &&
    2912  (pAllocationCallbacks->pfnFree != VMA_NULL))
    2913  {
    2914  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
    2915  }
    2916  else
    2917  {
    2918  VMA_SYSTEM_FREE(ptr);
    2919  }
    2920 }
    2921 
    2922 template<typename T>
    2923 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
    2924 {
    2925  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
    2926 }
    2927 
    2928 template<typename T>
    2929 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
    2930 {
    2931  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
    2932 }
    2933 
    2934 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
    2935 
    2936 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
    2937 
    2938 template<typename T>
    2939 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
    2940 {
    2941  ptr->~T();
    2942  VmaFree(pAllocationCallbacks, ptr);
    2943 }
    2944 
    2945 template<typename T>
    2946 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
    2947 {
    2948  if(ptr != VMA_NULL)
    2949  {
    2950  for(size_t i = count; i--; )
    2951  {
    2952  ptr[i].~T();
    2953  }
    2954  VmaFree(pAllocationCallbacks, ptr);
    2955  }
    2956 }
    2957 
    2958 // STL-compatible allocator.
    2959 template<typename T>
    2960 class VmaStlAllocator
    2961 {
    2962 public:
    2963  const VkAllocationCallbacks* const m_pCallbacks;
    2964  typedef T value_type;
    2965 
    2966  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
    2967  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
    2968 
    2969  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
    2970  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
    2971 
    2972  template<typename U>
    2973  bool operator==(const VmaStlAllocator<U>& rhs) const
    2974  {
    2975  return m_pCallbacks == rhs.m_pCallbacks;
    2976  }
    2977  template<typename U>
    2978  bool operator!=(const VmaStlAllocator<U>& rhs) const
    2979  {
    2980  return m_pCallbacks != rhs.m_pCallbacks;
    2981  }
    2982 
    2983  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
    2984 };
    2985 
    2986 #if VMA_USE_STL_VECTOR
    2987 
    2988 #define VmaVector std::vector
    2989 
    2990 template<typename T, typename allocatorT>
    2991 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
    2992 {
    2993  vec.insert(vec.begin() + index, item);
    2994 }
    2995 
    2996 template<typename T, typename allocatorT>
    2997 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
    2998 {
    2999  vec.erase(vec.begin() + index);
    3000 }
    3001 
    3002 #else // #if VMA_USE_STL_VECTOR
    3003 
    3004 /* Class with interface compatible with subset of std::vector.
    3005 T must be POD because constructors and destructors are not called and memcpy is
    3006 used for these objects. */
    3007 template<typename T, typename AllocatorT>
    3008 class VmaVector
    3009 {
    3010 public:
    3011  typedef T value_type;
    3012 
    3013  VmaVector(const AllocatorT& allocator) :
    3014  m_Allocator(allocator),
    3015  m_pArray(VMA_NULL),
    3016  m_Count(0),
    3017  m_Capacity(0)
    3018  {
    3019  }
    3020 
    3021  VmaVector(size_t count, const AllocatorT& allocator) :
    3022  m_Allocator(allocator),
    3023  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
    3024  m_Count(count),
    3025  m_Capacity(count)
    3026  {
    3027  }
    3028 
    3029  VmaVector(const VmaVector<T, AllocatorT>& src) :
    3030  m_Allocator(src.m_Allocator),
    3031  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
    3032  m_Count(src.m_Count),
    3033  m_Capacity(src.m_Count)
    3034  {
    3035  if(m_Count != 0)
    3036  {
    3037  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
    3038  }
    3039  }
    3040 
    3041  ~VmaVector()
    3042  {
    3043  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3044  }
    3045 
    3046  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
    3047  {
    3048  if(&rhs != this)
    3049  {
    3050  resize(rhs.m_Count);
    3051  if(m_Count != 0)
    3052  {
    3053  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
    3054  }
    3055  }
    3056  return *this;
    3057  }
    3058 
    3059  bool empty() const { return m_Count == 0; }
    3060  size_t size() const { return m_Count; }
    3061  T* data() { return m_pArray; }
    3062  const T* data() const { return m_pArray; }
    3063 
    3064  T& operator[](size_t index)
    3065  {
    3066  VMA_HEAVY_ASSERT(index < m_Count);
    3067  return m_pArray[index];
    3068  }
    3069  const T& operator[](size_t index) const
    3070  {
    3071  VMA_HEAVY_ASSERT(index < m_Count);
    3072  return m_pArray[index];
    3073  }
    3074 
    3075  T& front()
    3076  {
    3077  VMA_HEAVY_ASSERT(m_Count > 0);
    3078  return m_pArray[0];
    3079  }
    3080  const T& front() const
    3081  {
    3082  VMA_HEAVY_ASSERT(m_Count > 0);
    3083  return m_pArray[0];
    3084  }
    3085  T& back()
    3086  {
    3087  VMA_HEAVY_ASSERT(m_Count > 0);
    3088  return m_pArray[m_Count - 1];
    3089  }
    3090  const T& back() const
    3091  {
    3092  VMA_HEAVY_ASSERT(m_Count > 0);
    3093  return m_pArray[m_Count - 1];
    3094  }
    3095 
    3096  void reserve(size_t newCapacity, bool freeMemory = false)
    3097  {
    3098  newCapacity = VMA_MAX(newCapacity, m_Count);
    3099 
    3100  if((newCapacity < m_Capacity) && !freeMemory)
    3101  {
    3102  newCapacity = m_Capacity;
    3103  }
    3104 
    3105  if(newCapacity != m_Capacity)
    3106  {
    3107  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
    3108  if(m_Count != 0)
    3109  {
    3110  memcpy(newArray, m_pArray, m_Count * sizeof(T));
    3111  }
    3112  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3113  m_Capacity = newCapacity;
    3114  m_pArray = newArray;
    3115  }
    3116  }
    3117 
    3118  void resize(size_t newCount, bool freeMemory = false)
    3119  {
    3120  size_t newCapacity = m_Capacity;
    3121  if(newCount > m_Capacity)
    3122  {
    3123  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
    3124  }
    3125  else if(freeMemory)
    3126  {
    3127  newCapacity = newCount;
    3128  }
    3129 
    3130  if(newCapacity != m_Capacity)
    3131  {
    3132  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
    3133  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
    3134  if(elementsToCopy != 0)
    3135  {
    3136  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
    3137  }
    3138  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3139  m_Capacity = newCapacity;
    3140  m_pArray = newArray;
    3141  }
    3142 
    3143  m_Count = newCount;
    3144  }
    3145 
    3146  void clear(bool freeMemory = false)
    3147  {
    3148  resize(0, freeMemory);
    3149  }
    3150 
    3151  void insert(size_t index, const T& src)
    3152  {
    3153  VMA_HEAVY_ASSERT(index <= m_Count);
    3154  const size_t oldCount = size();
    3155  resize(oldCount + 1);
    3156  if(index < oldCount)
    3157  {
    3158  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
    3159  }
    3160  m_pArray[index] = src;
    3161  }
    3162 
    3163  void remove(size_t index)
    3164  {
    3165  VMA_HEAVY_ASSERT(index < m_Count);
    3166  const size_t oldCount = size();
    3167  if(index < oldCount - 1)
    3168  {
    3169  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
    3170  }
    3171  resize(oldCount - 1);
    3172  }
    3173 
    3174  void push_back(const T& src)
    3175  {
    3176  const size_t newIndex = size();
    3177  resize(newIndex + 1);
    3178  m_pArray[newIndex] = src;
    3179  }
    3180 
    3181  void pop_back()
    3182  {
    3183  VMA_HEAVY_ASSERT(m_Count > 0);
    3184  resize(size() - 1);
    3185  }
    3186 
    3187  void push_front(const T& src)
    3188  {
    3189  insert(0, src);
    3190  }
    3191 
    3192  void pop_front()
    3193  {
    3194  VMA_HEAVY_ASSERT(m_Count > 0);
    3195  remove(0);
    3196  }
    3197 
    3198  typedef T* iterator;
    3199 
    3200  iterator begin() { return m_pArray; }
    3201  iterator end() { return m_pArray + m_Count; }
    3202 
    3203 private:
    3204  AllocatorT m_Allocator;
    3205  T* m_pArray;
    3206  size_t m_Count;
    3207  size_t m_Capacity;
    3208 };
    3209 
    3210 template<typename T, typename allocatorT>
    3211 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
    3212 {
    3213  vec.insert(index, item);
    3214 }
    3215 
    3216 template<typename T, typename allocatorT>
    3217 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
    3218 {
    3219  vec.remove(index);
    3220 }
    3221 
    3222 #endif // #if VMA_USE_STL_VECTOR
    3223 
    3224 template<typename CmpLess, typename VectorT>
    3225 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
    3226 {
    3227  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    3228  vector.data(),
    3229  vector.data() + vector.size(),
    3230  value,
    3231  CmpLess()) - vector.data();
    3232  VmaVectorInsert(vector, indexToInsert, value);
    3233  return indexToInsert;
    3234 }
    3235 
    3236 template<typename CmpLess, typename VectorT>
    3237 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
    3238 {
    3239  CmpLess comparator;
    3240  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3241  vector.begin(),
    3242  vector.end(),
    3243  value,
    3244  comparator);
    3245  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
    3246  {
    3247  size_t indexToRemove = it - vector.begin();
    3248  VmaVectorRemove(vector, indexToRemove);
    3249  return true;
    3250  }
    3251  return false;
    3252 }
    3253 
    3254 template<typename CmpLess, typename VectorT>
    3255 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
    3256 {
    3257  CmpLess comparator;
    3258  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3259  vector.data(),
    3260  vector.data() + vector.size(),
    3261  value,
    3262  comparator);
    3263  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
    3264  {
    3265  return it - vector.begin();
    3266  }
    3267  else
    3268  {
    3269  return vector.size();
    3270  }
    3271 }
    3272 
    3274 // class VmaPoolAllocator
    3275 
    3276 /*
    3277 Allocator for objects of type T using a list of arrays (pools) to speed up
    3278 allocation. Number of elements that can be allocated is not bounded because
    3279 allocator can create multiple blocks.
    3280 */
    3281 template<typename T>
    3282 class VmaPoolAllocator
    3283 {
    3284  VMA_CLASS_NO_COPY(VmaPoolAllocator)
    3285 public:
    3286  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
    3287  ~VmaPoolAllocator();
    3288  void Clear();
    3289  T* Alloc();
    3290  void Free(T* ptr);
    3291 
    3292 private:
    3293  union Item
    3294  {
    3295  uint32_t NextFreeIndex;
    3296  T Value;
    3297  };
    3298 
    3299  struct ItemBlock
    3300  {
    3301  Item* pItems;
    3302  uint32_t FirstFreeIndex;
    3303  };
    3304 
    3305  const VkAllocationCallbacks* m_pAllocationCallbacks;
    3306  size_t m_ItemsPerBlock;
    3307  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
    3308 
    3309  ItemBlock& CreateNewBlock();
    3310 };
    3311 
    3312 template<typename T>
    3313 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
    3314  m_pAllocationCallbacks(pAllocationCallbacks),
    3315  m_ItemsPerBlock(itemsPerBlock),
    3316  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
    3317 {
    3318  VMA_ASSERT(itemsPerBlock > 0);
    3319 }
    3320 
    3321 template<typename T>
    3322 VmaPoolAllocator<T>::~VmaPoolAllocator()
    3323 {
    3324  Clear();
    3325 }
    3326 
    3327 template<typename T>
    3328 void VmaPoolAllocator<T>::Clear()
    3329 {
    3330  for(size_t i = m_ItemBlocks.size(); i--; )
    3331  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
    3332  m_ItemBlocks.clear();
    3333 }
    3334 
    3335 template<typename T>
    3336 T* VmaPoolAllocator<T>::Alloc()
    3337 {
    3338  for(size_t i = m_ItemBlocks.size(); i--; )
    3339  {
    3340  ItemBlock& block = m_ItemBlocks[i];
    3341  // This block has some free items: Use first one.
    3342  if(block.FirstFreeIndex != UINT32_MAX)
    3343  {
    3344  Item* const pItem = &block.pItems[block.FirstFreeIndex];
    3345  block.FirstFreeIndex = pItem->NextFreeIndex;
    3346  return &pItem->Value;
    3347  }
    3348  }
    3349 
    3350  // No block has free item: Create new one and use it.
    3351  ItemBlock& newBlock = CreateNewBlock();
    3352  Item* const pItem = &newBlock.pItems[0];
    3353  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
    3354  return &pItem->Value;
    3355 }
    3356 
    3357 template<typename T>
    3358 void VmaPoolAllocator<T>::Free(T* ptr)
    3359 {
    3360  // Search all memory blocks to find ptr.
    3361  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
    3362  {
    3363  ItemBlock& block = m_ItemBlocks[i];
    3364 
    3365  // Casting to union.
    3366  Item* pItemPtr;
    3367  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
    3368 
    3369  // Check if pItemPtr is in address range of this block.
    3370  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
    3371  {
    3372  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
    3373  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
    3374  block.FirstFreeIndex = index;
    3375  return;
    3376  }
    3377  }
    3378  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
    3379 }
    3380 
    3381 template<typename T>
    3382 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
    3383 {
    3384  ItemBlock newBlock = {
    3385  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
    3386 
    3387  m_ItemBlocks.push_back(newBlock);
    3388 
    3389  // Setup singly-linked list of all free items in this block.
    3390  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
    3391  newBlock.pItems[i].NextFreeIndex = i + 1;
    3392  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
    3393  return m_ItemBlocks.back();
    3394 }
    3395 
    3397 // class VmaRawList, VmaList
    3398 
    3399 #if VMA_USE_STL_LIST
    3400 
    3401 #define VmaList std::list
    3402 
    3403 #else // #if VMA_USE_STL_LIST
    3404 
    3405 template<typename T>
    3406 struct VmaListItem
    3407 {
    3408  VmaListItem* pPrev;
    3409  VmaListItem* pNext;
    3410  T Value;
    3411 };
    3412 
    3413 // Doubly linked list.
    3414 template<typename T>
    3415 class VmaRawList
    3416 {
    3417  VMA_CLASS_NO_COPY(VmaRawList)
    3418 public:
    3419  typedef VmaListItem<T> ItemType;
    3420 
    3421  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
    3422  ~VmaRawList();
    3423  void Clear();
    3424 
    3425  size_t GetCount() const { return m_Count; }
    3426  bool IsEmpty() const { return m_Count == 0; }
    3427 
    3428  ItemType* Front() { return m_pFront; }
    3429  const ItemType* Front() const { return m_pFront; }
    3430  ItemType* Back() { return m_pBack; }
    3431  const ItemType* Back() const { return m_pBack; }
    3432 
    3433  ItemType* PushBack();
    3434  ItemType* PushFront();
    3435  ItemType* PushBack(const T& value);
    3436  ItemType* PushFront(const T& value);
    3437  void PopBack();
    3438  void PopFront();
    3439 
    3440  // Item can be null - it means PushBack.
    3441  ItemType* InsertBefore(ItemType* pItem);
    3442  // Item can be null - it means PushFront.
    3443  ItemType* InsertAfter(ItemType* pItem);
    3444 
    3445  ItemType* InsertBefore(ItemType* pItem, const T& value);
    3446  ItemType* InsertAfter(ItemType* pItem, const T& value);
    3447 
    3448  void Remove(ItemType* pItem);
    3449 
    3450 private:
    3451  const VkAllocationCallbacks* const m_pAllocationCallbacks;
    3452  VmaPoolAllocator<ItemType> m_ItemAllocator;
    3453  ItemType* m_pFront;
    3454  ItemType* m_pBack;
    3455  size_t m_Count;
    3456 };
    3457 
    3458 template<typename T>
    3459 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
    3460  m_pAllocationCallbacks(pAllocationCallbacks),
    3461  m_ItemAllocator(pAllocationCallbacks, 128),
    3462  m_pFront(VMA_NULL),
    3463  m_pBack(VMA_NULL),
    3464  m_Count(0)
    3465 {
    3466 }
    3467 
    3468 template<typename T>
    3469 VmaRawList<T>::~VmaRawList()
    3470 {
    3471  // Intentionally not calling Clear, because that would be unnecessary
    3472  // computations to return all items to m_ItemAllocator as free.
    3473 }
    3474 
    3475 template<typename T>
    3476 void VmaRawList<T>::Clear()
    3477 {
    3478  if(IsEmpty() == false)
    3479  {
    3480  ItemType* pItem = m_pBack;
    3481  while(pItem != VMA_NULL)
    3482  {
    3483  ItemType* const pPrevItem = pItem->pPrev;
    3484  m_ItemAllocator.Free(pItem);
    3485  pItem = pPrevItem;
    3486  }
    3487  m_pFront = VMA_NULL;
    3488  m_pBack = VMA_NULL;
    3489  m_Count = 0;
    3490  }
    3491 }
    3492 
    3493 template<typename T>
    3494 VmaListItem<T>* VmaRawList<T>::PushBack()
    3495 {
    3496  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3497  pNewItem->pNext = VMA_NULL;
    3498  if(IsEmpty())
    3499  {
    3500  pNewItem->pPrev = VMA_NULL;
    3501  m_pFront = pNewItem;
    3502  m_pBack = pNewItem;
    3503  m_Count = 1;
    3504  }
    3505  else
    3506  {
    3507  pNewItem->pPrev = m_pBack;
    3508  m_pBack->pNext = pNewItem;
    3509  m_pBack = pNewItem;
    3510  ++m_Count;
    3511  }
    3512  return pNewItem;
    3513 }
    3514 
    3515 template<typename T>
    3516 VmaListItem<T>* VmaRawList<T>::PushFront()
    3517 {
    3518  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3519  pNewItem->pPrev = VMA_NULL;
    3520  if(IsEmpty())
    3521  {
    3522  pNewItem->pNext = VMA_NULL;
    3523  m_pFront = pNewItem;
    3524  m_pBack = pNewItem;
    3525  m_Count = 1;
    3526  }
    3527  else
    3528  {
    3529  pNewItem->pNext = m_pFront;
    3530  m_pFront->pPrev = pNewItem;
    3531  m_pFront = pNewItem;
    3532  ++m_Count;
    3533  }
    3534  return pNewItem;
    3535 }
    3536 
    3537 template<typename T>
    3538 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
    3539 {
    3540  ItemType* const pNewItem = PushBack();
    3541  pNewItem->Value = value;
    3542  return pNewItem;
    3543 }
    3544 
    3545 template<typename T>
    3546 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
    3547 {
    3548  ItemType* const pNewItem = PushFront();
    3549  pNewItem->Value = value;
    3550  return pNewItem;
    3551 }
    3552 
    3553 template<typename T>
    3554 void VmaRawList<T>::PopBack()
    3555 {
    3556  VMA_HEAVY_ASSERT(m_Count > 0);
    3557  ItemType* const pBackItem = m_pBack;
    3558  ItemType* const pPrevItem = pBackItem->pPrev;
    3559  if(pPrevItem != VMA_NULL)
    3560  {
    3561  pPrevItem->pNext = VMA_NULL;
    3562  }
    3563  m_pBack = pPrevItem;
    3564  m_ItemAllocator.Free(pBackItem);
    3565  --m_Count;
    3566 }
    3567 
    3568 template<typename T>
    3569 void VmaRawList<T>::PopFront()
    3570 {
    3571  VMA_HEAVY_ASSERT(m_Count > 0);
    3572  ItemType* const pFrontItem = m_pFront;
    3573  ItemType* const pNextItem = pFrontItem->pNext;
    3574  if(pNextItem != VMA_NULL)
    3575  {
    3576  pNextItem->pPrev = VMA_NULL;
    3577  }
    3578  m_pFront = pNextItem;
    3579  m_ItemAllocator.Free(pFrontItem);
    3580  --m_Count;
    3581 }
    3582 
    3583 template<typename T>
    3584 void VmaRawList<T>::Remove(ItemType* pItem)
    3585 {
    3586  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
    3587  VMA_HEAVY_ASSERT(m_Count > 0);
    3588 
    3589  if(pItem->pPrev != VMA_NULL)
    3590  {
    3591  pItem->pPrev->pNext = pItem->pNext;
    3592  }
    3593  else
    3594  {
    3595  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3596  m_pFront = pItem->pNext;
    3597  }
    3598 
    3599  if(pItem->pNext != VMA_NULL)
    3600  {
    3601  pItem->pNext->pPrev = pItem->pPrev;
    3602  }
    3603  else
    3604  {
    3605  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3606  m_pBack = pItem->pPrev;
    3607  }
    3608 
    3609  m_ItemAllocator.Free(pItem);
    3610  --m_Count;
    3611 }
    3612 
    3613 template<typename T>
    3614 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
    3615 {
    3616  if(pItem != VMA_NULL)
    3617  {
    3618  ItemType* const prevItem = pItem->pPrev;
    3619  ItemType* const newItem = m_ItemAllocator.Alloc();
    3620  newItem->pPrev = prevItem;
    3621  newItem->pNext = pItem;
    3622  pItem->pPrev = newItem;
    3623  if(prevItem != VMA_NULL)
    3624  {
    3625  prevItem->pNext = newItem;
    3626  }
    3627  else
    3628  {
    3629  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3630  m_pFront = newItem;
    3631  }
    3632  ++m_Count;
    3633  return newItem;
    3634  }
    3635  else
    3636  return PushBack();
    3637 }
    3638 
    3639 template<typename T>
    3640 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
    3641 {
    3642  if(pItem != VMA_NULL)
    3643  {
    3644  ItemType* const nextItem = pItem->pNext;
    3645  ItemType* const newItem = m_ItemAllocator.Alloc();
    3646  newItem->pNext = nextItem;
    3647  newItem->pPrev = pItem;
    3648  pItem->pNext = newItem;
    3649  if(nextItem != VMA_NULL)
    3650  {
    3651  nextItem->pPrev = newItem;
    3652  }
    3653  else
    3654  {
    3655  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3656  m_pBack = newItem;
    3657  }
    3658  ++m_Count;
    3659  return newItem;
    3660  }
    3661  else
    3662  return PushFront();
    3663 }
    3664 
    3665 template<typename T>
    3666 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
    3667 {
    3668  ItemType* const newItem = InsertBefore(pItem);
    3669  newItem->Value = value;
    3670  return newItem;
    3671 }
    3672 
    3673 template<typename T>
    3674 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
    3675 {
    3676  ItemType* const newItem = InsertAfter(pItem);
    3677  newItem->Value = value;
    3678  return newItem;
    3679 }
    3680 
    3681 template<typename T, typename AllocatorT>
    3682 class VmaList
    3683 {
    3684  VMA_CLASS_NO_COPY(VmaList)
    3685 public:
    3686  class iterator
    3687  {
    3688  public:
    3689  iterator() :
    3690  m_pList(VMA_NULL),
    3691  m_pItem(VMA_NULL)
    3692  {
    3693  }
    3694 
    3695  T& operator*() const
    3696  {
    3697  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3698  return m_pItem->Value;
    3699  }
    3700  T* operator->() const
    3701  {
    3702  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3703  return &m_pItem->Value;
    3704  }
    3705 
    3706  iterator& operator++()
    3707  {
    3708  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3709  m_pItem = m_pItem->pNext;
    3710  return *this;
    3711  }
    3712  iterator& operator--()
    3713  {
    3714  if(m_pItem != VMA_NULL)
    3715  {
    3716  m_pItem = m_pItem->pPrev;
    3717  }
    3718  else
    3719  {
    3720  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3721  m_pItem = m_pList->Back();
    3722  }
    3723  return *this;
    3724  }
    3725 
    3726  iterator operator++(int)
    3727  {
    3728  iterator result = *this;
    3729  ++*this;
    3730  return result;
    3731  }
    3732  iterator operator--(int)
    3733  {
    3734  iterator result = *this;
    3735  --*this;
    3736  return result;
    3737  }
    3738 
    3739  bool operator==(const iterator& rhs) const
    3740  {
    3741  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3742  return m_pItem == rhs.m_pItem;
    3743  }
    3744  bool operator!=(const iterator& rhs) const
    3745  {
    3746  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3747  return m_pItem != rhs.m_pItem;
    3748  }
    3749 
    3750  private:
    3751  VmaRawList<T>* m_pList;
    3752  VmaListItem<T>* m_pItem;
    3753 
    3754  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
    3755  m_pList(pList),
    3756  m_pItem(pItem)
    3757  {
    3758  }
    3759 
    3760  friend class VmaList<T, AllocatorT>;
    3761  };
    3762 
    3763  class const_iterator
    3764  {
    3765  public:
    3766  const_iterator() :
    3767  m_pList(VMA_NULL),
    3768  m_pItem(VMA_NULL)
    3769  {
    3770  }
    3771 
    3772  const_iterator(const iterator& src) :
    3773  m_pList(src.m_pList),
    3774  m_pItem(src.m_pItem)
    3775  {
    3776  }
    3777 
    3778  const T& operator*() const
    3779  {
    3780  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3781  return m_pItem->Value;
    3782  }
    3783  const T* operator->() const
    3784  {
    3785  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3786  return &m_pItem->Value;
    3787  }
    3788 
    3789  const_iterator& operator++()
    3790  {
    3791  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3792  m_pItem = m_pItem->pNext;
    3793  return *this;
    3794  }
    3795  const_iterator& operator--()
    3796  {
    3797  if(m_pItem != VMA_NULL)
    3798  {
    3799  m_pItem = m_pItem->pPrev;
    3800  }
    3801  else
    3802  {
    3803  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3804  m_pItem = m_pList->Back();
    3805  }
    3806  return *this;
    3807  }
    3808 
    3809  const_iterator operator++(int)
    3810  {
    3811  const_iterator result = *this;
    3812  ++*this;
    3813  return result;
    3814  }
    3815  const_iterator operator--(int)
    3816  {
    3817  const_iterator result = *this;
    3818  --*this;
    3819  return result;
    3820  }
    3821 
    3822  bool operator==(const const_iterator& rhs) const
    3823  {
    3824  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3825  return m_pItem == rhs.m_pItem;
    3826  }
    3827  bool operator!=(const const_iterator& rhs) const
    3828  {
    3829  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3830  return m_pItem != rhs.m_pItem;
    3831  }
    3832 
    3833  private:
    3834  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
    3835  m_pList(pList),
    3836  m_pItem(pItem)
    3837  {
    3838  }
    3839 
    3840  const VmaRawList<T>* m_pList;
    3841  const VmaListItem<T>* m_pItem;
    3842 
    3843  friend class VmaList<T, AllocatorT>;
    3844  };
    3845 
    3846  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
    3847 
    3848  bool empty() const { return m_RawList.IsEmpty(); }
    3849  size_t size() const { return m_RawList.GetCount(); }
    3850 
    3851  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
    3852  iterator end() { return iterator(&m_RawList, VMA_NULL); }
    3853 
    3854  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
    3855  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
    3856 
    3857  void clear() { m_RawList.Clear(); }
    3858  void push_back(const T& value) { m_RawList.PushBack(value); }
    3859  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
    3860  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
    3861 
    3862 private:
    3863  VmaRawList<T> m_RawList;
    3864 };
    3865 
    3866 #endif // #if VMA_USE_STL_LIST
    3867 
    3869 // class VmaMap
    3870 
    3871 // Unused in this version.
    3872 #if 0
    3873 
    3874 #if VMA_USE_STL_UNORDERED_MAP
    3875 
    3876 #define VmaPair std::pair
    3877 
    3878 #define VMA_MAP_TYPE(KeyT, ValueT) \
    3879  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
    3880 
    3881 #else // #if VMA_USE_STL_UNORDERED_MAP
    3882 
    3883 template<typename T1, typename T2>
    3884 struct VmaPair
    3885 {
    3886  T1 first;
    3887  T2 second;
    3888 
    3889  VmaPair() : first(), second() { }
    3890  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
    3891 };
    3892 
    3893 /* Class compatible with subset of interface of std::unordered_map.
    3894 KeyT, ValueT must be POD because they will be stored in VmaVector.
    3895 */
    3896 template<typename KeyT, typename ValueT>
    3897 class VmaMap
    3898 {
    3899 public:
    3900  typedef VmaPair<KeyT, ValueT> PairType;
    3901  typedef PairType* iterator;
    3902 
    3903  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
    3904 
    3905  iterator begin() { return m_Vector.begin(); }
    3906  iterator end() { return m_Vector.end(); }
    3907 
    3908  void insert(const PairType& pair);
    3909  iterator find(const KeyT& key);
    3910  void erase(iterator it);
    3911 
    3912 private:
    3913  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
    3914 };
    3915 
    3916 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
    3917 
    3918 template<typename FirstT, typename SecondT>
    3919 struct VmaPairFirstLess
    3920 {
    3921  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
    3922  {
    3923  return lhs.first < rhs.first;
    3924  }
    3925  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
    3926  {
    3927  return lhs.first < rhsFirst;
    3928  }
    3929 };
    3930 
    3931 template<typename KeyT, typename ValueT>
    3932 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
    3933 {
    3934  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    3935  m_Vector.data(),
    3936  m_Vector.data() + m_Vector.size(),
    3937  pair,
    3938  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
    3939  VmaVectorInsert(m_Vector, indexToInsert, pair);
    3940 }
    3941 
    3942 template<typename KeyT, typename ValueT>
    3943 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
    3944 {
    3945  PairType* it = VmaBinaryFindFirstNotLess(
    3946  m_Vector.data(),
    3947  m_Vector.data() + m_Vector.size(),
    3948  key,
    3949  VmaPairFirstLess<KeyT, ValueT>());
    3950  if((it != m_Vector.end()) && (it->first == key))
    3951  {
    3952  return it;
    3953  }
    3954  else
    3955  {
    3956  return m_Vector.end();
    3957  }
    3958 }
    3959 
    3960 template<typename KeyT, typename ValueT>
    3961 void VmaMap<KeyT, ValueT>::erase(iterator it)
    3962 {
    3963  VmaVectorRemove(m_Vector, it - m_Vector.begin());
    3964 }
    3965 
    3966 #endif // #if VMA_USE_STL_UNORDERED_MAP
    3967 
    3968 #endif // #if 0
    3969 
    3971 
    3972 class VmaDeviceMemoryBlock;
    3973 
    3974 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
    3975 
    3976 struct VmaAllocation_T
    3977 {
    3978  VMA_CLASS_NO_COPY(VmaAllocation_T)
    3979 private:
    3980  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
    3981 
    3982  enum FLAGS
    3983  {
    3984  FLAG_USER_DATA_STRING = 0x01,
    3985  };
    3986 
    3987 public:
    3988  enum ALLOCATION_TYPE
    3989  {
    3990  ALLOCATION_TYPE_NONE,
    3991  ALLOCATION_TYPE_BLOCK,
    3992  ALLOCATION_TYPE_DEDICATED,
    3993  };
    3994 
    3995  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
    3996  m_Alignment(1),
    3997  m_Size(0),
    3998  m_pUserData(VMA_NULL),
    3999  m_LastUseFrameIndex(currentFrameIndex),
    4000  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
    4001  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
    4002  m_MapCount(0),
    4003  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
    4004  {
    4005 #if VMA_STATS_STRING_ENABLED
    4006  m_CreationFrameIndex = currentFrameIndex;
    4007  m_BufferImageUsage = 0;
    4008 #endif
    4009  }
    4010 
    4011  ~VmaAllocation_T()
    4012  {
    4013  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
    4014 
    4015  // Check if owned string was freed.
    4016  VMA_ASSERT(m_pUserData == VMA_NULL);
    4017  }
    4018 
    4019  void InitBlockAllocation(
    4020  VmaPool hPool,
    4021  VmaDeviceMemoryBlock* block,
    4022  VkDeviceSize offset,
    4023  VkDeviceSize alignment,
    4024  VkDeviceSize size,
    4025  VmaSuballocationType suballocationType,
    4026  bool mapped,
    4027  bool canBecomeLost)
    4028  {
    4029  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4030  VMA_ASSERT(block != VMA_NULL);
    4031  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4032  m_Alignment = alignment;
    4033  m_Size = size;
    4034  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4035  m_SuballocationType = (uint8_t)suballocationType;
    4036  m_BlockAllocation.m_hPool = hPool;
    4037  m_BlockAllocation.m_Block = block;
    4038  m_BlockAllocation.m_Offset = offset;
    4039  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
    4040  }
    4041 
    4042  void InitLost()
    4043  {
    4044  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4045  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
    4046  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4047  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
    4048  m_BlockAllocation.m_Block = VMA_NULL;
    4049  m_BlockAllocation.m_Offset = 0;
    4050  m_BlockAllocation.m_CanBecomeLost = true;
    4051  }
    4052 
    4053  void ChangeBlockAllocation(
    4054  VmaAllocator hAllocator,
    4055  VmaDeviceMemoryBlock* block,
    4056  VkDeviceSize offset);
    4057 
    4058  // pMappedData not null means allocation is created with MAPPED flag.
    4059  void InitDedicatedAllocation(
    4060  uint32_t memoryTypeIndex,
    4061  VkDeviceMemory hMemory,
    4062  VmaSuballocationType suballocationType,
    4063  void* pMappedData,
    4064  VkDeviceSize size)
    4065  {
    4066  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4067  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
    4068  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
    4069  m_Alignment = 0;
    4070  m_Size = size;
    4071  m_SuballocationType = (uint8_t)suballocationType;
    4072  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4073  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
    4074  m_DedicatedAllocation.m_hMemory = hMemory;
    4075  m_DedicatedAllocation.m_pMappedData = pMappedData;
    4076  }
    4077 
    4078  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
    4079  VkDeviceSize GetAlignment() const { return m_Alignment; }
    4080  VkDeviceSize GetSize() const { return m_Size; }
    4081  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
    4082  void* GetUserData() const { return m_pUserData; }
    4083  void SetUserData(VmaAllocator hAllocator, void* pUserData);
    4084  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
    4085 
    4086  VmaDeviceMemoryBlock* GetBlock() const
    4087  {
    4088  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    4089  return m_BlockAllocation.m_Block;
    4090  }
    4091  VkDeviceSize GetOffset() const;
    4092  VkDeviceMemory GetMemory() const;
    4093  uint32_t GetMemoryTypeIndex() const;
    4094  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
    4095  void* GetMappedData() const;
    4096  bool CanBecomeLost() const;
    4097  VmaPool GetPool() const;
    4098 
    4099  uint32_t GetLastUseFrameIndex() const
    4100  {
    4101  return m_LastUseFrameIndex.load();
    4102  }
    4103  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
    4104  {
    4105  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
    4106  }
    4107  /*
    4108  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
    4109  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
    4110  - Else, returns false.
    4111 
    4112  If hAllocation is already lost, assert - you should not call it then.
    4113  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
    4114  */
    4115  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4116 
    4117  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
    4118  {
    4119  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
    4120  outInfo.blockCount = 1;
    4121  outInfo.allocationCount = 1;
    4122  outInfo.unusedRangeCount = 0;
    4123  outInfo.usedBytes = m_Size;
    4124  outInfo.unusedBytes = 0;
    4125  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
    4126  outInfo.unusedRangeSizeMin = UINT64_MAX;
    4127  outInfo.unusedRangeSizeMax = 0;
    4128  }
    4129 
    4130  void BlockAllocMap();
    4131  void BlockAllocUnmap();
    4132  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
    4133  void DedicatedAllocUnmap(VmaAllocator hAllocator);
    4134 
    4135 #if VMA_STATS_STRING_ENABLED
    4136  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
    4137  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
    4138 
    4139  void InitBufferImageUsage(uint32_t bufferImageUsage)
    4140  {
    4141  VMA_ASSERT(m_BufferImageUsage == 0);
    4142  m_BufferImageUsage = bufferImageUsage;
    4143  }
    4144 
    4145  void PrintParameters(class VmaJsonWriter& json) const;
    4146 #endif
    4147 
    4148 private:
    4149  VkDeviceSize m_Alignment;
    4150  VkDeviceSize m_Size;
    4151  void* m_pUserData;
    4152  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
    4153  uint8_t m_Type; // ALLOCATION_TYPE
    4154  uint8_t m_SuballocationType; // VmaSuballocationType
    4155  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
    4156  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
    4157  uint8_t m_MapCount;
    4158  uint8_t m_Flags; // enum FLAGS
    4159 
    4160  // Allocation out of VmaDeviceMemoryBlock.
    4161  struct BlockAllocation
    4162  {
    4163  VmaPool m_hPool; // Null if belongs to general memory.
    4164  VmaDeviceMemoryBlock* m_Block;
    4165  VkDeviceSize m_Offset;
    4166  bool m_CanBecomeLost;
    4167  };
    4168 
    4169  // Allocation for an object that has its own private VkDeviceMemory.
    4170  struct DedicatedAllocation
    4171  {
    4172  uint32_t m_MemoryTypeIndex;
    4173  VkDeviceMemory m_hMemory;
    4174  void* m_pMappedData; // Not null means memory is mapped.
    4175  };
    4176 
    4177  union
    4178  {
    4179  // Allocation out of VmaDeviceMemoryBlock.
    4180  BlockAllocation m_BlockAllocation;
    4181  // Allocation for an object that has its own private VkDeviceMemory.
    4182  DedicatedAllocation m_DedicatedAllocation;
    4183  };
    4184 
    4185 #if VMA_STATS_STRING_ENABLED
    4186  uint32_t m_CreationFrameIndex;
    4187  uint32_t m_BufferImageUsage; // 0 if unknown.
    4188 #endif
    4189 
    4190  void FreeUserDataString(VmaAllocator hAllocator);
    4191 };
    4192 
    4193 /*
    4194 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
    4195 allocated memory block or free.
    4196 */
    4197 struct VmaSuballocation
    4198 {
    4199  VkDeviceSize offset;
    4200  VkDeviceSize size;
    4201  VmaAllocation hAllocation;
    4202  VmaSuballocationType type;
    4203 };
    4204 
    4205 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
    4206 
    4207 // Cost of one additional allocation lost, as equivalent in bytes.
    4208 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
    4209 
    4210 /*
    4211 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
    4212 
    4213 If canMakeOtherLost was false:
    4214 - item points to a FREE suballocation.
    4215 - itemsToMakeLostCount is 0.
    4216 
    4217 If canMakeOtherLost was true:
    4218 - item points to first of sequence of suballocations, which are either FREE,
    4219  or point to VmaAllocations that can become lost.
    4220 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
    4221  the requested allocation to succeed.
    4222 */
    4223 struct VmaAllocationRequest
    4224 {
    4225  VkDeviceSize offset;
    4226  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
    4227  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
    4228  VmaSuballocationList::iterator item;
    4229  size_t itemsToMakeLostCount;
    4230 
    4231  VkDeviceSize CalcCost() const
    4232  {
    4233  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
    4234  }
    4235 };
    4236 
    4237 /*
    4238 Data structure used for bookkeeping of allocations and unused ranges of memory
    4239 in a single VkDeviceMemory block.
    4240 */
    4241 class VmaBlockMetadata
    4242 {
    4243  VMA_CLASS_NO_COPY(VmaBlockMetadata)
    4244 public:
    4245  VmaBlockMetadata(VmaAllocator hAllocator);
    4246  ~VmaBlockMetadata();
    4247  void Init(VkDeviceSize size);
    4248 
    4249  // Validates all data structures inside this object. If not valid, returns false.
    4250  bool Validate() const;
    4251  VkDeviceSize GetSize() const { return m_Size; }
    4252  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
    4253  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
    4254  VkDeviceSize GetUnusedRangeSizeMax() const;
    4255  // Returns true if this block is empty - contains only single free suballocation.
    4256  bool IsEmpty() const;
    4257 
    4258  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
    4259  void AddPoolStats(VmaPoolStats& inoutStats) const;
    4260 
    4261 #if VMA_STATS_STRING_ENABLED
    4262  void PrintDetailedMap(class VmaJsonWriter& json) const;
    4263 #endif
    4264 
    4265  // Tries to find a place for suballocation with given parameters inside this block.
    4266  // If succeeded, fills pAllocationRequest and returns true.
    4267  // If failed, returns false.
    4268  bool CreateAllocationRequest(
    4269  uint32_t currentFrameIndex,
    4270  uint32_t frameInUseCount,
    4271  VkDeviceSize bufferImageGranularity,
    4272  VkDeviceSize allocSize,
    4273  VkDeviceSize allocAlignment,
    4274  VmaSuballocationType allocType,
    4275  bool canMakeOtherLost,
    4276  VmaAllocationRequest* pAllocationRequest);
    4277 
    4278  bool MakeRequestedAllocationsLost(
    4279  uint32_t currentFrameIndex,
    4280  uint32_t frameInUseCount,
    4281  VmaAllocationRequest* pAllocationRequest);
    4282 
    4283  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4284 
    4285  VkResult CheckCorruption(const void* pBlockData);
    4286 
    4287  // Makes actual allocation based on request. Request must already be checked and valid.
    4288  void Alloc(
    4289  const VmaAllocationRequest& request,
    4290  VmaSuballocationType type,
    4291  VkDeviceSize allocSize,
    4292  VmaAllocation hAllocation);
    4293 
    4294  // Frees suballocation assigned to given memory region.
    4295  void Free(const VmaAllocation allocation);
    4296  void FreeAtOffset(VkDeviceSize offset);
    4297 
    4298 private:
    4299  VkDeviceSize m_Size;
    4300  uint32_t m_FreeCount;
    4301  VkDeviceSize m_SumFreeSize;
    4302  VmaSuballocationList m_Suballocations;
    4303  // Suballocations that are free and have size greater than certain threshold.
    4304  // Sorted by size, ascending.
    4305  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
    4306 
    4307  bool ValidateFreeSuballocationList() const;
    4308 
    4309  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
    4310  // If yes, fills pOffset and returns true. If no, returns false.
    4311  bool CheckAllocation(
    4312  uint32_t currentFrameIndex,
    4313  uint32_t frameInUseCount,
    4314  VkDeviceSize bufferImageGranularity,
    4315  VkDeviceSize allocSize,
    4316  VkDeviceSize allocAlignment,
    4317  VmaSuballocationType allocType,
    4318  VmaSuballocationList::const_iterator suballocItem,
    4319  bool canMakeOtherLost,
    4320  VkDeviceSize* pOffset,
    4321  size_t* itemsToMakeLostCount,
    4322  VkDeviceSize* pSumFreeSize,
    4323  VkDeviceSize* pSumItemSize) const;
    4324  // Given free suballocation, it merges it with following one, which must also be free.
    4325  void MergeFreeWithNext(VmaSuballocationList::iterator item);
    4326  // Releases given suballocation, making it free.
    4327  // Merges it with adjacent free suballocations if applicable.
    4328  // Returns iterator to new free suballocation at this place.
    4329  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
    4330  // Given free suballocation, it inserts it into sorted list of
    4331  // m_FreeSuballocationsBySize if it's suitable.
    4332  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
    4333  // Given free suballocation, it removes it from sorted list of
    4334  // m_FreeSuballocationsBySize if it's suitable.
    4335  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
    4336 };
    4337 
    4338 /*
    4339 Represents a single block of device memory (`VkDeviceMemory`) with all the
    4340 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
    4341 
    4342 Thread-safety: This class must be externally synchronized.
    4343 */
    4344 class VmaDeviceMemoryBlock
    4345 {
    4346  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
    4347 public:
    4348  VmaBlockMetadata m_Metadata;
    4349 
    4350  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
    4351 
    4352  ~VmaDeviceMemoryBlock()
    4353  {
    4354  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
    4355  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    4356  }
    4357 
    4358  // Always call after construction.
    4359  void Init(
    4360  uint32_t newMemoryTypeIndex,
    4361  VkDeviceMemory newMemory,
    4362  VkDeviceSize newSize,
    4363  uint32_t id);
    4364  // Always call before destruction.
    4365  void Destroy(VmaAllocator allocator);
    4366 
    4367  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
    4368  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4369  uint32_t GetId() const { return m_Id; }
    4370  void* GetMappedData() const { return m_pMappedData; }
    4371 
    4372  // Validates all data structures inside this object. If not valid, returns false.
    4373  bool Validate() const;
    4374 
    4375  VkResult CheckCorruption(VmaAllocator hAllocator);
    4376 
    4377  // ppData can be null.
    4378  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
    4379  void Unmap(VmaAllocator hAllocator, uint32_t count);
    4380 
    4381  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4382  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4383 
    4384  VkResult BindBufferMemory(
    4385  const VmaAllocator hAllocator,
    4386  const VmaAllocation hAllocation,
    4387  VkBuffer hBuffer);
    4388  VkResult BindImageMemory(
    4389  const VmaAllocator hAllocator,
    4390  const VmaAllocation hAllocation,
    4391  VkImage hImage);
    4392 
    4393 private:
    4394  uint32_t m_MemoryTypeIndex;
    4395  uint32_t m_Id;
    4396  VkDeviceMemory m_hMemory;
    4397 
    4398  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
    4399  // Also protects m_MapCount, m_pMappedData.
    4400  VMA_MUTEX m_Mutex;
    4401  uint32_t m_MapCount;
    4402  void* m_pMappedData;
    4403 };
    4404 
    4405 struct VmaPointerLess
    4406 {
    4407  bool operator()(const void* lhs, const void* rhs) const
    4408  {
    4409  return lhs < rhs;
    4410  }
    4411 };
    4412 
    4413 class VmaDefragmentator;
    4414 
    4415 /*
    4416 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
    4417 Vulkan memory type.
    4418 
    4419 Synchronized internally with a mutex.
    4420 */
    4421 struct VmaBlockVector
    4422 {
    4423  VMA_CLASS_NO_COPY(VmaBlockVector)
    4424 public:
    4425  VmaBlockVector(
    4426  VmaAllocator hAllocator,
    4427  uint32_t memoryTypeIndex,
    4428  VkDeviceSize preferredBlockSize,
    4429  size_t minBlockCount,
    4430  size_t maxBlockCount,
    4431  VkDeviceSize bufferImageGranularity,
    4432  uint32_t frameInUseCount,
    4433  bool isCustomPool);
    4434  ~VmaBlockVector();
    4435 
    4436  VkResult CreateMinBlocks();
    4437 
    4438  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4439  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
    4440  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
    4441  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
    4442 
    4443  void GetPoolStats(VmaPoolStats* pStats);
    4444 
    4445  bool IsEmpty() const { return m_Blocks.empty(); }
    4446  bool IsCorruptionDetectionEnabled() const;
    4447 
    4448  VkResult Allocate(
    4449  VmaPool hCurrentPool,
    4450  uint32_t currentFrameIndex,
    4451  VkDeviceSize size,
    4452  VkDeviceSize alignment,
    4453  const VmaAllocationCreateInfo& createInfo,
    4454  VmaSuballocationType suballocType,
    4455  VmaAllocation* pAllocation);
    4456 
    4457  void Free(
    4458  VmaAllocation hAllocation);
    4459 
    4460  // Adds statistics of this BlockVector to pStats.
    4461  void AddStats(VmaStats* pStats);
    4462 
    4463 #if VMA_STATS_STRING_ENABLED
    4464  void PrintDetailedMap(class VmaJsonWriter& json);
    4465 #endif
    4466 
    4467  void MakePoolAllocationsLost(
    4468  uint32_t currentFrameIndex,
    4469  size_t* pLostAllocationCount);
    4470  VkResult CheckCorruption();
    4471 
    4472  VmaDefragmentator* EnsureDefragmentator(
    4473  VmaAllocator hAllocator,
    4474  uint32_t currentFrameIndex);
    4475 
    4476  VkResult Defragment(
    4477  VmaDefragmentationStats* pDefragmentationStats,
    4478  VkDeviceSize& maxBytesToMove,
    4479  uint32_t& maxAllocationsToMove);
    4480 
    4481  void DestroyDefragmentator();
    4482 
    4483 private:
    4484  friend class VmaDefragmentator;
    4485 
    4486  const VmaAllocator m_hAllocator;
    4487  const uint32_t m_MemoryTypeIndex;
    4488  const VkDeviceSize m_PreferredBlockSize;
    4489  const size_t m_MinBlockCount;
    4490  const size_t m_MaxBlockCount;
    4491  const VkDeviceSize m_BufferImageGranularity;
    4492  const uint32_t m_FrameInUseCount;
    4493  const bool m_IsCustomPool;
    4494  VMA_MUTEX m_Mutex;
    4495  // Incrementally sorted by sumFreeSize, ascending.
    4496  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
    4497  /* There can be at most one allocation that is completely empty - a
    4498  hysteresis to avoid pessimistic case of alternating creation and destruction
    4499  of a VkDeviceMemory. */
    4500  bool m_HasEmptyBlock;
    4501  VmaDefragmentator* m_pDefragmentator;
    4502  uint32_t m_NextBlockId;
    4503 
    4504  VkDeviceSize CalcMaxBlockSize() const;
    4505 
    4506  // Finds and removes given block from vector.
    4507  void Remove(VmaDeviceMemoryBlock* pBlock);
    4508 
    4509  // Performs single step in sorting m_Blocks. They may not be fully sorted
    4510  // after this call.
    4511  void IncrementallySortBlocks();
    4512 
    4513  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
    4514 };
    4515 
    4516 struct VmaPool_T
    4517 {
    4518  VMA_CLASS_NO_COPY(VmaPool_T)
    4519 public:
    4520  VmaBlockVector m_BlockVector;
    4521 
    4522  VmaPool_T(
    4523  VmaAllocator hAllocator,
    4524  const VmaPoolCreateInfo& createInfo);
    4525  ~VmaPool_T();
    4526 
    4527  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
    4528  uint32_t GetId() const { return m_Id; }
    4529  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
    4530 
    4531 #if VMA_STATS_STRING_ENABLED
    4532  //void PrintDetailedMap(class VmaStringBuilder& sb);
    4533 #endif
    4534 
    4535 private:
    4536  uint32_t m_Id;
    4537 };
    4538 
    4539 class VmaDefragmentator
    4540 {
    4541  VMA_CLASS_NO_COPY(VmaDefragmentator)
    4542 private:
    4543  const VmaAllocator m_hAllocator;
    4544  VmaBlockVector* const m_pBlockVector;
    4545  uint32_t m_CurrentFrameIndex;
    4546  VkDeviceSize m_BytesMoved;
    4547  uint32_t m_AllocationsMoved;
    4548 
    4549  struct AllocationInfo
    4550  {
    4551  VmaAllocation m_hAllocation;
    4552  VkBool32* m_pChanged;
    4553 
    4554  AllocationInfo() :
    4555  m_hAllocation(VK_NULL_HANDLE),
    4556  m_pChanged(VMA_NULL)
    4557  {
    4558  }
    4559  };
    4560 
    4561  struct AllocationInfoSizeGreater
    4562  {
    4563  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
    4564  {
    4565  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
    4566  }
    4567  };
    4568 
    4569  // Used between AddAllocation and Defragment.
    4570  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4571 
    4572  struct BlockInfo
    4573  {
    4574  VmaDeviceMemoryBlock* m_pBlock;
    4575  bool m_HasNonMovableAllocations;
    4576  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4577 
    4578  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
    4579  m_pBlock(VMA_NULL),
    4580  m_HasNonMovableAllocations(true),
    4581  m_Allocations(pAllocationCallbacks),
    4582  m_pMappedDataForDefragmentation(VMA_NULL)
    4583  {
    4584  }
    4585 
    4586  void CalcHasNonMovableAllocations()
    4587  {
    4588  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
    4589  const size_t defragmentAllocCount = m_Allocations.size();
    4590  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
    4591  }
    4592 
    4593  void SortAllocationsBySizeDescecnding()
    4594  {
    4595  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
    4596  }
    4597 
    4598  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
    4599  void Unmap(VmaAllocator hAllocator);
    4600 
    4601  private:
    4602  // Not null if mapped for defragmentation only, not originally mapped.
    4603  void* m_pMappedDataForDefragmentation;
    4604  };
    4605 
    4606  struct BlockPointerLess
    4607  {
    4608  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
    4609  {
    4610  return pLhsBlockInfo->m_pBlock < pRhsBlock;
    4611  }
    4612  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4613  {
    4614  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
    4615  }
    4616  };
    4617 
    4618  // 1. Blocks with some non-movable allocations go first.
    4619  // 2. Blocks with smaller sumFreeSize go first.
    4620  struct BlockInfoCompareMoveDestination
    4621  {
    4622  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4623  {
    4624  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
    4625  {
    4626  return true;
    4627  }
    4628  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
    4629  {
    4630  return false;
    4631  }
    4632  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
    4633  {
    4634  return true;
    4635  }
    4636  return false;
    4637  }
    4638  };
    4639 
    4640  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
    4641  BlockInfoVector m_Blocks;
    4642 
    4643  VkResult DefragmentRound(
    4644  VkDeviceSize maxBytesToMove,
    4645  uint32_t maxAllocationsToMove);
    4646 
    4647  static bool MoveMakesSense(
    4648  size_t dstBlockIndex, VkDeviceSize dstOffset,
    4649  size_t srcBlockIndex, VkDeviceSize srcOffset);
    4650 
    4651 public:
    4652  VmaDefragmentator(
    4653  VmaAllocator hAllocator,
    4654  VmaBlockVector* pBlockVector,
    4655  uint32_t currentFrameIndex);
    4656 
    4657  ~VmaDefragmentator();
    4658 
    4659  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
    4660  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
    4661 
    4662  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
    4663 
    4664  VkResult Defragment(
    4665  VkDeviceSize maxBytesToMove,
    4666  uint32_t maxAllocationsToMove);
    4667 };
    4668 
    4669 // Main allocator object.
    4670 struct VmaAllocator_T
    4671 {
    4672  VMA_CLASS_NO_COPY(VmaAllocator_T)
    4673 public:
    4674  bool m_UseMutex;
    4675  bool m_UseKhrDedicatedAllocation;
    4676  VkDevice m_hDevice;
    4677  bool m_AllocationCallbacksSpecified;
    4678  VkAllocationCallbacks m_AllocationCallbacks;
    4679  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
    4680 
    4681  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
    4682  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
    4683  VMA_MUTEX m_HeapSizeLimitMutex;
    4684 
    4685  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
    4686  VkPhysicalDeviceMemoryProperties m_MemProps;
    4687 
    4688  // Default pools.
    4689  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
    4690 
    4691  // Each vector is sorted by memory (handle value).
    4692  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
    4693  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
    4694  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
    4695 
    4696  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
    4697  ~VmaAllocator_T();
    4698 
    4699  const VkAllocationCallbacks* GetAllocationCallbacks() const
    4700  {
    4701  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
    4702  }
    4703  const VmaVulkanFunctions& GetVulkanFunctions() const
    4704  {
    4705  return m_VulkanFunctions;
    4706  }
    4707 
    4708  VkDeviceSize GetBufferImageGranularity() const
    4709  {
    4710  return VMA_MAX(
    4711  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
    4712  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
    4713  }
    4714 
    4715  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
    4716  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
    4717 
    4718  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
    4719  {
    4720  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
    4721  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
    4722  }
    4723  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
    4724  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
    4725  {
    4726  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
    4727  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    4728  }
    4729  // Minimum alignment for all allocations in specific memory type.
    4730  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
    4731  {
    4732  return IsMemoryTypeNonCoherent(memTypeIndex) ?
    4733  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
    4734  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
    4735  }
    4736 
    4737  bool IsIntegratedGpu() const
    4738  {
    4739  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
    4740  }
    4741 
    4742  void GetBufferMemoryRequirements(
    4743  VkBuffer hBuffer,
    4744  VkMemoryRequirements& memReq,
    4745  bool& requiresDedicatedAllocation,
    4746  bool& prefersDedicatedAllocation) const;
    4747  void GetImageMemoryRequirements(
    4748  VkImage hImage,
    4749  VkMemoryRequirements& memReq,
    4750  bool& requiresDedicatedAllocation,
    4751  bool& prefersDedicatedAllocation) const;
    4752 
    4753  // Main allocation function.
    4754  VkResult AllocateMemory(
    4755  const VkMemoryRequirements& vkMemReq,
    4756  bool requiresDedicatedAllocation,
    4757  bool prefersDedicatedAllocation,
    4758  VkBuffer dedicatedBuffer,
    4759  VkImage dedicatedImage,
    4760  const VmaAllocationCreateInfo& createInfo,
    4761  VmaSuballocationType suballocType,
    4762  VmaAllocation* pAllocation);
    4763 
    4764  // Main deallocation function.
    4765  void FreeMemory(const VmaAllocation allocation);
    4766 
    4767  void CalculateStats(VmaStats* pStats);
    4768 
    4769 #if VMA_STATS_STRING_ENABLED
    4770  void PrintDetailedMap(class VmaJsonWriter& json);
    4771 #endif
    4772 
    4773  VkResult Defragment(
    4774  VmaAllocation* pAllocations,
    4775  size_t allocationCount,
    4776  VkBool32* pAllocationsChanged,
    4777  const VmaDefragmentationInfo* pDefragmentationInfo,
    4778  VmaDefragmentationStats* pDefragmentationStats);
    4779 
    4780  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
    4781  bool TouchAllocation(VmaAllocation hAllocation);
    4782 
    4783  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
    4784  void DestroyPool(VmaPool pool);
    4785  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
    4786 
    4787  void SetCurrentFrameIndex(uint32_t frameIndex);
    4788 
    4789  void MakePoolAllocationsLost(
    4790  VmaPool hPool,
    4791  size_t* pLostAllocationCount);
    4792  VkResult CheckPoolCorruption(VmaPool hPool);
    4793  VkResult CheckCorruption(uint32_t memoryTypeBits);
    4794 
    4795  void CreateLostAllocation(VmaAllocation* pAllocation);
    4796 
    4797  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
    4798  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
    4799 
    4800  VkResult Map(VmaAllocation hAllocation, void** ppData);
    4801  void Unmap(VmaAllocation hAllocation);
    4802 
    4803  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
    4804  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
    4805 
    4806  void FlushOrInvalidateAllocation(
    4807  VmaAllocation hAllocation,
    4808  VkDeviceSize offset, VkDeviceSize size,
    4809  VMA_CACHE_OPERATION op);
    4810 
    4811  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
    4812 
    4813 private:
    4814  VkDeviceSize m_PreferredLargeHeapBlockSize;
    4815 
    4816  VkPhysicalDevice m_PhysicalDevice;
    4817  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
    4818 
    4819  VMA_MUTEX m_PoolsMutex;
    4820  // Protected by m_PoolsMutex. Sorted by pointer value.
    4821  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
    4822  uint32_t m_NextPoolId;
    4823 
    4824  VmaVulkanFunctions m_VulkanFunctions;
    4825 
    4826  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
    4827 
    4828  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
    4829 
    4830  VkResult AllocateMemoryOfType(
    4831  VkDeviceSize size,
    4832  VkDeviceSize alignment,
    4833  bool dedicatedAllocation,
    4834  VkBuffer dedicatedBuffer,
    4835  VkImage dedicatedImage,
    4836  const VmaAllocationCreateInfo& createInfo,
    4837  uint32_t memTypeIndex,
    4838  VmaSuballocationType suballocType,
    4839  VmaAllocation* pAllocation);
    4840 
    4841  // Allocates and registers new VkDeviceMemory specifically for single allocation.
    4842  VkResult AllocateDedicatedMemory(
    4843  VkDeviceSize size,
    4844  VmaSuballocationType suballocType,
    4845  uint32_t memTypeIndex,
    4846  bool map,
    4847  bool isUserDataString,
    4848  void* pUserData,
    4849  VkBuffer dedicatedBuffer,
    4850  VkImage dedicatedImage,
    4851  VmaAllocation* pAllocation);
    4852 
    4853  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
    4854  void FreeDedicatedMemory(VmaAllocation allocation);
    4855 };
    4856 
    4858 // Memory allocation #2 after VmaAllocator_T definition
    4859 
    4860 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
    4861 {
    4862  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
    4863 }
    4864 
    4865 static void VmaFree(VmaAllocator hAllocator, void* ptr)
    4866 {
    4867  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
    4868 }
    4869 
    4870 template<typename T>
    4871 static T* VmaAllocate(VmaAllocator hAllocator)
    4872 {
    4873  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
    4874 }
    4875 
    4876 template<typename T>
    4877 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
    4878 {
    4879  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
    4880 }
    4881 
    4882 template<typename T>
    4883 static void vma_delete(VmaAllocator hAllocator, T* ptr)
    4884 {
    4885  if(ptr != VMA_NULL)
    4886  {
    4887  ptr->~T();
    4888  VmaFree(hAllocator, ptr);
    4889  }
    4890 }
    4891 
    4892 template<typename T>
    4893 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
    4894 {
    4895  if(ptr != VMA_NULL)
    4896  {
    4897  for(size_t i = count; i--; )
    4898  ptr[i].~T();
    4899  VmaFree(hAllocator, ptr);
    4900  }
    4901 }
    4902 
    4904 // VmaStringBuilder
    4905 
    4906 #if VMA_STATS_STRING_ENABLED
    4907 
    4908 class VmaStringBuilder
    4909 {
    4910 public:
    4911  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
    4912  size_t GetLength() const { return m_Data.size(); }
    4913  const char* GetData() const { return m_Data.data(); }
    4914 
    4915  void Add(char ch) { m_Data.push_back(ch); }
    4916  void Add(const char* pStr);
    4917  void AddNewLine() { Add('\n'); }
    4918  void AddNumber(uint32_t num);
    4919  void AddNumber(uint64_t num);
    4920  void AddPointer(const void* ptr);
    4921 
    4922 private:
    4923  VmaVector< char, VmaStlAllocator<char> > m_Data;
    4924 };
    4925 
    4926 void VmaStringBuilder::Add(const char* pStr)
    4927 {
    4928  const size_t strLen = strlen(pStr);
    4929  if(strLen > 0)
    4930  {
    4931  const size_t oldCount = m_Data.size();
    4932  m_Data.resize(oldCount + strLen);
    4933  memcpy(m_Data.data() + oldCount, pStr, strLen);
    4934  }
    4935 }
    4936 
    4937 void VmaStringBuilder::AddNumber(uint32_t num)
    4938 {
    4939  char buf[11];
    4940  VmaUint32ToStr(buf, sizeof(buf), num);
    4941  Add(buf);
    4942 }
    4943 
    4944 void VmaStringBuilder::AddNumber(uint64_t num)
    4945 {
    4946  char buf[21];
    4947  VmaUint64ToStr(buf, sizeof(buf), num);
    4948  Add(buf);
    4949 }
    4950 
    4951 void VmaStringBuilder::AddPointer(const void* ptr)
    4952 {
    4953  char buf[21];
    4954  VmaPtrToStr(buf, sizeof(buf), ptr);
    4955  Add(buf);
    4956 }
    4957 
    4958 #endif // #if VMA_STATS_STRING_ENABLED
    4959 
    4961 // VmaJsonWriter
    4962 
    4963 #if VMA_STATS_STRING_ENABLED
    4964 
    4965 class VmaJsonWriter
    4966 {
    4967  VMA_CLASS_NO_COPY(VmaJsonWriter)
    4968 public:
    4969  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
    4970  ~VmaJsonWriter();
    4971 
    4972  void BeginObject(bool singleLine = false);
    4973  void EndObject();
    4974 
    4975  void BeginArray(bool singleLine = false);
    4976  void EndArray();
    4977 
    4978  void WriteString(const char* pStr);
    4979  void BeginString(const char* pStr = VMA_NULL);
    4980  void ContinueString(const char* pStr);
    4981  void ContinueString(uint32_t n);
    4982  void ContinueString(uint64_t n);
    4983  void ContinueString_Pointer(const void* ptr);
    4984  void EndString(const char* pStr = VMA_NULL);
    4985 
    4986  void WriteNumber(uint32_t n);
    4987  void WriteNumber(uint64_t n);
    4988  void WriteBool(bool b);
    4989  void WriteNull();
    4990 
    4991 private:
    4992  static const char* const INDENT;
    4993 
    4994  enum COLLECTION_TYPE
    4995  {
    4996  COLLECTION_TYPE_OBJECT,
    4997  COLLECTION_TYPE_ARRAY,
    4998  };
    4999  struct StackItem
    5000  {
    5001  COLLECTION_TYPE type;
    5002  uint32_t valueCount;
    5003  bool singleLineMode;
    5004  };
    5005 
    5006  VmaStringBuilder& m_SB;
    5007  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
    5008  bool m_InsideString;
    5009 
    5010  void BeginValue(bool isString);
    5011  void WriteIndent(bool oneLess = false);
    5012 };
    5013 
    5014 const char* const VmaJsonWriter::INDENT = " ";
    5015 
    5016 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
    5017  m_SB(sb),
    5018  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
    5019  m_InsideString(false)
    5020 {
    5021 }
    5022 
    5023 VmaJsonWriter::~VmaJsonWriter()
    5024 {
    5025  VMA_ASSERT(!m_InsideString);
    5026  VMA_ASSERT(m_Stack.empty());
    5027 }
    5028 
    5029 void VmaJsonWriter::BeginObject(bool singleLine)
    5030 {
    5031  VMA_ASSERT(!m_InsideString);
    5032 
    5033  BeginValue(false);
    5034  m_SB.Add('{');
    5035 
    5036  StackItem item;
    5037  item.type = COLLECTION_TYPE_OBJECT;
    5038  item.valueCount = 0;
    5039  item.singleLineMode = singleLine;
    5040  m_Stack.push_back(item);
    5041 }
    5042 
    5043 void VmaJsonWriter::EndObject()
    5044 {
    5045  VMA_ASSERT(!m_InsideString);
    5046 
    5047  WriteIndent(true);
    5048  m_SB.Add('}');
    5049 
    5050  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
    5051  m_Stack.pop_back();
    5052 }
    5053 
    5054 void VmaJsonWriter::BeginArray(bool singleLine)
    5055 {
    5056  VMA_ASSERT(!m_InsideString);
    5057 
    5058  BeginValue(false);
    5059  m_SB.Add('[');
    5060 
    5061  StackItem item;
    5062  item.type = COLLECTION_TYPE_ARRAY;
    5063  item.valueCount = 0;
    5064  item.singleLineMode = singleLine;
    5065  m_Stack.push_back(item);
    5066 }
    5067 
    5068 void VmaJsonWriter::EndArray()
    5069 {
    5070  VMA_ASSERT(!m_InsideString);
    5071 
    5072  WriteIndent(true);
    5073  m_SB.Add(']');
    5074 
    5075  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
    5076  m_Stack.pop_back();
    5077 }
    5078 
    5079 void VmaJsonWriter::WriteString(const char* pStr)
    5080 {
    5081  BeginString(pStr);
    5082  EndString();
    5083 }
    5084 
    5085 void VmaJsonWriter::BeginString(const char* pStr)
    5086 {
    5087  VMA_ASSERT(!m_InsideString);
    5088 
    5089  BeginValue(true);
    5090  m_SB.Add('"');
    5091  m_InsideString = true;
    5092  if(pStr != VMA_NULL && pStr[0] != '\0')
    5093  {
    5094  ContinueString(pStr);
    5095  }
    5096 }
    5097 
    5098 void VmaJsonWriter::ContinueString(const char* pStr)
    5099 {
    5100  VMA_ASSERT(m_InsideString);
    5101 
    5102  const size_t strLen = strlen(pStr);
    5103  for(size_t i = 0; i < strLen; ++i)
    5104  {
    5105  char ch = pStr[i];
    5106  if(ch == '\'')
    5107  {
    5108  m_SB.Add("\\\\");
    5109  }
    5110  else if(ch == '"')
    5111  {
    5112  m_SB.Add("\\\"");
    5113  }
    5114  else if(ch >= 32)
    5115  {
    5116  m_SB.Add(ch);
    5117  }
    5118  else switch(ch)
    5119  {
    5120  case '\b':
    5121  m_SB.Add("\\b");
    5122  break;
    5123  case '\f':
    5124  m_SB.Add("\\f");
    5125  break;
    5126  case '\n':
    5127  m_SB.Add("\\n");
    5128  break;
    5129  case '\r':
    5130  m_SB.Add("\\r");
    5131  break;
    5132  case '\t':
    5133  m_SB.Add("\\t");
    5134  break;
    5135  default:
    5136  VMA_ASSERT(0 && "Character not currently supported.");
    5137  break;
    5138  }
    5139  }
    5140 }
    5141 
    5142 void VmaJsonWriter::ContinueString(uint32_t n)
    5143 {
    5144  VMA_ASSERT(m_InsideString);
    5145  m_SB.AddNumber(n);
    5146 }
    5147 
    5148 void VmaJsonWriter::ContinueString(uint64_t n)
    5149 {
    5150  VMA_ASSERT(m_InsideString);
    5151  m_SB.AddNumber(n);
    5152 }
    5153 
    5154 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
    5155 {
    5156  VMA_ASSERT(m_InsideString);
    5157  m_SB.AddPointer(ptr);
    5158 }
    5159 
    5160 void VmaJsonWriter::EndString(const char* pStr)
    5161 {
    5162  VMA_ASSERT(m_InsideString);
    5163  if(pStr != VMA_NULL && pStr[0] != '\0')
    5164  {
    5165  ContinueString(pStr);
    5166  }
    5167  m_SB.Add('"');
    5168  m_InsideString = false;
    5169 }
    5170 
    5171 void VmaJsonWriter::WriteNumber(uint32_t n)
    5172 {
    5173  VMA_ASSERT(!m_InsideString);
    5174  BeginValue(false);
    5175  m_SB.AddNumber(n);
    5176 }
    5177 
    5178 void VmaJsonWriter::WriteNumber(uint64_t n)
    5179 {
    5180  VMA_ASSERT(!m_InsideString);
    5181  BeginValue(false);
    5182  m_SB.AddNumber(n);
    5183 }
    5184 
    5185 void VmaJsonWriter::WriteBool(bool b)
    5186 {
    5187  VMA_ASSERT(!m_InsideString);
    5188  BeginValue(false);
    5189  m_SB.Add(b ? "true" : "false");
    5190 }
    5191 
    5192 void VmaJsonWriter::WriteNull()
    5193 {
    5194  VMA_ASSERT(!m_InsideString);
    5195  BeginValue(false);
    5196  m_SB.Add("null");
    5197 }
    5198 
    5199 void VmaJsonWriter::BeginValue(bool isString)
    5200 {
    5201  if(!m_Stack.empty())
    5202  {
    5203  StackItem& currItem = m_Stack.back();
    5204  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5205  currItem.valueCount % 2 == 0)
    5206  {
    5207  VMA_ASSERT(isString);
    5208  }
    5209 
    5210  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5211  currItem.valueCount % 2 != 0)
    5212  {
    5213  m_SB.Add(": ");
    5214  }
    5215  else if(currItem.valueCount > 0)
    5216  {
    5217  m_SB.Add(", ");
    5218  WriteIndent();
    5219  }
    5220  else
    5221  {
    5222  WriteIndent();
    5223  }
    5224  ++currItem.valueCount;
    5225  }
    5226 }
    5227 
    5228 void VmaJsonWriter::WriteIndent(bool oneLess)
    5229 {
    5230  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
    5231  {
    5232  m_SB.AddNewLine();
    5233 
    5234  size_t count = m_Stack.size();
    5235  if(count > 0 && oneLess)
    5236  {
    5237  --count;
    5238  }
    5239  for(size_t i = 0; i < count; ++i)
    5240  {
    5241  m_SB.Add(INDENT);
    5242  }
    5243  }
    5244 }
    5245 
    5246 #endif // #if VMA_STATS_STRING_ENABLED
    5247 
    5249 
    5250 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
    5251 {
    5252  if(IsUserDataString())
    5253  {
    5254  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
    5255 
    5256  FreeUserDataString(hAllocator);
    5257 
    5258  if(pUserData != VMA_NULL)
    5259  {
    5260  const char* const newStrSrc = (char*)pUserData;
    5261  const size_t newStrLen = strlen(newStrSrc);
    5262  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
    5263  memcpy(newStrDst, newStrSrc, newStrLen + 1);
    5264  m_pUserData = newStrDst;
    5265  }
    5266  }
    5267  else
    5268  {
    5269  m_pUserData = pUserData;
    5270  }
    5271 }
    5272 
    5273 void VmaAllocation_T::ChangeBlockAllocation(
    5274  VmaAllocator hAllocator,
    5275  VmaDeviceMemoryBlock* block,
    5276  VkDeviceSize offset)
    5277 {
    5278  VMA_ASSERT(block != VMA_NULL);
    5279  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5280 
    5281  // Move mapping reference counter from old block to new block.
    5282  if(block != m_BlockAllocation.m_Block)
    5283  {
    5284  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
    5285  if(IsPersistentMap())
    5286  ++mapRefCount;
    5287  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
    5288  block->Map(hAllocator, mapRefCount, VMA_NULL);
    5289  }
    5290 
    5291  m_BlockAllocation.m_Block = block;
    5292  m_BlockAllocation.m_Offset = offset;
    5293 }
    5294 
    5295 VkDeviceSize VmaAllocation_T::GetOffset() const
    5296 {
    5297  switch(m_Type)
    5298  {
    5299  case ALLOCATION_TYPE_BLOCK:
    5300  return m_BlockAllocation.m_Offset;
    5301  case ALLOCATION_TYPE_DEDICATED:
    5302  return 0;
    5303  default:
    5304  VMA_ASSERT(0);
    5305  return 0;
    5306  }
    5307 }
    5308 
    5309 VkDeviceMemory VmaAllocation_T::GetMemory() const
    5310 {
    5311  switch(m_Type)
    5312  {
    5313  case ALLOCATION_TYPE_BLOCK:
    5314  return m_BlockAllocation.m_Block->GetDeviceMemory();
    5315  case ALLOCATION_TYPE_DEDICATED:
    5316  return m_DedicatedAllocation.m_hMemory;
    5317  default:
    5318  VMA_ASSERT(0);
    5319  return VK_NULL_HANDLE;
    5320  }
    5321 }
    5322 
    5323 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
    5324 {
    5325  switch(m_Type)
    5326  {
    5327  case ALLOCATION_TYPE_BLOCK:
    5328  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
    5329  case ALLOCATION_TYPE_DEDICATED:
    5330  return m_DedicatedAllocation.m_MemoryTypeIndex;
    5331  default:
    5332  VMA_ASSERT(0);
    5333  return UINT32_MAX;
    5334  }
    5335 }
    5336 
    5337 void* VmaAllocation_T::GetMappedData() const
    5338 {
    5339  switch(m_Type)
    5340  {
    5341  case ALLOCATION_TYPE_BLOCK:
    5342  if(m_MapCount != 0)
    5343  {
    5344  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
    5345  VMA_ASSERT(pBlockData != VMA_NULL);
    5346  return (char*)pBlockData + m_BlockAllocation.m_Offset;
    5347  }
    5348  else
    5349  {
    5350  return VMA_NULL;
    5351  }
    5352  break;
    5353  case ALLOCATION_TYPE_DEDICATED:
    5354  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
    5355  return m_DedicatedAllocation.m_pMappedData;
    5356  default:
    5357  VMA_ASSERT(0);
    5358  return VMA_NULL;
    5359  }
    5360 }
    5361 
    5362 bool VmaAllocation_T::CanBecomeLost() const
    5363 {
    5364  switch(m_Type)
    5365  {
    5366  case ALLOCATION_TYPE_BLOCK:
    5367  return m_BlockAllocation.m_CanBecomeLost;
    5368  case ALLOCATION_TYPE_DEDICATED:
    5369  return false;
    5370  default:
    5371  VMA_ASSERT(0);
    5372  return false;
    5373  }
    5374 }
    5375 
    5376 VmaPool VmaAllocation_T::GetPool() const
    5377 {
    5378  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5379  return m_BlockAllocation.m_hPool;
    5380 }
    5381 
    5382 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    5383 {
    5384  VMA_ASSERT(CanBecomeLost());
    5385 
    5386  /*
    5387  Warning: This is a carefully designed algorithm.
    5388  Do not modify unless you really know what you're doing :)
    5389  */
    5390  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
    5391  for(;;)
    5392  {
    5393  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    5394  {
    5395  VMA_ASSERT(0);
    5396  return false;
    5397  }
    5398  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
    5399  {
    5400  return false;
    5401  }
    5402  else // Last use time earlier than current time.
    5403  {
    5404  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
    5405  {
    5406  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
    5407  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
    5408  return true;
    5409  }
    5410  }
    5411  }
    5412 }
    5413 
    5414 #if VMA_STATS_STRING_ENABLED
    5415 
    5416 // Correspond to values of enum VmaSuballocationType.
    5417 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
    5418  "FREE",
    5419  "UNKNOWN",
    5420  "BUFFER",
    5421  "IMAGE_UNKNOWN",
    5422  "IMAGE_LINEAR",
    5423  "IMAGE_OPTIMAL",
    5424 };
    5425 
    5426 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
    5427 {
    5428  json.WriteString("Type");
    5429  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
    5430 
    5431  json.WriteString("Size");
    5432  json.WriteNumber(m_Size);
    5433 
    5434  if(m_pUserData != VMA_NULL)
    5435  {
    5436  json.WriteString("UserData");
    5437  if(IsUserDataString())
    5438  {
    5439  json.WriteString((const char*)m_pUserData);
    5440  }
    5441  else
    5442  {
    5443  json.BeginString();
    5444  json.ContinueString_Pointer(m_pUserData);
    5445  json.EndString();
    5446  }
    5447  }
    5448 
    5449  json.WriteString("CreationFrameIndex");
    5450  json.WriteNumber(m_CreationFrameIndex);
    5451 
    5452  json.WriteString("LastUseFrameIndex");
    5453  json.WriteNumber(GetLastUseFrameIndex());
    5454 
    5455  if(m_BufferImageUsage != 0)
    5456  {
    5457  json.WriteString("Usage");
    5458  json.WriteNumber(m_BufferImageUsage);
    5459  }
    5460 }
    5461 
    5462 #endif
    5463 
    5464 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
    5465 {
    5466  VMA_ASSERT(IsUserDataString());
    5467  if(m_pUserData != VMA_NULL)
    5468  {
    5469  char* const oldStr = (char*)m_pUserData;
    5470  const size_t oldStrLen = strlen(oldStr);
    5471  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
    5472  m_pUserData = VMA_NULL;
    5473  }
    5474 }
    5475 
    5476 void VmaAllocation_T::BlockAllocMap()
    5477 {
    5478  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5479 
    5480  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5481  {
    5482  ++m_MapCount;
    5483  }
    5484  else
    5485  {
    5486  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
    5487  }
    5488 }
    5489 
    5490 void VmaAllocation_T::BlockAllocUnmap()
    5491 {
    5492  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5493 
    5494  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5495  {
    5496  --m_MapCount;
    5497  }
    5498  else
    5499  {
    5500  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
    5501  }
    5502 }
    5503 
    5504 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
    5505 {
    5506  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5507 
    5508  if(m_MapCount != 0)
    5509  {
    5510  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5511  {
    5512  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
    5513  *ppData = m_DedicatedAllocation.m_pMappedData;
    5514  ++m_MapCount;
    5515  return VK_SUCCESS;
    5516  }
    5517  else
    5518  {
    5519  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
    5520  return VK_ERROR_MEMORY_MAP_FAILED;
    5521  }
    5522  }
    5523  else
    5524  {
    5525  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    5526  hAllocator->m_hDevice,
    5527  m_DedicatedAllocation.m_hMemory,
    5528  0, // offset
    5529  VK_WHOLE_SIZE,
    5530  0, // flags
    5531  ppData);
    5532  if(result == VK_SUCCESS)
    5533  {
    5534  m_DedicatedAllocation.m_pMappedData = *ppData;
    5535  m_MapCount = 1;
    5536  }
    5537  return result;
    5538  }
    5539 }
    5540 
    5541 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
    5542 {
    5543  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5544 
    5545  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5546  {
    5547  --m_MapCount;
    5548  if(m_MapCount == 0)
    5549  {
    5550  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
    5551  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
    5552  hAllocator->m_hDevice,
    5553  m_DedicatedAllocation.m_hMemory);
    5554  }
    5555  }
    5556  else
    5557  {
    5558  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
    5559  }
    5560 }
    5561 
    5562 #if VMA_STATS_STRING_ENABLED
    5563 
    5564 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
    5565 {
    5566  json.BeginObject();
    5567 
    5568  json.WriteString("Blocks");
    5569  json.WriteNumber(stat.blockCount);
    5570 
    5571  json.WriteString("Allocations");
    5572  json.WriteNumber(stat.allocationCount);
    5573 
    5574  json.WriteString("UnusedRanges");
    5575  json.WriteNumber(stat.unusedRangeCount);
    5576 
    5577  json.WriteString("UsedBytes");
    5578  json.WriteNumber(stat.usedBytes);
    5579 
    5580  json.WriteString("UnusedBytes");
    5581  json.WriteNumber(stat.unusedBytes);
    5582 
    5583  if(stat.allocationCount > 1)
    5584  {
    5585  json.WriteString("AllocationSize");
    5586  json.BeginObject(true);
    5587  json.WriteString("Min");
    5588  json.WriteNumber(stat.allocationSizeMin);
    5589  json.WriteString("Avg");
    5590  json.WriteNumber(stat.allocationSizeAvg);
    5591  json.WriteString("Max");
    5592  json.WriteNumber(stat.allocationSizeMax);
    5593  json.EndObject();
    5594  }
    5595 
    5596  if(stat.unusedRangeCount > 1)
    5597  {
    5598  json.WriteString("UnusedRangeSize");
    5599  json.BeginObject(true);
    5600  json.WriteString("Min");
    5601  json.WriteNumber(stat.unusedRangeSizeMin);
    5602  json.WriteString("Avg");
    5603  json.WriteNumber(stat.unusedRangeSizeAvg);
    5604  json.WriteString("Max");
    5605  json.WriteNumber(stat.unusedRangeSizeMax);
    5606  json.EndObject();
    5607  }
    5608 
    5609  json.EndObject();
    5610 }
    5611 
    5612 #endif // #if VMA_STATS_STRING_ENABLED
    5613 
    5614 struct VmaSuballocationItemSizeLess
    5615 {
    5616  bool operator()(
    5617  const VmaSuballocationList::iterator lhs,
    5618  const VmaSuballocationList::iterator rhs) const
    5619  {
    5620  return lhs->size < rhs->size;
    5621  }
    5622  bool operator()(
    5623  const VmaSuballocationList::iterator lhs,
    5624  VkDeviceSize rhsSize) const
    5625  {
    5626  return lhs->size < rhsSize;
    5627  }
    5628 };
    5629 
    5631 // class VmaBlockMetadata
    5632 
    5633 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
    5634  m_Size(0),
    5635  m_FreeCount(0),
    5636  m_SumFreeSize(0),
    5637  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
    5638  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
    5639 {
    5640 }
    5641 
    5642 VmaBlockMetadata::~VmaBlockMetadata()
    5643 {
    5644 }
    5645 
    5646 void VmaBlockMetadata::Init(VkDeviceSize size)
    5647 {
    5648  m_Size = size;
    5649  m_FreeCount = 1;
    5650  m_SumFreeSize = size;
    5651 
    5652  VmaSuballocation suballoc = {};
    5653  suballoc.offset = 0;
    5654  suballoc.size = size;
    5655  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    5656  suballoc.hAllocation = VK_NULL_HANDLE;
    5657 
    5658  m_Suballocations.push_back(suballoc);
    5659  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
    5660  --suballocItem;
    5661  m_FreeSuballocationsBySize.push_back(suballocItem);
    5662 }
    5663 
    5664 bool VmaBlockMetadata::Validate() const
    5665 {
    5666  if(m_Suballocations.empty())
    5667  {
    5668  return false;
    5669  }
    5670 
    5671  // Expected offset of new suballocation as calculates from previous ones.
    5672  VkDeviceSize calculatedOffset = 0;
    5673  // Expected number of free suballocations as calculated from traversing their list.
    5674  uint32_t calculatedFreeCount = 0;
    5675  // Expected sum size of free suballocations as calculated from traversing their list.
    5676  VkDeviceSize calculatedSumFreeSize = 0;
    5677  // Expected number of free suballocations that should be registered in
    5678  // m_FreeSuballocationsBySize calculated from traversing their list.
    5679  size_t freeSuballocationsToRegister = 0;
    5680  // True if previous visited suballocation was free.
    5681  bool prevFree = false;
    5682 
    5683  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5684  suballocItem != m_Suballocations.cend();
    5685  ++suballocItem)
    5686  {
    5687  const VmaSuballocation& subAlloc = *suballocItem;
    5688 
    5689  // Actual offset of this suballocation doesn't match expected one.
    5690  if(subAlloc.offset != calculatedOffset)
    5691  {
    5692  return false;
    5693  }
    5694 
    5695  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
    5696  // Two adjacent free suballocations are invalid. They should be merged.
    5697  if(prevFree && currFree)
    5698  {
    5699  return false;
    5700  }
    5701 
    5702  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
    5703  {
    5704  return false;
    5705  }
    5706 
    5707  if(currFree)
    5708  {
    5709  calculatedSumFreeSize += subAlloc.size;
    5710  ++calculatedFreeCount;
    5711  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    5712  {
    5713  ++freeSuballocationsToRegister;
    5714  }
    5715 
    5716  // Margin required between allocations - every free space must be at least that large.
    5717  if(subAlloc.size < VMA_DEBUG_MARGIN)
    5718  {
    5719  return false;
    5720  }
    5721  }
    5722  else
    5723  {
    5724  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
    5725  {
    5726  return false;
    5727  }
    5728  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
    5729  {
    5730  return false;
    5731  }
    5732 
    5733  // Margin required between allocations - previous allocation must be free.
    5734  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
    5735  {
    5736  return false;
    5737  }
    5738  }
    5739 
    5740  calculatedOffset += subAlloc.size;
    5741  prevFree = currFree;
    5742  }
    5743 
    5744  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
    5745  // match expected one.
    5746  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
    5747  {
    5748  return false;
    5749  }
    5750 
    5751  VkDeviceSize lastSize = 0;
    5752  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
    5753  {
    5754  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
    5755 
    5756  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
    5757  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
    5758  {
    5759  return false;
    5760  }
    5761  // They must be sorted by size ascending.
    5762  if(suballocItem->size < lastSize)
    5763  {
    5764  return false;
    5765  }
    5766 
    5767  lastSize = suballocItem->size;
    5768  }
    5769 
    5770  // Check if totals match calculacted values.
    5771  if(!ValidateFreeSuballocationList() ||
    5772  (calculatedOffset != m_Size) ||
    5773  (calculatedSumFreeSize != m_SumFreeSize) ||
    5774  (calculatedFreeCount != m_FreeCount))
    5775  {
    5776  return false;
    5777  }
    5778 
    5779  return true;
    5780 }
    5781 
    5782 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
    5783 {
    5784  if(!m_FreeSuballocationsBySize.empty())
    5785  {
    5786  return m_FreeSuballocationsBySize.back()->size;
    5787  }
    5788  else
    5789  {
    5790  return 0;
    5791  }
    5792 }
    5793 
    5794 bool VmaBlockMetadata::IsEmpty() const
    5795 {
    5796  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
    5797 }
    5798 
    5799 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
    5800 {
    5801  outInfo.blockCount = 1;
    5802 
    5803  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    5804  outInfo.allocationCount = rangeCount - m_FreeCount;
    5805  outInfo.unusedRangeCount = m_FreeCount;
    5806 
    5807  outInfo.unusedBytes = m_SumFreeSize;
    5808  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
    5809 
    5810  outInfo.allocationSizeMin = UINT64_MAX;
    5811  outInfo.allocationSizeMax = 0;
    5812  outInfo.unusedRangeSizeMin = UINT64_MAX;
    5813  outInfo.unusedRangeSizeMax = 0;
    5814 
    5815  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5816  suballocItem != m_Suballocations.cend();
    5817  ++suballocItem)
    5818  {
    5819  const VmaSuballocation& suballoc = *suballocItem;
    5820  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
    5821  {
    5822  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
    5823  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
    5824  }
    5825  else
    5826  {
    5827  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
    5828  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
    5829  }
    5830  }
    5831 }
    5832 
    5833 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
    5834 {
    5835  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    5836 
    5837  inoutStats.size += m_Size;
    5838  inoutStats.unusedSize += m_SumFreeSize;
    5839  inoutStats.allocationCount += rangeCount - m_FreeCount;
    5840  inoutStats.unusedRangeCount += m_FreeCount;
    5841  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
    5842 }
    5843 
    5844 #if VMA_STATS_STRING_ENABLED
    5845 
    5846 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
    5847 {
    5848  json.BeginObject();
    5849 
    5850  json.WriteString("TotalBytes");
    5851  json.WriteNumber(m_Size);
    5852 
    5853  json.WriteString("UnusedBytes");
    5854  json.WriteNumber(m_SumFreeSize);
    5855 
    5856  json.WriteString("Allocations");
    5857  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
    5858 
    5859  json.WriteString("UnusedRanges");
    5860  json.WriteNumber(m_FreeCount);
    5861 
    5862  json.WriteString("Suballocations");
    5863  json.BeginArray();
    5864  size_t i = 0;
    5865  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5866  suballocItem != m_Suballocations.cend();
    5867  ++suballocItem, ++i)
    5868  {
    5869  json.BeginObject(true);
    5870 
    5871  json.WriteString("Offset");
    5872  json.WriteNumber(suballocItem->offset);
    5873 
    5874  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    5875  {
    5876  json.WriteString("Type");
    5877  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
    5878 
    5879  json.WriteString("Size");
    5880  json.WriteNumber(suballocItem->size);
    5881  }
    5882  else
    5883  {
    5884  suballocItem->hAllocation->PrintParameters(json);
    5885  }
    5886 
    5887  json.EndObject();
    5888  }
    5889  json.EndArray();
    5890 
    5891  json.EndObject();
    5892 }
    5893 
    5894 #endif // #if VMA_STATS_STRING_ENABLED
    5895 
    5896 /*
    5897 How many suitable free suballocations to analyze before choosing best one.
    5898 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
    5899  be chosen.
    5900 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
    5901  suballocations will be analized and best one will be chosen.
    5902 - Any other value is also acceptable.
    5903 */
    5904 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
    5905 
    5906 bool VmaBlockMetadata::CreateAllocationRequest(
    5907  uint32_t currentFrameIndex,
    5908  uint32_t frameInUseCount,
    5909  VkDeviceSize bufferImageGranularity,
    5910  VkDeviceSize allocSize,
    5911  VkDeviceSize allocAlignment,
    5912  VmaSuballocationType allocType,
    5913  bool canMakeOtherLost,
    5914  VmaAllocationRequest* pAllocationRequest)
    5915 {
    5916  VMA_ASSERT(allocSize > 0);
    5917  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    5918  VMA_ASSERT(pAllocationRequest != VMA_NULL);
    5919  VMA_HEAVY_ASSERT(Validate());
    5920 
    5921  // There is not enough total free space in this block to fullfill the request: Early return.
    5922  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
    5923  {
    5924  return false;
    5925  }
    5926 
    5927  // New algorithm, efficiently searching freeSuballocationsBySize.
    5928  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
    5929  if(freeSuballocCount > 0)
    5930  {
    5931  if(VMA_BEST_FIT)
    5932  {
    5933  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
    5934  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    5935  m_FreeSuballocationsBySize.data(),
    5936  m_FreeSuballocationsBySize.data() + freeSuballocCount,
    5937  allocSize + 2 * VMA_DEBUG_MARGIN,
    5938  VmaSuballocationItemSizeLess());
    5939  size_t index = it - m_FreeSuballocationsBySize.data();
    5940  for(; index < freeSuballocCount; ++index)
    5941  {
    5942  if(CheckAllocation(
    5943  currentFrameIndex,
    5944  frameInUseCount,
    5945  bufferImageGranularity,
    5946  allocSize,
    5947  allocAlignment,
    5948  allocType,
    5949  m_FreeSuballocationsBySize[index],
    5950  false, // canMakeOtherLost
    5951  &pAllocationRequest->offset,
    5952  &pAllocationRequest->itemsToMakeLostCount,
    5953  &pAllocationRequest->sumFreeSize,
    5954  &pAllocationRequest->sumItemSize))
    5955  {
    5956  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    5957  return true;
    5958  }
    5959  }
    5960  }
    5961  else
    5962  {
    5963  // Search staring from biggest suballocations.
    5964  for(size_t index = freeSuballocCount; index--; )
    5965  {
    5966  if(CheckAllocation(
    5967  currentFrameIndex,
    5968  frameInUseCount,
    5969  bufferImageGranularity,
    5970  allocSize,
    5971  allocAlignment,
    5972  allocType,
    5973  m_FreeSuballocationsBySize[index],
    5974  false, // canMakeOtherLost
    5975  &pAllocationRequest->offset,
    5976  &pAllocationRequest->itemsToMakeLostCount,
    5977  &pAllocationRequest->sumFreeSize,
    5978  &pAllocationRequest->sumItemSize))
    5979  {
    5980  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    5981  return true;
    5982  }
    5983  }
    5984  }
    5985  }
    5986 
    5987  if(canMakeOtherLost)
    5988  {
    5989  // Brute-force algorithm. TODO: Come up with something better.
    5990 
    5991  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
    5992  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
    5993 
    5994  VmaAllocationRequest tmpAllocRequest = {};
    5995  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
    5996  suballocIt != m_Suballocations.end();
    5997  ++suballocIt)
    5998  {
    5999  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
    6000  suballocIt->hAllocation->CanBecomeLost())
    6001  {
    6002  if(CheckAllocation(
    6003  currentFrameIndex,
    6004  frameInUseCount,
    6005  bufferImageGranularity,
    6006  allocSize,
    6007  allocAlignment,
    6008  allocType,
    6009  suballocIt,
    6010  canMakeOtherLost,
    6011  &tmpAllocRequest.offset,
    6012  &tmpAllocRequest.itemsToMakeLostCount,
    6013  &tmpAllocRequest.sumFreeSize,
    6014  &tmpAllocRequest.sumItemSize))
    6015  {
    6016  tmpAllocRequest.item = suballocIt;
    6017 
    6018  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
    6019  {
    6020  *pAllocationRequest = tmpAllocRequest;
    6021  }
    6022  }
    6023  }
    6024  }
    6025 
    6026  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
    6027  {
    6028  return true;
    6029  }
    6030  }
    6031 
    6032  return false;
    6033 }
    6034 
    6035 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
    6036  uint32_t currentFrameIndex,
    6037  uint32_t frameInUseCount,
    6038  VmaAllocationRequest* pAllocationRequest)
    6039 {
    6040  while(pAllocationRequest->itemsToMakeLostCount > 0)
    6041  {
    6042  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
    6043  {
    6044  ++pAllocationRequest->item;
    6045  }
    6046  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6047  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
    6048  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
    6049  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6050  {
    6051  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
    6052  --pAllocationRequest->itemsToMakeLostCount;
    6053  }
    6054  else
    6055  {
    6056  return false;
    6057  }
    6058  }
    6059 
    6060  VMA_HEAVY_ASSERT(Validate());
    6061  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6062  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6063 
    6064  return true;
    6065 }
    6066 
    6067 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    6068 {
    6069  uint32_t lostAllocationCount = 0;
    6070  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6071  it != m_Suballocations.end();
    6072  ++it)
    6073  {
    6074  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
    6075  it->hAllocation->CanBecomeLost() &&
    6076  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6077  {
    6078  it = FreeSuballocation(it);
    6079  ++lostAllocationCount;
    6080  }
    6081  }
    6082  return lostAllocationCount;
    6083 }
    6084 
    6085 VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData)
    6086 {
    6087  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6088  it != m_Suballocations.end();
    6089  ++it)
    6090  {
    6091  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6092  {
    6093  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
    6094  {
    6095  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
    6096  return VK_ERROR_VALIDATION_FAILED_EXT;
    6097  }
    6098  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
    6099  {
    6100  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
    6101  return VK_ERROR_VALIDATION_FAILED_EXT;
    6102  }
    6103  }
    6104  }
    6105 
    6106  return VK_SUCCESS;
    6107 }
    6108 
    6109 void VmaBlockMetadata::Alloc(
    6110  const VmaAllocationRequest& request,
    6111  VmaSuballocationType type,
    6112  VkDeviceSize allocSize,
    6113  VmaAllocation hAllocation)
    6114 {
    6115  VMA_ASSERT(request.item != m_Suballocations.end());
    6116  VmaSuballocation& suballoc = *request.item;
    6117  // Given suballocation is a free block.
    6118  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6119  // Given offset is inside this suballocation.
    6120  VMA_ASSERT(request.offset >= suballoc.offset);
    6121  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
    6122  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
    6123  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
    6124 
    6125  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
    6126  // it to become used.
    6127  UnregisterFreeSuballocation(request.item);
    6128 
    6129  suballoc.offset = request.offset;
    6130  suballoc.size = allocSize;
    6131  suballoc.type = type;
    6132  suballoc.hAllocation = hAllocation;
    6133 
    6134  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
    6135  if(paddingEnd)
    6136  {
    6137  VmaSuballocation paddingSuballoc = {};
    6138  paddingSuballoc.offset = request.offset + allocSize;
    6139  paddingSuballoc.size = paddingEnd;
    6140  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6141  VmaSuballocationList::iterator next = request.item;
    6142  ++next;
    6143  const VmaSuballocationList::iterator paddingEndItem =
    6144  m_Suballocations.insert(next, paddingSuballoc);
    6145  RegisterFreeSuballocation(paddingEndItem);
    6146  }
    6147 
    6148  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
    6149  if(paddingBegin)
    6150  {
    6151  VmaSuballocation paddingSuballoc = {};
    6152  paddingSuballoc.offset = request.offset - paddingBegin;
    6153  paddingSuballoc.size = paddingBegin;
    6154  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6155  const VmaSuballocationList::iterator paddingBeginItem =
    6156  m_Suballocations.insert(request.item, paddingSuballoc);
    6157  RegisterFreeSuballocation(paddingBeginItem);
    6158  }
    6159 
    6160  // Update totals.
    6161  m_FreeCount = m_FreeCount - 1;
    6162  if(paddingBegin > 0)
    6163  {
    6164  ++m_FreeCount;
    6165  }
    6166  if(paddingEnd > 0)
    6167  {
    6168  ++m_FreeCount;
    6169  }
    6170  m_SumFreeSize -= allocSize;
    6171 }
    6172 
    6173 void VmaBlockMetadata::Free(const VmaAllocation allocation)
    6174 {
    6175  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6176  suballocItem != m_Suballocations.end();
    6177  ++suballocItem)
    6178  {
    6179  VmaSuballocation& suballoc = *suballocItem;
    6180  if(suballoc.hAllocation == allocation)
    6181  {
    6182  FreeSuballocation(suballocItem);
    6183  VMA_HEAVY_ASSERT(Validate());
    6184  return;
    6185  }
    6186  }
    6187  VMA_ASSERT(0 && "Not found!");
    6188 }
    6189 
    6190 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
    6191 {
    6192  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6193  suballocItem != m_Suballocations.end();
    6194  ++suballocItem)
    6195  {
    6196  VmaSuballocation& suballoc = *suballocItem;
    6197  if(suballoc.offset == offset)
    6198  {
    6199  FreeSuballocation(suballocItem);
    6200  return;
    6201  }
    6202  }
    6203  VMA_ASSERT(0 && "Not found!");
    6204 }
    6205 
    6206 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
    6207 {
    6208  VkDeviceSize lastSize = 0;
    6209  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
    6210  {
    6211  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
    6212 
    6213  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6214  {
    6215  VMA_ASSERT(0);
    6216  return false;
    6217  }
    6218  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6219  {
    6220  VMA_ASSERT(0);
    6221  return false;
    6222  }
    6223  if(it->size < lastSize)
    6224  {
    6225  VMA_ASSERT(0);
    6226  return false;
    6227  }
    6228 
    6229  lastSize = it->size;
    6230  }
    6231  return true;
    6232 }
    6233 
    6234 bool VmaBlockMetadata::CheckAllocation(
    6235  uint32_t currentFrameIndex,
    6236  uint32_t frameInUseCount,
    6237  VkDeviceSize bufferImageGranularity,
    6238  VkDeviceSize allocSize,
    6239  VkDeviceSize allocAlignment,
    6240  VmaSuballocationType allocType,
    6241  VmaSuballocationList::const_iterator suballocItem,
    6242  bool canMakeOtherLost,
    6243  VkDeviceSize* pOffset,
    6244  size_t* itemsToMakeLostCount,
    6245  VkDeviceSize* pSumFreeSize,
    6246  VkDeviceSize* pSumItemSize) const
    6247 {
    6248  VMA_ASSERT(allocSize > 0);
    6249  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6250  VMA_ASSERT(suballocItem != m_Suballocations.cend());
    6251  VMA_ASSERT(pOffset != VMA_NULL);
    6252 
    6253  *itemsToMakeLostCount = 0;
    6254  *pSumFreeSize = 0;
    6255  *pSumItemSize = 0;
    6256 
    6257  if(canMakeOtherLost)
    6258  {
    6259  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6260  {
    6261  *pSumFreeSize = suballocItem->size;
    6262  }
    6263  else
    6264  {
    6265  if(suballocItem->hAllocation->CanBecomeLost() &&
    6266  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6267  {
    6268  ++*itemsToMakeLostCount;
    6269  *pSumItemSize = suballocItem->size;
    6270  }
    6271  else
    6272  {
    6273  return false;
    6274  }
    6275  }
    6276 
    6277  // Remaining size is too small for this request: Early return.
    6278  if(m_Size - suballocItem->offset < allocSize)
    6279  {
    6280  return false;
    6281  }
    6282 
    6283  // Start from offset equal to beginning of this suballocation.
    6284  *pOffset = suballocItem->offset;
    6285 
    6286  // Apply VMA_DEBUG_MARGIN at the beginning.
    6287  if(VMA_DEBUG_MARGIN > 0)
    6288  {
    6289  *pOffset += VMA_DEBUG_MARGIN;
    6290  }
    6291 
    6292  // Apply alignment.
    6293  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6294 
    6295  // Check previous suballocations for BufferImageGranularity conflicts.
    6296  // Make bigger alignment if necessary.
    6297  if(bufferImageGranularity > 1)
    6298  {
    6299  bool bufferImageGranularityConflict = false;
    6300  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6301  while(prevSuballocItem != m_Suballocations.cbegin())
    6302  {
    6303  --prevSuballocItem;
    6304  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6305  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6306  {
    6307  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6308  {
    6309  bufferImageGranularityConflict = true;
    6310  break;
    6311  }
    6312  }
    6313  else
    6314  // Already on previous page.
    6315  break;
    6316  }
    6317  if(bufferImageGranularityConflict)
    6318  {
    6319  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6320  }
    6321  }
    6322 
    6323  // Now that we have final *pOffset, check if we are past suballocItem.
    6324  // If yes, return false - this function should be called for another suballocItem as starting point.
    6325  if(*pOffset >= suballocItem->offset + suballocItem->size)
    6326  {
    6327  return false;
    6328  }
    6329 
    6330  // Calculate padding at the beginning based on current offset.
    6331  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
    6332 
    6333  // Calculate required margin at the end.
    6334  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6335 
    6336  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
    6337  // Another early return check.
    6338  if(suballocItem->offset + totalSize > m_Size)
    6339  {
    6340  return false;
    6341  }
    6342 
    6343  // Advance lastSuballocItem until desired size is reached.
    6344  // Update itemsToMakeLostCount.
    6345  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
    6346  if(totalSize > suballocItem->size)
    6347  {
    6348  VkDeviceSize remainingSize = totalSize - suballocItem->size;
    6349  while(remainingSize > 0)
    6350  {
    6351  ++lastSuballocItem;
    6352  if(lastSuballocItem == m_Suballocations.cend())
    6353  {
    6354  return false;
    6355  }
    6356  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6357  {
    6358  *pSumFreeSize += lastSuballocItem->size;
    6359  }
    6360  else
    6361  {
    6362  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
    6363  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
    6364  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6365  {
    6366  ++*itemsToMakeLostCount;
    6367  *pSumItemSize += lastSuballocItem->size;
    6368  }
    6369  else
    6370  {
    6371  return false;
    6372  }
    6373  }
    6374  remainingSize = (lastSuballocItem->size < remainingSize) ?
    6375  remainingSize - lastSuballocItem->size : 0;
    6376  }
    6377  }
    6378 
    6379  // Check next suballocations for BufferImageGranularity conflicts.
    6380  // If conflict exists, we must mark more allocations lost or fail.
    6381  if(bufferImageGranularity > 1)
    6382  {
    6383  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
    6384  ++nextSuballocItem;
    6385  while(nextSuballocItem != m_Suballocations.cend())
    6386  {
    6387  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6388  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6389  {
    6390  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6391  {
    6392  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
    6393  if(nextSuballoc.hAllocation->CanBecomeLost() &&
    6394  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6395  {
    6396  ++*itemsToMakeLostCount;
    6397  }
    6398  else
    6399  {
    6400  return false;
    6401  }
    6402  }
    6403  }
    6404  else
    6405  {
    6406  // Already on next page.
    6407  break;
    6408  }
    6409  ++nextSuballocItem;
    6410  }
    6411  }
    6412  }
    6413  else
    6414  {
    6415  const VmaSuballocation& suballoc = *suballocItem;
    6416  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6417 
    6418  *pSumFreeSize = suballoc.size;
    6419 
    6420  // Size of this suballocation is too small for this request: Early return.
    6421  if(suballoc.size < allocSize)
    6422  {
    6423  return false;
    6424  }
    6425 
    6426  // Start from offset equal to beginning of this suballocation.
    6427  *pOffset = suballoc.offset;
    6428 
    6429  // Apply VMA_DEBUG_MARGIN at the beginning.
    6430  if(VMA_DEBUG_MARGIN > 0)
    6431  {
    6432  *pOffset += VMA_DEBUG_MARGIN;
    6433  }
    6434 
    6435  // Apply alignment.
    6436  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6437 
    6438  // Check previous suballocations for BufferImageGranularity conflicts.
    6439  // Make bigger alignment if necessary.
    6440  if(bufferImageGranularity > 1)
    6441  {
    6442  bool bufferImageGranularityConflict = false;
    6443  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6444  while(prevSuballocItem != m_Suballocations.cbegin())
    6445  {
    6446  --prevSuballocItem;
    6447  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6448  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6449  {
    6450  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6451  {
    6452  bufferImageGranularityConflict = true;
    6453  break;
    6454  }
    6455  }
    6456  else
    6457  // Already on previous page.
    6458  break;
    6459  }
    6460  if(bufferImageGranularityConflict)
    6461  {
    6462  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6463  }
    6464  }
    6465 
    6466  // Calculate padding at the beginning based on current offset.
    6467  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
    6468 
    6469  // Calculate required margin at the end.
    6470  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6471 
    6472  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
    6473  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
    6474  {
    6475  return false;
    6476  }
    6477 
    6478  // Check next suballocations for BufferImageGranularity conflicts.
    6479  // If conflict exists, allocation cannot be made here.
    6480  if(bufferImageGranularity > 1)
    6481  {
    6482  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
    6483  ++nextSuballocItem;
    6484  while(nextSuballocItem != m_Suballocations.cend())
    6485  {
    6486  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6487  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6488  {
    6489  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6490  {
    6491  return false;
    6492  }
    6493  }
    6494  else
    6495  {
    6496  // Already on next page.
    6497  break;
    6498  }
    6499  ++nextSuballocItem;
    6500  }
    6501  }
    6502  }
    6503 
    6504  // All tests passed: Success. pOffset is already filled.
    6505  return true;
    6506 }
    6507 
    6508 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
    6509 {
    6510  VMA_ASSERT(item != m_Suballocations.end());
    6511  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6512 
    6513  VmaSuballocationList::iterator nextItem = item;
    6514  ++nextItem;
    6515  VMA_ASSERT(nextItem != m_Suballocations.end());
    6516  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
    6517 
    6518  item->size += nextItem->size;
    6519  --m_FreeCount;
    6520  m_Suballocations.erase(nextItem);
    6521 }
    6522 
    6523 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
    6524 {
    6525  // Change this suballocation to be marked as free.
    6526  VmaSuballocation& suballoc = *suballocItem;
    6527  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6528  suballoc.hAllocation = VK_NULL_HANDLE;
    6529 
    6530  // Update totals.
    6531  ++m_FreeCount;
    6532  m_SumFreeSize += suballoc.size;
    6533 
    6534  // Merge with previous and/or next suballocation if it's also free.
    6535  bool mergeWithNext = false;
    6536  bool mergeWithPrev = false;
    6537 
    6538  VmaSuballocationList::iterator nextItem = suballocItem;
    6539  ++nextItem;
    6540  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
    6541  {
    6542  mergeWithNext = true;
    6543  }
    6544 
    6545  VmaSuballocationList::iterator prevItem = suballocItem;
    6546  if(suballocItem != m_Suballocations.begin())
    6547  {
    6548  --prevItem;
    6549  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6550  {
    6551  mergeWithPrev = true;
    6552  }
    6553  }
    6554 
    6555  if(mergeWithNext)
    6556  {
    6557  UnregisterFreeSuballocation(nextItem);
    6558  MergeFreeWithNext(suballocItem);
    6559  }
    6560 
    6561  if(mergeWithPrev)
    6562  {
    6563  UnregisterFreeSuballocation(prevItem);
    6564  MergeFreeWithNext(prevItem);
    6565  RegisterFreeSuballocation(prevItem);
    6566  return prevItem;
    6567  }
    6568  else
    6569  {
    6570  RegisterFreeSuballocation(suballocItem);
    6571  return suballocItem;
    6572  }
    6573 }
    6574 
    6575 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
    6576 {
    6577  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6578  VMA_ASSERT(item->size > 0);
    6579 
    6580  // You may want to enable this validation at the beginning or at the end of
    6581  // this function, depending on what do you want to check.
    6582  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6583 
    6584  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6585  {
    6586  if(m_FreeSuballocationsBySize.empty())
    6587  {
    6588  m_FreeSuballocationsBySize.push_back(item);
    6589  }
    6590  else
    6591  {
    6592  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
    6593  }
    6594  }
    6595 
    6596  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6597 }
    6598 
    6599 
    6600 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
    6601 {
    6602  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6603  VMA_ASSERT(item->size > 0);
    6604 
    6605  // You may want to enable this validation at the beginning or at the end of
    6606  // this function, depending on what do you want to check.
    6607  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6608 
    6609  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6610  {
    6611  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6612  m_FreeSuballocationsBySize.data(),
    6613  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
    6614  item,
    6615  VmaSuballocationItemSizeLess());
    6616  for(size_t index = it - m_FreeSuballocationsBySize.data();
    6617  index < m_FreeSuballocationsBySize.size();
    6618  ++index)
    6619  {
    6620  if(m_FreeSuballocationsBySize[index] == item)
    6621  {
    6622  VmaVectorRemove(m_FreeSuballocationsBySize, index);
    6623  return;
    6624  }
    6625  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
    6626  }
    6627  VMA_ASSERT(0 && "Not found.");
    6628  }
    6629 
    6630  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6631 }
    6632 
    6634 // class VmaDeviceMemoryBlock
    6635 
    6636 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
    6637  m_Metadata(hAllocator),
    6638  m_MemoryTypeIndex(UINT32_MAX),
    6639  m_Id(0),
    6640  m_hMemory(VK_NULL_HANDLE),
    6641  m_MapCount(0),
    6642  m_pMappedData(VMA_NULL)
    6643 {
    6644 }
    6645 
    6646 void VmaDeviceMemoryBlock::Init(
    6647  uint32_t newMemoryTypeIndex,
    6648  VkDeviceMemory newMemory,
    6649  VkDeviceSize newSize,
    6650  uint32_t id)
    6651 {
    6652  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    6653 
    6654  m_MemoryTypeIndex = newMemoryTypeIndex;
    6655  m_Id = id;
    6656  m_hMemory = newMemory;
    6657 
    6658  m_Metadata.Init(newSize);
    6659 }
    6660 
    6661 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
    6662 {
    6663  // This is the most important assert in the entire library.
    6664  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
    6665  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
    6666 
    6667  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
    6668  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
    6669  m_hMemory = VK_NULL_HANDLE;
    6670 }
    6671 
    6672 bool VmaDeviceMemoryBlock::Validate() const
    6673 {
    6674  if((m_hMemory == VK_NULL_HANDLE) ||
    6675  (m_Metadata.GetSize() == 0))
    6676  {
    6677  return false;
    6678  }
    6679 
    6680  return m_Metadata.Validate();
    6681 }
    6682 
    6683 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
    6684 {
    6685  void* pData = nullptr;
    6686  VkResult res = Map(hAllocator, 1, &pData);
    6687  if(res != VK_SUCCESS)
    6688  {
    6689  return res;
    6690  }
    6691 
    6692  res = m_Metadata.CheckCorruption(pData);
    6693 
    6694  Unmap(hAllocator, 1);
    6695 
    6696  return res;
    6697 }
    6698 
    6699 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
    6700 {
    6701  if(count == 0)
    6702  {
    6703  return VK_SUCCESS;
    6704  }
    6705 
    6706  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6707  if(m_MapCount != 0)
    6708  {
    6709  m_MapCount += count;
    6710  VMA_ASSERT(m_pMappedData != VMA_NULL);
    6711  if(ppData != VMA_NULL)
    6712  {
    6713  *ppData = m_pMappedData;
    6714  }
    6715  return VK_SUCCESS;
    6716  }
    6717  else
    6718  {
    6719  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    6720  hAllocator->m_hDevice,
    6721  m_hMemory,
    6722  0, // offset
    6723  VK_WHOLE_SIZE,
    6724  0, // flags
    6725  &m_pMappedData);
    6726  if(result == VK_SUCCESS)
    6727  {
    6728  if(ppData != VMA_NULL)
    6729  {
    6730  *ppData = m_pMappedData;
    6731  }
    6732  m_MapCount = count;
    6733  }
    6734  return result;
    6735  }
    6736 }
    6737 
    6738 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
    6739 {
    6740  if(count == 0)
    6741  {
    6742  return;
    6743  }
    6744 
    6745  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6746  if(m_MapCount >= count)
    6747  {
    6748  m_MapCount -= count;
    6749  if(m_MapCount == 0)
    6750  {
    6751  m_pMappedData = VMA_NULL;
    6752  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
    6753  }
    6754  }
    6755  else
    6756  {
    6757  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
    6758  }
    6759 }
    6760 
    6761 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6762 {
    6763  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    6764  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    6765 
    6766  void* pData;
    6767  VkResult res = Map(hAllocator, 1, &pData);
    6768  if(res != VK_SUCCESS)
    6769  {
    6770  return res;
    6771  }
    6772 
    6773  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
    6774  VmaWriteMagicValue(pData, allocOffset + allocSize);
    6775 
    6776  Unmap(hAllocator, 1);
    6777 
    6778  return VK_SUCCESS;
    6779 }
    6780 
    6781 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6782 {
    6783  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    6784  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    6785 
    6786  void* pData;
    6787  VkResult res = Map(hAllocator, 1, &pData);
    6788  if(res != VK_SUCCESS)
    6789  {
    6790  return res;
    6791  }
    6792 
    6793  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
    6794  {
    6795  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
    6796  }
    6797  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
    6798  {
    6799  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
    6800  }
    6801 
    6802  Unmap(hAllocator, 1);
    6803 
    6804  return VK_SUCCESS;
    6805 }
    6806 
    6807 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
    6808  const VmaAllocator hAllocator,
    6809  const VmaAllocation hAllocation,
    6810  VkBuffer hBuffer)
    6811 {
    6812  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    6813  hAllocation->GetBlock() == this);
    6814  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    6815  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6816  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
    6817  hAllocator->m_hDevice,
    6818  hBuffer,
    6819  m_hMemory,
    6820  hAllocation->GetOffset());
    6821 }
    6822 
    6823 VkResult VmaDeviceMemoryBlock::BindImageMemory(
    6824  const VmaAllocator hAllocator,
    6825  const VmaAllocation hAllocation,
    6826  VkImage hImage)
    6827 {
    6828  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    6829  hAllocation->GetBlock() == this);
    6830  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    6831  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6832  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
    6833  hAllocator->m_hDevice,
    6834  hImage,
    6835  m_hMemory,
    6836  hAllocation->GetOffset());
    6837 }
    6838 
    6839 static void InitStatInfo(VmaStatInfo& outInfo)
    6840 {
    6841  memset(&outInfo, 0, sizeof(outInfo));
    6842  outInfo.allocationSizeMin = UINT64_MAX;
    6843  outInfo.unusedRangeSizeMin = UINT64_MAX;
    6844 }
    6845 
    6846 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
    6847 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
    6848 {
    6849  inoutInfo.blockCount += srcInfo.blockCount;
    6850  inoutInfo.allocationCount += srcInfo.allocationCount;
    6851  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
    6852  inoutInfo.usedBytes += srcInfo.usedBytes;
    6853  inoutInfo.unusedBytes += srcInfo.unusedBytes;
    6854  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
    6855  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
    6856  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
    6857  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
    6858 }
    6859 
    6860 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
    6861 {
    6862  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
    6863  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
    6864  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
    6865  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
    6866 }
    6867 
    6868 VmaPool_T::VmaPool_T(
    6869  VmaAllocator hAllocator,
    6870  const VmaPoolCreateInfo& createInfo) :
    6871  m_BlockVector(
    6872  hAllocator,
    6873  createInfo.memoryTypeIndex,
    6874  createInfo.blockSize,
    6875  createInfo.minBlockCount,
    6876  createInfo.maxBlockCount,
    6877  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
    6878  createInfo.frameInUseCount,
    6879  true), // isCustomPool
    6880  m_Id(0)
    6881 {
    6882 }
    6883 
    6884 VmaPool_T::~VmaPool_T()
    6885 {
    6886 }
    6887 
    6888 #if VMA_STATS_STRING_ENABLED
    6889 
    6890 #endif // #if VMA_STATS_STRING_ENABLED
    6891 
    6892 VmaBlockVector::VmaBlockVector(
    6893  VmaAllocator hAllocator,
    6894  uint32_t memoryTypeIndex,
    6895  VkDeviceSize preferredBlockSize,
    6896  size_t minBlockCount,
    6897  size_t maxBlockCount,
    6898  VkDeviceSize bufferImageGranularity,
    6899  uint32_t frameInUseCount,
    6900  bool isCustomPool) :
    6901  m_hAllocator(hAllocator),
    6902  m_MemoryTypeIndex(memoryTypeIndex),
    6903  m_PreferredBlockSize(preferredBlockSize),
    6904  m_MinBlockCount(minBlockCount),
    6905  m_MaxBlockCount(maxBlockCount),
    6906  m_BufferImageGranularity(bufferImageGranularity),
    6907  m_FrameInUseCount(frameInUseCount),
    6908  m_IsCustomPool(isCustomPool),
    6909  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
    6910  m_HasEmptyBlock(false),
    6911  m_pDefragmentator(VMA_NULL),
    6912  m_NextBlockId(0)
    6913 {
    6914 }
    6915 
    6916 VmaBlockVector::~VmaBlockVector()
    6917 {
    6918  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
    6919 
    6920  for(size_t i = m_Blocks.size(); i--; )
    6921  {
    6922  m_Blocks[i]->Destroy(m_hAllocator);
    6923  vma_delete(m_hAllocator, m_Blocks[i]);
    6924  }
    6925 }
    6926 
    6927 VkResult VmaBlockVector::CreateMinBlocks()
    6928 {
    6929  for(size_t i = 0; i < m_MinBlockCount; ++i)
    6930  {
    6931  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
    6932  if(res != VK_SUCCESS)
    6933  {
    6934  return res;
    6935  }
    6936  }
    6937  return VK_SUCCESS;
    6938 }
    6939 
    6940 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
    6941 {
    6942  pStats->size = 0;
    6943  pStats->unusedSize = 0;
    6944  pStats->allocationCount = 0;
    6945  pStats->unusedRangeCount = 0;
    6946  pStats->unusedRangeSizeMax = 0;
    6947 
    6948  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    6949 
    6950  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    6951  {
    6952  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    6953  VMA_ASSERT(pBlock);
    6954  VMA_HEAVY_ASSERT(pBlock->Validate());
    6955  pBlock->m_Metadata.AddPoolStats(*pStats);
    6956  }
    6957 }
    6958 
    6959 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
    6960 {
    6961  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    6962  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
    6963  (VMA_DEBUG_MARGIN > 0) &&
    6964  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
    6965 }
    6966 
    6967 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
    6968 
    6969 VkResult VmaBlockVector::Allocate(
    6970  VmaPool hCurrentPool,
    6971  uint32_t currentFrameIndex,
    6972  VkDeviceSize size,
    6973  VkDeviceSize alignment,
    6974  const VmaAllocationCreateInfo& createInfo,
    6975  VmaSuballocationType suballocType,
    6976  VmaAllocation* pAllocation)
    6977 {
    6978  // Early reject: requested allocation size is larger that maximum block size for this block vector.
    6979  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
    6980  {
    6981  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    6982  }
    6983 
    6984  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    6985  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
    6986 
    6987  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    6988 
    6989  // 1. Search existing allocations. Try to allocate without making other allocations lost.
    6990  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    6991  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    6992  {
    6993  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    6994  VMA_ASSERT(pCurrBlock);
    6995  VmaAllocationRequest currRequest = {};
    6996  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    6997  currentFrameIndex,
    6998  m_FrameInUseCount,
    6999  m_BufferImageGranularity,
    7000  size,
    7001  alignment,
    7002  suballocType,
    7003  false, // canMakeOtherLost
    7004  &currRequest))
    7005  {
    7006  // Allocate from pCurrBlock.
    7007  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
    7008 
    7009  if(mapped)
    7010  {
    7011  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
    7012  if(res != VK_SUCCESS)
    7013  {
    7014  return res;
    7015  }
    7016  }
    7017 
    7018  // We no longer have an empty Allocation.
    7019  if(pCurrBlock->m_Metadata.IsEmpty())
    7020  {
    7021  m_HasEmptyBlock = false;
    7022  }
    7023 
    7024  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7025  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation);
    7026  (*pAllocation)->InitBlockAllocation(
    7027  hCurrentPool,
    7028  pCurrBlock,
    7029  currRequest.offset,
    7030  alignment,
    7031  size,
    7032  suballocType,
    7033  mapped,
    7034  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7035  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
    7036  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7037  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7038  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7039  {
    7040  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7041  }
    7042  if(IsCorruptionDetectionEnabled())
    7043  {
    7044  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
    7045  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7046  }
    7047  return VK_SUCCESS;
    7048  }
    7049  }
    7050 
    7051  const bool canCreateNewBlock =
    7052  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
    7053  (m_Blocks.size() < m_MaxBlockCount);
    7054 
    7055  // 2. Try to create new block.
    7056  if(canCreateNewBlock)
    7057  {
    7058  // Calculate optimal size for new block.
    7059  VkDeviceSize newBlockSize = m_PreferredBlockSize;
    7060  uint32_t newBlockSizeShift = 0;
    7061  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
    7062 
    7063  // Allocating blocks of other sizes is allowed only in default pools.
    7064  // In custom pools block size is fixed.
    7065  if(m_IsCustomPool == false)
    7066  {
    7067  // Allocate 1/8, 1/4, 1/2 as first blocks.
    7068  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
    7069  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
    7070  {
    7071  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7072  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
    7073  {
    7074  newBlockSize = smallerNewBlockSize;
    7075  ++newBlockSizeShift;
    7076  }
    7077  else
    7078  {
    7079  break;
    7080  }
    7081  }
    7082  }
    7083 
    7084  size_t newBlockIndex = 0;
    7085  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
    7086  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
    7087  if(m_IsCustomPool == false)
    7088  {
    7089  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
    7090  {
    7091  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7092  if(smallerNewBlockSize >= size)
    7093  {
    7094  newBlockSize = smallerNewBlockSize;
    7095  ++newBlockSizeShift;
    7096  res = CreateBlock(newBlockSize, &newBlockIndex);
    7097  }
    7098  else
    7099  {
    7100  break;
    7101  }
    7102  }
    7103  }
    7104 
    7105  if(res == VK_SUCCESS)
    7106  {
    7107  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
    7108  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size);
    7109 
    7110  if(mapped)
    7111  {
    7112  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
    7113  if(res != VK_SUCCESS)
    7114  {
    7115  return res;
    7116  }
    7117  }
    7118 
    7119  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
    7120  VmaAllocationRequest allocRequest;
    7121  if(pBlock->m_Metadata.CreateAllocationRequest(
    7122  currentFrameIndex,
    7123  m_FrameInUseCount,
    7124  m_BufferImageGranularity,
    7125  size,
    7126  alignment,
    7127  suballocType,
    7128  false, // canMakeOtherLost
    7129  &allocRequest))
    7130  {
    7131  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7132  pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation);
    7133  (*pAllocation)->InitBlockAllocation(
    7134  hCurrentPool,
    7135  pBlock,
    7136  allocRequest.offset,
    7137  alignment,
    7138  size,
    7139  suballocType,
    7140  mapped,
    7141  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7142  VMA_HEAVY_ASSERT(pBlock->Validate());
    7143  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
    7144  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7145  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7146  {
    7147  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7148  }
    7149  if(IsCorruptionDetectionEnabled())
    7150  {
    7151  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
    7152  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7153  }
    7154  return VK_SUCCESS;
    7155  }
    7156  else
    7157  {
    7158  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
    7159  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7160  }
    7161  }
    7162  }
    7163 
    7164  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
    7165 
    7166  // 3. Try to allocate from existing blocks with making other allocations lost.
    7167  if(canMakeOtherLost)
    7168  {
    7169  uint32_t tryIndex = 0;
    7170  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
    7171  {
    7172  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
    7173  VmaAllocationRequest bestRequest = {};
    7174  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
    7175 
    7176  // 1. Search existing allocations.
    7177  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7178  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7179  {
    7180  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7181  VMA_ASSERT(pCurrBlock);
    7182  VmaAllocationRequest currRequest = {};
    7183  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7184  currentFrameIndex,
    7185  m_FrameInUseCount,
    7186  m_BufferImageGranularity,
    7187  size,
    7188  alignment,
    7189  suballocType,
    7190  canMakeOtherLost,
    7191  &currRequest))
    7192  {
    7193  const VkDeviceSize currRequestCost = currRequest.CalcCost();
    7194  if(pBestRequestBlock == VMA_NULL ||
    7195  currRequestCost < bestRequestCost)
    7196  {
    7197  pBestRequestBlock = pCurrBlock;
    7198  bestRequest = currRequest;
    7199  bestRequestCost = currRequestCost;
    7200 
    7201  if(bestRequestCost == 0)
    7202  {
    7203  break;
    7204  }
    7205  }
    7206  }
    7207  }
    7208 
    7209  if(pBestRequestBlock != VMA_NULL)
    7210  {
    7211  if(mapped)
    7212  {
    7213  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
    7214  if(res != VK_SUCCESS)
    7215  {
    7216  return res;
    7217  }
    7218  }
    7219 
    7220  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
    7221  currentFrameIndex,
    7222  m_FrameInUseCount,
    7223  &bestRequest))
    7224  {
    7225  // We no longer have an empty Allocation.
    7226  if(pBestRequestBlock->m_Metadata.IsEmpty())
    7227  {
    7228  m_HasEmptyBlock = false;
    7229  }
    7230  // Allocate from this pBlock.
    7231  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7232  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation);
    7233  (*pAllocation)->InitBlockAllocation(
    7234  hCurrentPool,
    7235  pBestRequestBlock,
    7236  bestRequest.offset,
    7237  alignment,
    7238  size,
    7239  suballocType,
    7240  mapped,
    7241  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7242  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
    7243  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7244  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7245  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7246  {
    7247  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7248  }
    7249  if(IsCorruptionDetectionEnabled())
    7250  {
    7251  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
    7252  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7253  }
    7254  return VK_SUCCESS;
    7255  }
    7256  // else: Some allocations must have been touched while we are here. Next try.
    7257  }
    7258  else
    7259  {
    7260  // Could not find place in any of the blocks - break outer loop.
    7261  break;
    7262  }
    7263  }
    7264  /* Maximum number of tries exceeded - a very unlike event when many other
    7265  threads are simultaneously touching allocations making it impossible to make
    7266  lost at the same time as we try to allocate. */
    7267  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
    7268  {
    7269  return VK_ERROR_TOO_MANY_OBJECTS;
    7270  }
    7271  }
    7272 
    7273  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7274 }
    7275 
    7276 void VmaBlockVector::Free(
    7277  VmaAllocation hAllocation)
    7278 {
    7279  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
    7280 
    7281  // Scope for lock.
    7282  {
    7283  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7284 
    7285  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    7286 
    7287  if(IsCorruptionDetectionEnabled())
    7288  {
    7289  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
    7290  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
    7291  }
    7292 
    7293  if(hAllocation->IsPersistentMap())
    7294  {
    7295  pBlock->Unmap(m_hAllocator, 1);
    7296  }
    7297 
    7298  pBlock->m_Metadata.Free(hAllocation);
    7299  VMA_HEAVY_ASSERT(pBlock->Validate());
    7300 
    7301  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
    7302 
    7303  // pBlock became empty after this deallocation.
    7304  if(pBlock->m_Metadata.IsEmpty())
    7305  {
    7306  // Already has empty Allocation. We don't want to have two, so delete this one.
    7307  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
    7308  {
    7309  pBlockToDelete = pBlock;
    7310  Remove(pBlock);
    7311  }
    7312  // We now have first empty block.
    7313  else
    7314  {
    7315  m_HasEmptyBlock = true;
    7316  }
    7317  }
    7318  // pBlock didn't become empty, but we have another empty block - find and free that one.
    7319  // (This is optional, heuristics.)
    7320  else if(m_HasEmptyBlock)
    7321  {
    7322  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
    7323  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
    7324  {
    7325  pBlockToDelete = pLastBlock;
    7326  m_Blocks.pop_back();
    7327  m_HasEmptyBlock = false;
    7328  }
    7329  }
    7330 
    7331  IncrementallySortBlocks();
    7332  }
    7333 
    7334  // Destruction of a free Allocation. Deferred until this point, outside of mutex
    7335  // lock, for performance reason.
    7336  if(pBlockToDelete != VMA_NULL)
    7337  {
    7338  VMA_DEBUG_LOG(" Deleted empty allocation");
    7339  pBlockToDelete->Destroy(m_hAllocator);
    7340  vma_delete(m_hAllocator, pBlockToDelete);
    7341  }
    7342 }
    7343 
    7344 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
    7345 {
    7346  VkDeviceSize result = 0;
    7347  for(size_t i = m_Blocks.size(); i--; )
    7348  {
    7349  result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize());
    7350  if(result >= m_PreferredBlockSize)
    7351  {
    7352  break;
    7353  }
    7354  }
    7355  return result;
    7356 }
    7357 
    7358 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
    7359 {
    7360  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7361  {
    7362  if(m_Blocks[blockIndex] == pBlock)
    7363  {
    7364  VmaVectorRemove(m_Blocks, blockIndex);
    7365  return;
    7366  }
    7367  }
    7368  VMA_ASSERT(0);
    7369 }
    7370 
    7371 void VmaBlockVector::IncrementallySortBlocks()
    7372 {
    7373  // Bubble sort only until first swap.
    7374  for(size_t i = 1; i < m_Blocks.size(); ++i)
    7375  {
    7376  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
    7377  {
    7378  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
    7379  return;
    7380  }
    7381  }
    7382 }
    7383 
    7384 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
    7385 {
    7386  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    7387  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
    7388  allocInfo.allocationSize = blockSize;
    7389  VkDeviceMemory mem = VK_NULL_HANDLE;
    7390  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
    7391  if(res < 0)
    7392  {
    7393  return res;
    7394  }
    7395 
    7396  // New VkDeviceMemory successfully created.
    7397 
    7398  // Create new Allocation for it.
    7399  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
    7400  pBlock->Init(
    7401  m_MemoryTypeIndex,
    7402  mem,
    7403  allocInfo.allocationSize,
    7404  m_NextBlockId++);
    7405 
    7406  m_Blocks.push_back(pBlock);
    7407  if(pNewBlockIndex != VMA_NULL)
    7408  {
    7409  *pNewBlockIndex = m_Blocks.size() - 1;
    7410  }
    7411 
    7412  return VK_SUCCESS;
    7413 }
    7414 
    7415 #if VMA_STATS_STRING_ENABLED
    7416 
    7417 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
    7418 {
    7419  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7420 
    7421  json.BeginObject();
    7422 
    7423  if(m_IsCustomPool)
    7424  {
    7425  json.WriteString("MemoryTypeIndex");
    7426  json.WriteNumber(m_MemoryTypeIndex);
    7427 
    7428  json.WriteString("BlockSize");
    7429  json.WriteNumber(m_PreferredBlockSize);
    7430 
    7431  json.WriteString("BlockCount");
    7432  json.BeginObject(true);
    7433  if(m_MinBlockCount > 0)
    7434  {
    7435  json.WriteString("Min");
    7436  json.WriteNumber((uint64_t)m_MinBlockCount);
    7437  }
    7438  if(m_MaxBlockCount < SIZE_MAX)
    7439  {
    7440  json.WriteString("Max");
    7441  json.WriteNumber((uint64_t)m_MaxBlockCount);
    7442  }
    7443  json.WriteString("Cur");
    7444  json.WriteNumber((uint64_t)m_Blocks.size());
    7445  json.EndObject();
    7446 
    7447  if(m_FrameInUseCount > 0)
    7448  {
    7449  json.WriteString("FrameInUseCount");
    7450  json.WriteNumber(m_FrameInUseCount);
    7451  }
    7452  }
    7453  else
    7454  {
    7455  json.WriteString("PreferredBlockSize");
    7456  json.WriteNumber(m_PreferredBlockSize);
    7457  }
    7458 
    7459  json.WriteString("Blocks");
    7460  json.BeginObject();
    7461  for(size_t i = 0; i < m_Blocks.size(); ++i)
    7462  {
    7463  json.BeginString();
    7464  json.ContinueString(m_Blocks[i]->GetId());
    7465  json.EndString();
    7466 
    7467  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
    7468  }
    7469  json.EndObject();
    7470 
    7471  json.EndObject();
    7472 }
    7473 
    7474 #endif // #if VMA_STATS_STRING_ENABLED
    7475 
    7476 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
    7477  VmaAllocator hAllocator,
    7478  uint32_t currentFrameIndex)
    7479 {
    7480  if(m_pDefragmentator == VMA_NULL)
    7481  {
    7482  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
    7483  hAllocator,
    7484  this,
    7485  currentFrameIndex);
    7486  }
    7487 
    7488  return m_pDefragmentator;
    7489 }
    7490 
    7491 VkResult VmaBlockVector::Defragment(
    7492  VmaDefragmentationStats* pDefragmentationStats,
    7493  VkDeviceSize& maxBytesToMove,
    7494  uint32_t& maxAllocationsToMove)
    7495 {
    7496  if(m_pDefragmentator == VMA_NULL)
    7497  {
    7498  return VK_SUCCESS;
    7499  }
    7500 
    7501  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7502 
    7503  // Defragment.
    7504  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
    7505 
    7506  // Accumulate statistics.
    7507  if(pDefragmentationStats != VMA_NULL)
    7508  {
    7509  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
    7510  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
    7511  pDefragmentationStats->bytesMoved += bytesMoved;
    7512  pDefragmentationStats->allocationsMoved += allocationsMoved;
    7513  VMA_ASSERT(bytesMoved <= maxBytesToMove);
    7514  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
    7515  maxBytesToMove -= bytesMoved;
    7516  maxAllocationsToMove -= allocationsMoved;
    7517  }
    7518 
    7519  // Free empty blocks.
    7520  m_HasEmptyBlock = false;
    7521  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
    7522  {
    7523  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
    7524  if(pBlock->m_Metadata.IsEmpty())
    7525  {
    7526  if(m_Blocks.size() > m_MinBlockCount)
    7527  {
    7528  if(pDefragmentationStats != VMA_NULL)
    7529  {
    7530  ++pDefragmentationStats->deviceMemoryBlocksFreed;
    7531  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
    7532  }
    7533 
    7534  VmaVectorRemove(m_Blocks, blockIndex);
    7535  pBlock->Destroy(m_hAllocator);
    7536  vma_delete(m_hAllocator, pBlock);
    7537  }
    7538  else
    7539  {
    7540  m_HasEmptyBlock = true;
    7541  }
    7542  }
    7543  }
    7544 
    7545  return result;
    7546 }
    7547 
    7548 void VmaBlockVector::DestroyDefragmentator()
    7549 {
    7550  if(m_pDefragmentator != VMA_NULL)
    7551  {
    7552  vma_delete(m_hAllocator, m_pDefragmentator);
    7553  m_pDefragmentator = VMA_NULL;
    7554  }
    7555 }
    7556 
    7557 void VmaBlockVector::MakePoolAllocationsLost(
    7558  uint32_t currentFrameIndex,
    7559  size_t* pLostAllocationCount)
    7560 {
    7561  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7562  size_t lostAllocationCount = 0;
    7563  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7564  {
    7565  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7566  VMA_ASSERT(pBlock);
    7567  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
    7568  }
    7569  if(pLostAllocationCount != VMA_NULL)
    7570  {
    7571  *pLostAllocationCount = lostAllocationCount;
    7572  }
    7573 }
    7574 
    7575 VkResult VmaBlockVector::CheckCorruption()
    7576 {
    7577  if(!IsCorruptionDetectionEnabled())
    7578  {
    7579  return VK_ERROR_FEATURE_NOT_PRESENT;
    7580  }
    7581 
    7582  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7583  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7584  {
    7585  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7586  VMA_ASSERT(pBlock);
    7587  VkResult res = pBlock->CheckCorruption(m_hAllocator);
    7588  if(res != VK_SUCCESS)
    7589  {
    7590  return res;
    7591  }
    7592  }
    7593  return VK_SUCCESS;
    7594 }
    7595 
    7596 void VmaBlockVector::AddStats(VmaStats* pStats)
    7597 {
    7598  const uint32_t memTypeIndex = m_MemoryTypeIndex;
    7599  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
    7600 
    7601  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7602 
    7603  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7604  {
    7605  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7606  VMA_ASSERT(pBlock);
    7607  VMA_HEAVY_ASSERT(pBlock->Validate());
    7608  VmaStatInfo allocationStatInfo;
    7609  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
    7610  VmaAddStatInfo(pStats->total, allocationStatInfo);
    7611  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    7612  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    7613  }
    7614 }
    7615 
    7617 // VmaDefragmentator members definition
    7618 
    7619 VmaDefragmentator::VmaDefragmentator(
    7620  VmaAllocator hAllocator,
    7621  VmaBlockVector* pBlockVector,
    7622  uint32_t currentFrameIndex) :
    7623  m_hAllocator(hAllocator),
    7624  m_pBlockVector(pBlockVector),
    7625  m_CurrentFrameIndex(currentFrameIndex),
    7626  m_BytesMoved(0),
    7627  m_AllocationsMoved(0),
    7628  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
    7629  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
    7630 {
    7631 }
    7632 
    7633 VmaDefragmentator::~VmaDefragmentator()
    7634 {
    7635  for(size_t i = m_Blocks.size(); i--; )
    7636  {
    7637  vma_delete(m_hAllocator, m_Blocks[i]);
    7638  }
    7639 }
    7640 
    7641 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
    7642 {
    7643  AllocationInfo allocInfo;
    7644  allocInfo.m_hAllocation = hAlloc;
    7645  allocInfo.m_pChanged = pChanged;
    7646  m_Allocations.push_back(allocInfo);
    7647 }
    7648 
    7649 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
    7650 {
    7651  // It has already been mapped for defragmentation.
    7652  if(m_pMappedDataForDefragmentation)
    7653  {
    7654  *ppMappedData = m_pMappedDataForDefragmentation;
    7655  return VK_SUCCESS;
    7656  }
    7657 
    7658  // It is originally mapped.
    7659  if(m_pBlock->GetMappedData())
    7660  {
    7661  *ppMappedData = m_pBlock->GetMappedData();
    7662  return VK_SUCCESS;
    7663  }
    7664 
    7665  // Map on first usage.
    7666  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
    7667  *ppMappedData = m_pMappedDataForDefragmentation;
    7668  return res;
    7669 }
    7670 
    7671 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
    7672 {
    7673  if(m_pMappedDataForDefragmentation != VMA_NULL)
    7674  {
    7675  m_pBlock->Unmap(hAllocator, 1);
    7676  }
    7677 }
    7678 
    7679 VkResult VmaDefragmentator::DefragmentRound(
    7680  VkDeviceSize maxBytesToMove,
    7681  uint32_t maxAllocationsToMove)
    7682 {
    7683  if(m_Blocks.empty())
    7684  {
    7685  return VK_SUCCESS;
    7686  }
    7687 
    7688  size_t srcBlockIndex = m_Blocks.size() - 1;
    7689  size_t srcAllocIndex = SIZE_MAX;
    7690  for(;;)
    7691  {
    7692  // 1. Find next allocation to move.
    7693  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
    7694  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
    7695  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
    7696  {
    7697  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
    7698  {
    7699  // Finished: no more allocations to process.
    7700  if(srcBlockIndex == 0)
    7701  {
    7702  return VK_SUCCESS;
    7703  }
    7704  else
    7705  {
    7706  --srcBlockIndex;
    7707  srcAllocIndex = SIZE_MAX;
    7708  }
    7709  }
    7710  else
    7711  {
    7712  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
    7713  }
    7714  }
    7715 
    7716  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
    7717  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
    7718 
    7719  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
    7720  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
    7721  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
    7722  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
    7723 
    7724  // 2. Try to find new place for this allocation in preceding or current block.
    7725  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
    7726  {
    7727  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
    7728  VmaAllocationRequest dstAllocRequest;
    7729  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
    7730  m_CurrentFrameIndex,
    7731  m_pBlockVector->GetFrameInUseCount(),
    7732  m_pBlockVector->GetBufferImageGranularity(),
    7733  size,
    7734  alignment,
    7735  suballocType,
    7736  false, // canMakeOtherLost
    7737  &dstAllocRequest) &&
    7738  MoveMakesSense(
    7739  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
    7740  {
    7741  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
    7742 
    7743  // Reached limit on number of allocations or bytes to move.
    7744  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
    7745  (m_BytesMoved + size > maxBytesToMove))
    7746  {
    7747  return VK_INCOMPLETE;
    7748  }
    7749 
    7750  void* pDstMappedData = VMA_NULL;
    7751  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
    7752  if(res != VK_SUCCESS)
    7753  {
    7754  return res;
    7755  }
    7756 
    7757  void* pSrcMappedData = VMA_NULL;
    7758  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
    7759  if(res != VK_SUCCESS)
    7760  {
    7761  return res;
    7762  }
    7763 
    7764  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
    7765  memcpy(
    7766  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
    7767  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
    7768  static_cast<size_t>(size));
    7769 
    7770  if(VMA_DEBUG_MARGIN > 0)
    7771  {
    7772  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
    7773  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
    7774  }
    7775 
    7776  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
    7777  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
    7778 
    7779  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
    7780 
    7781  if(allocInfo.m_pChanged != VMA_NULL)
    7782  {
    7783  *allocInfo.m_pChanged = VK_TRUE;
    7784  }
    7785 
    7786  ++m_AllocationsMoved;
    7787  m_BytesMoved += size;
    7788 
    7789  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
    7790 
    7791  break;
    7792  }
    7793  }
    7794 
    7795  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
    7796 
    7797  if(srcAllocIndex > 0)
    7798  {
    7799  --srcAllocIndex;
    7800  }
    7801  else
    7802  {
    7803  if(srcBlockIndex > 0)
    7804  {
    7805  --srcBlockIndex;
    7806  srcAllocIndex = SIZE_MAX;
    7807  }
    7808  else
    7809  {
    7810  return VK_SUCCESS;
    7811  }
    7812  }
    7813  }
    7814 }
    7815 
    7816 VkResult VmaDefragmentator::Defragment(
    7817  VkDeviceSize maxBytesToMove,
    7818  uint32_t maxAllocationsToMove)
    7819 {
    7820  if(m_Allocations.empty())
    7821  {
    7822  return VK_SUCCESS;
    7823  }
    7824 
    7825  // Create block info for each block.
    7826  const size_t blockCount = m_pBlockVector->m_Blocks.size();
    7827  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    7828  {
    7829  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
    7830  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
    7831  m_Blocks.push_back(pBlockInfo);
    7832  }
    7833 
    7834  // Sort them by m_pBlock pointer value.
    7835  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
    7836 
    7837  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
    7838  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
    7839  {
    7840  AllocationInfo& allocInfo = m_Allocations[blockIndex];
    7841  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
    7842  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    7843  {
    7844  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
    7845  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
    7846  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
    7847  {
    7848  (*it)->m_Allocations.push_back(allocInfo);
    7849  }
    7850  else
    7851  {
    7852  VMA_ASSERT(0);
    7853  }
    7854  }
    7855  }
    7856  m_Allocations.clear();
    7857 
    7858  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    7859  {
    7860  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
    7861  pBlockInfo->CalcHasNonMovableAllocations();
    7862  pBlockInfo->SortAllocationsBySizeDescecnding();
    7863  }
    7864 
    7865  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
    7866  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
    7867 
    7868  // Execute defragmentation rounds (the main part).
    7869  VkResult result = VK_SUCCESS;
    7870  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
    7871  {
    7872  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
    7873  }
    7874 
    7875  // Unmap blocks that were mapped for defragmentation.
    7876  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    7877  {
    7878  m_Blocks[blockIndex]->Unmap(m_hAllocator);
    7879  }
    7880 
    7881  return result;
    7882 }
    7883 
    7884 bool VmaDefragmentator::MoveMakesSense(
    7885  size_t dstBlockIndex, VkDeviceSize dstOffset,
    7886  size_t srcBlockIndex, VkDeviceSize srcOffset)
    7887 {
    7888  if(dstBlockIndex < srcBlockIndex)
    7889  {
    7890  return true;
    7891  }
    7892  if(dstBlockIndex > srcBlockIndex)
    7893  {
    7894  return false;
    7895  }
    7896  if(dstOffset < srcOffset)
    7897  {
    7898  return true;
    7899  }
    7900  return false;
    7901 }
    7902 
    7904 // VmaAllocator_T
    7905 
    7906 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
    7907  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
    7908  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
    7909  m_hDevice(pCreateInfo->device),
    7910  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
    7911  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
    7912  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
    7913  m_PreferredLargeHeapBlockSize(0),
    7914  m_PhysicalDevice(pCreateInfo->physicalDevice),
    7915  m_CurrentFrameIndex(0),
    7916  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
    7917  m_NextPoolId(0)
    7918 {
    7919  if(VMA_DEBUG_DETECT_CORRUPTION)
    7920  {
    7921  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
    7922  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
    7923  }
    7924 
    7925  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
    7926 
    7927 #if !(VMA_DEDICATED_ALLOCATION)
    7928  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
    7929  {
    7930  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
    7931  }
    7932 #endif
    7933 
    7934  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
    7935  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
    7936  memset(&m_MemProps, 0, sizeof(m_MemProps));
    7937 
    7938  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
    7939  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
    7940 
    7941  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    7942  {
    7943  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
    7944  }
    7945 
    7946  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
    7947  {
    7948  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
    7949  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
    7950  }
    7951 
    7952  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
    7953 
    7954  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
    7955  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
    7956 
    7957  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
    7958  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
    7959 
    7960  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
    7961  {
    7962  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
    7963  {
    7964  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
    7965  if(limit != VK_WHOLE_SIZE)
    7966  {
    7967  m_HeapSizeLimit[heapIndex] = limit;
    7968  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
    7969  {
    7970  m_MemProps.memoryHeaps[heapIndex].size = limit;
    7971  }
    7972  }
    7973  }
    7974  }
    7975 
    7976  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    7977  {
    7978  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
    7979 
    7980  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
    7981  this,
    7982  memTypeIndex,
    7983  preferredBlockSize,
    7984  0,
    7985  SIZE_MAX,
    7986  GetBufferImageGranularity(),
    7987  pCreateInfo->frameInUseCount,
    7988  false); // isCustomPool
    7989  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
    7990  // becase minBlockCount is 0.
    7991  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
    7992 
    7993  }
    7994 }
    7995 
    7996 VmaAllocator_T::~VmaAllocator_T()
    7997 {
    7998  VMA_ASSERT(m_Pools.empty());
    7999 
    8000  for(size_t i = GetMemoryTypeCount(); i--; )
    8001  {
    8002  vma_delete(this, m_pDedicatedAllocations[i]);
    8003  vma_delete(this, m_pBlockVectors[i]);
    8004  }
    8005 }
    8006 
    8007 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
    8008 {
    8009 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8010  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
    8011  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
    8012  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    8013  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
    8014  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
    8015  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
    8016  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
    8017  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
    8018  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
    8019  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
    8020  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
    8021  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
    8022  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
    8023  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
    8024  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
    8025  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
    8026 #if VMA_DEDICATED_ALLOCATION
    8027  if(m_UseKhrDedicatedAllocation)
    8028  {
    8029  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
    8030  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
    8031  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
    8032  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
    8033  }
    8034 #endif // #if VMA_DEDICATED_ALLOCATION
    8035 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8036 
    8037 #define VMA_COPY_IF_NOT_NULL(funcName) \
    8038  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
    8039 
    8040  if(pVulkanFunctions != VMA_NULL)
    8041  {
    8042  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
    8043  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
    8044  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
    8045  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
    8046  VMA_COPY_IF_NOT_NULL(vkMapMemory);
    8047  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
    8048  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
    8049  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
    8050  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
    8051  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
    8052  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
    8053  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
    8054  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
    8055  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
    8056  VMA_COPY_IF_NOT_NULL(vkCreateImage);
    8057  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
    8058 #if VMA_DEDICATED_ALLOCATION
    8059  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
    8060  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
    8061 #endif
    8062  }
    8063 
    8064 #undef VMA_COPY_IF_NOT_NULL
    8065 
    8066  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
    8067  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
    8068  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
    8069  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
    8070  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
    8071  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
    8072  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
    8073  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
    8074  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
    8075  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
    8076  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
    8077  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
    8078  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
    8079  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
    8080  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
    8081  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
    8082  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
    8083  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
    8084 #if VMA_DEDICATED_ALLOCATION
    8085  if(m_UseKhrDedicatedAllocation)
    8086  {
    8087  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
    8088  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
    8089  }
    8090 #endif
    8091 }
    8092 
    8093 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
    8094 {
    8095  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    8096  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
    8097  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
    8098  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
    8099 }
    8100 
    8101 VkResult VmaAllocator_T::AllocateMemoryOfType(
    8102  VkDeviceSize size,
    8103  VkDeviceSize alignment,
    8104  bool dedicatedAllocation,
    8105  VkBuffer dedicatedBuffer,
    8106  VkImage dedicatedImage,
    8107  const VmaAllocationCreateInfo& createInfo,
    8108  uint32_t memTypeIndex,
    8109  VmaSuballocationType suballocType,
    8110  VmaAllocation* pAllocation)
    8111 {
    8112  VMA_ASSERT(pAllocation != VMA_NULL);
    8113  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
    8114 
    8115  VmaAllocationCreateInfo finalCreateInfo = createInfo;
    8116 
    8117  // If memory type is not HOST_VISIBLE, disable MAPPED.
    8118  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8119  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    8120  {
    8121  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
    8122  }
    8123 
    8124  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
    8125  VMA_ASSERT(blockVector);
    8126 
    8127  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
    8128  bool preferDedicatedMemory =
    8129  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
    8130  dedicatedAllocation ||
    8131  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
    8132  size > preferredBlockSize / 2;
    8133 
    8134  if(preferDedicatedMemory &&
    8135  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
    8136  finalCreateInfo.pool == VK_NULL_HANDLE)
    8137  {
    8138  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
    8139  }
    8140 
    8141  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
    8142  {
    8143  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8144  {
    8145  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8146  }
    8147  else
    8148  {
    8149  return AllocateDedicatedMemory(
    8150  size,
    8151  suballocType,
    8152  memTypeIndex,
    8153  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8154  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8155  finalCreateInfo.pUserData,
    8156  dedicatedBuffer,
    8157  dedicatedImage,
    8158  pAllocation);
    8159  }
    8160  }
    8161  else
    8162  {
    8163  VkResult res = blockVector->Allocate(
    8164  VK_NULL_HANDLE, // hCurrentPool
    8165  m_CurrentFrameIndex.load(),
    8166  size,
    8167  alignment,
    8168  finalCreateInfo,
    8169  suballocType,
    8170  pAllocation);
    8171  if(res == VK_SUCCESS)
    8172  {
    8173  return res;
    8174  }
    8175 
    8176  // 5. Try dedicated memory.
    8177  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8178  {
    8179  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8180  }
    8181  else
    8182  {
    8183  res = AllocateDedicatedMemory(
    8184  size,
    8185  suballocType,
    8186  memTypeIndex,
    8187  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8188  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8189  finalCreateInfo.pUserData,
    8190  dedicatedBuffer,
    8191  dedicatedImage,
    8192  pAllocation);
    8193  if(res == VK_SUCCESS)
    8194  {
    8195  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
    8196  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
    8197  return VK_SUCCESS;
    8198  }
    8199  else
    8200  {
    8201  // Everything failed: Return error code.
    8202  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8203  return res;
    8204  }
    8205  }
    8206  }
    8207 }
    8208 
    8209 VkResult VmaAllocator_T::AllocateDedicatedMemory(
    8210  VkDeviceSize size,
    8211  VmaSuballocationType suballocType,
    8212  uint32_t memTypeIndex,
    8213  bool map,
    8214  bool isUserDataString,
    8215  void* pUserData,
    8216  VkBuffer dedicatedBuffer,
    8217  VkImage dedicatedImage,
    8218  VmaAllocation* pAllocation)
    8219 {
    8220  VMA_ASSERT(pAllocation);
    8221 
    8222  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    8223  allocInfo.memoryTypeIndex = memTypeIndex;
    8224  allocInfo.allocationSize = size;
    8225 
    8226 #if VMA_DEDICATED_ALLOCATION
    8227  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
    8228  if(m_UseKhrDedicatedAllocation)
    8229  {
    8230  if(dedicatedBuffer != VK_NULL_HANDLE)
    8231  {
    8232  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
    8233  dedicatedAllocInfo.buffer = dedicatedBuffer;
    8234  allocInfo.pNext = &dedicatedAllocInfo;
    8235  }
    8236  else if(dedicatedImage != VK_NULL_HANDLE)
    8237  {
    8238  dedicatedAllocInfo.image = dedicatedImage;
    8239  allocInfo.pNext = &dedicatedAllocInfo;
    8240  }
    8241  }
    8242 #endif // #if VMA_DEDICATED_ALLOCATION
    8243 
    8244  // Allocate VkDeviceMemory.
    8245  VkDeviceMemory hMemory = VK_NULL_HANDLE;
    8246  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
    8247  if(res < 0)
    8248  {
    8249  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8250  return res;
    8251  }
    8252 
    8253  void* pMappedData = VMA_NULL;
    8254  if(map)
    8255  {
    8256  res = (*m_VulkanFunctions.vkMapMemory)(
    8257  m_hDevice,
    8258  hMemory,
    8259  0,
    8260  VK_WHOLE_SIZE,
    8261  0,
    8262  &pMappedData);
    8263  if(res < 0)
    8264  {
    8265  VMA_DEBUG_LOG(" vkMapMemory FAILED");
    8266  FreeVulkanMemory(memTypeIndex, size, hMemory);
    8267  return res;
    8268  }
    8269  }
    8270 
    8271  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
    8272  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
    8273  (*pAllocation)->SetUserData(this, pUserData);
    8274  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    8275  {
    8276  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    8277  }
    8278 
    8279  // Register it in m_pDedicatedAllocations.
    8280  {
    8281  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    8282  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    8283  VMA_ASSERT(pDedicatedAllocations);
    8284  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
    8285  }
    8286 
    8287  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
    8288 
    8289  return VK_SUCCESS;
    8290 }
    8291 
    8292 void VmaAllocator_T::GetBufferMemoryRequirements(
    8293  VkBuffer hBuffer,
    8294  VkMemoryRequirements& memReq,
    8295  bool& requiresDedicatedAllocation,
    8296  bool& prefersDedicatedAllocation) const
    8297 {
    8298 #if VMA_DEDICATED_ALLOCATION
    8299  if(m_UseKhrDedicatedAllocation)
    8300  {
    8301  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
    8302  memReqInfo.buffer = hBuffer;
    8303 
    8304  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    8305 
    8306  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    8307  memReq2.pNext = &memDedicatedReq;
    8308 
    8309  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    8310 
    8311  memReq = memReq2.memoryRequirements;
    8312  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    8313  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    8314  }
    8315  else
    8316 #endif // #if VMA_DEDICATED_ALLOCATION
    8317  {
    8318  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
    8319  requiresDedicatedAllocation = false;
    8320  prefersDedicatedAllocation = false;
    8321  }
    8322 }
    8323 
    8324 void VmaAllocator_T::GetImageMemoryRequirements(
    8325  VkImage hImage,
    8326  VkMemoryRequirements& memReq,
    8327  bool& requiresDedicatedAllocation,
    8328  bool& prefersDedicatedAllocation) const
    8329 {
    8330 #if VMA_DEDICATED_ALLOCATION
    8331  if(m_UseKhrDedicatedAllocation)
    8332  {
    8333  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
    8334  memReqInfo.image = hImage;
    8335 
    8336  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    8337 
    8338  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    8339  memReq2.pNext = &memDedicatedReq;
    8340 
    8341  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    8342 
    8343  memReq = memReq2.memoryRequirements;
    8344  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    8345  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    8346  }
    8347  else
    8348 #endif // #if VMA_DEDICATED_ALLOCATION
    8349  {
    8350  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
    8351  requiresDedicatedAllocation = false;
    8352  prefersDedicatedAllocation = false;
    8353  }
    8354 }
    8355 
    8356 VkResult VmaAllocator_T::AllocateMemory(
    8357  const VkMemoryRequirements& vkMemReq,
    8358  bool requiresDedicatedAllocation,
    8359  bool prefersDedicatedAllocation,
    8360  VkBuffer dedicatedBuffer,
    8361  VkImage dedicatedImage,
    8362  const VmaAllocationCreateInfo& createInfo,
    8363  VmaSuballocationType suballocType,
    8364  VmaAllocation* pAllocation)
    8365 {
    8366  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
    8367  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8368  {
    8369  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
    8370  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8371  }
    8372  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8373  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
    8374  {
    8375  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
    8376  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8377  }
    8378  if(requiresDedicatedAllocation)
    8379  {
    8380  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8381  {
    8382  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
    8383  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8384  }
    8385  if(createInfo.pool != VK_NULL_HANDLE)
    8386  {
    8387  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
    8388  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8389  }
    8390  }
    8391  if((createInfo.pool != VK_NULL_HANDLE) &&
    8392  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
    8393  {
    8394  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
    8395  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8396  }
    8397 
    8398  if(createInfo.pool != VK_NULL_HANDLE)
    8399  {
    8400  const VkDeviceSize alignmentForPool = VMA_MAX(
    8401  vkMemReq.alignment,
    8402  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
    8403  return createInfo.pool->m_BlockVector.Allocate(
    8404  createInfo.pool,
    8405  m_CurrentFrameIndex.load(),
    8406  vkMemReq.size,
    8407  alignmentForPool,
    8408  createInfo,
    8409  suballocType,
    8410  pAllocation);
    8411  }
    8412  else
    8413  {
    8414  // Bit mask of memory Vulkan types acceptable for this allocation.
    8415  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
    8416  uint32_t memTypeIndex = UINT32_MAX;
    8417  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    8418  if(res == VK_SUCCESS)
    8419  {
    8420  VkDeviceSize alignmentForMemType = VMA_MAX(
    8421  vkMemReq.alignment,
    8422  GetMemoryTypeMinAlignment(memTypeIndex));
    8423 
    8424  res = AllocateMemoryOfType(
    8425  vkMemReq.size,
    8426  alignmentForMemType,
    8427  requiresDedicatedAllocation || prefersDedicatedAllocation,
    8428  dedicatedBuffer,
    8429  dedicatedImage,
    8430  createInfo,
    8431  memTypeIndex,
    8432  suballocType,
    8433  pAllocation);
    8434  // Succeeded on first try.
    8435  if(res == VK_SUCCESS)
    8436  {
    8437  return res;
    8438  }
    8439  // Allocation from this memory type failed. Try other compatible memory types.
    8440  else
    8441  {
    8442  for(;;)
    8443  {
    8444  // Remove old memTypeIndex from list of possibilities.
    8445  memoryTypeBits &= ~(1u << memTypeIndex);
    8446  // Find alternative memTypeIndex.
    8447  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    8448  if(res == VK_SUCCESS)
    8449  {
    8450  alignmentForMemType = VMA_MAX(
    8451  vkMemReq.alignment,
    8452  GetMemoryTypeMinAlignment(memTypeIndex));
    8453 
    8454  res = AllocateMemoryOfType(
    8455  vkMemReq.size,
    8456  alignmentForMemType,
    8457  requiresDedicatedAllocation || prefersDedicatedAllocation,
    8458  dedicatedBuffer,
    8459  dedicatedImage,
    8460  createInfo,
    8461  memTypeIndex,
    8462  suballocType,
    8463  pAllocation);
    8464  // Allocation from this alternative memory type succeeded.
    8465  if(res == VK_SUCCESS)
    8466  {
    8467  return res;
    8468  }
    8469  // else: Allocation from this memory type failed. Try next one - next loop iteration.
    8470  }
    8471  // No other matching memory type index could be found.
    8472  else
    8473  {
    8474  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
    8475  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8476  }
    8477  }
    8478  }
    8479  }
    8480  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
    8481  else
    8482  return res;
    8483  }
    8484 }
    8485 
    8486 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
    8487 {
    8488  VMA_ASSERT(allocation);
    8489 
    8490  if(allocation->CanBecomeLost() == false ||
    8491  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    8492  {
    8493  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    8494  {
    8495  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
    8496  }
    8497 
    8498  switch(allocation->GetType())
    8499  {
    8500  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    8501  {
    8502  VmaBlockVector* pBlockVector = VMA_NULL;
    8503  VmaPool hPool = allocation->GetPool();
    8504  if(hPool != VK_NULL_HANDLE)
    8505  {
    8506  pBlockVector = &hPool->m_BlockVector;
    8507  }
    8508  else
    8509  {
    8510  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    8511  pBlockVector = m_pBlockVectors[memTypeIndex];
    8512  }
    8513  pBlockVector->Free(allocation);
    8514  }
    8515  break;
    8516  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    8517  FreeDedicatedMemory(allocation);
    8518  break;
    8519  default:
    8520  VMA_ASSERT(0);
    8521  }
    8522  }
    8523 
    8524  allocation->SetUserData(this, VMA_NULL);
    8525  vma_delete(this, allocation);
    8526 }
    8527 
    8528 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
    8529 {
    8530  // Initialize.
    8531  InitStatInfo(pStats->total);
    8532  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
    8533  InitStatInfo(pStats->memoryType[i]);
    8534  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    8535  InitStatInfo(pStats->memoryHeap[i]);
    8536 
    8537  // Process default pools.
    8538  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    8539  {
    8540  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    8541  VMA_ASSERT(pBlockVector);
    8542  pBlockVector->AddStats(pStats);
    8543  }
    8544 
    8545  // Process custom pools.
    8546  {
    8547  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    8548  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    8549  {
    8550  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
    8551  }
    8552  }
    8553 
    8554  // Process dedicated allocations.
    8555  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    8556  {
    8557  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    8558  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    8559  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    8560  VMA_ASSERT(pDedicatedAllocVector);
    8561  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
    8562  {
    8563  VmaStatInfo allocationStatInfo;
    8564  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
    8565  VmaAddStatInfo(pStats->total, allocationStatInfo);
    8566  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    8567  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    8568  }
    8569  }
    8570 
    8571  // Postprocess.
    8572  VmaPostprocessCalcStatInfo(pStats->total);
    8573  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
    8574  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
    8575  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
    8576  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
    8577 }
    8578 
    8579 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
    8580 
    8581 VkResult VmaAllocator_T::Defragment(
    8582  VmaAllocation* pAllocations,
    8583  size_t allocationCount,
    8584  VkBool32* pAllocationsChanged,
    8585  const VmaDefragmentationInfo* pDefragmentationInfo,
    8586  VmaDefragmentationStats* pDefragmentationStats)
    8587 {
    8588  if(pAllocationsChanged != VMA_NULL)
    8589  {
    8590  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
    8591  }
    8592  if(pDefragmentationStats != VMA_NULL)
    8593  {
    8594  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
    8595  }
    8596 
    8597  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
    8598 
    8599  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
    8600 
    8601  const size_t poolCount = m_Pools.size();
    8602 
    8603  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
    8604  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
    8605  {
    8606  VmaAllocation hAlloc = pAllocations[allocIndex];
    8607  VMA_ASSERT(hAlloc);
    8608  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
    8609  // DedicatedAlloc cannot be defragmented.
    8610  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
    8611  // Only HOST_VISIBLE memory types can be defragmented.
    8612  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
    8613  // Lost allocation cannot be defragmented.
    8614  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
    8615  {
    8616  VmaBlockVector* pAllocBlockVector = VMA_NULL;
    8617 
    8618  const VmaPool hAllocPool = hAlloc->GetPool();
    8619  // This allocation belongs to custom pool.
    8620  if(hAllocPool != VK_NULL_HANDLE)
    8621  {
    8622  pAllocBlockVector = &hAllocPool->GetBlockVector();
    8623  }
    8624  // This allocation belongs to general pool.
    8625  else
    8626  {
    8627  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
    8628  }
    8629 
    8630  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
    8631 
    8632  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
    8633  &pAllocationsChanged[allocIndex] : VMA_NULL;
    8634  pDefragmentator->AddAllocation(hAlloc, pChanged);
    8635  }
    8636  }
    8637 
    8638  VkResult result = VK_SUCCESS;
    8639 
    8640  // ======== Main processing.
    8641 
    8642  VkDeviceSize maxBytesToMove = SIZE_MAX;
    8643  uint32_t maxAllocationsToMove = UINT32_MAX;
    8644  if(pDefragmentationInfo != VMA_NULL)
    8645  {
    8646  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
    8647  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
    8648  }
    8649 
    8650  // Process standard memory.
    8651  for(uint32_t memTypeIndex = 0;
    8652  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
    8653  ++memTypeIndex)
    8654  {
    8655  // Only HOST_VISIBLE memory types can be defragmented.
    8656  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    8657  {
    8658  result = m_pBlockVectors[memTypeIndex]->Defragment(
    8659  pDefragmentationStats,
    8660  maxBytesToMove,
    8661  maxAllocationsToMove);
    8662  }
    8663  }
    8664 
    8665  // Process custom pools.
    8666  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
    8667  {
    8668  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
    8669  pDefragmentationStats,
    8670  maxBytesToMove,
    8671  maxAllocationsToMove);
    8672  }
    8673 
    8674  // ======== Destroy defragmentators.
    8675 
    8676  // Process custom pools.
    8677  for(size_t poolIndex = poolCount; poolIndex--; )
    8678  {
    8679  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
    8680  }
    8681 
    8682  // Process standard memory.
    8683  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
    8684  {
    8685  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    8686  {
    8687  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
    8688  }
    8689  }
    8690 
    8691  return result;
    8692 }
    8693 
    8694 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
    8695 {
    8696  if(hAllocation->CanBecomeLost())
    8697  {
    8698  /*
    8699  Warning: This is a carefully designed algorithm.
    8700  Do not modify unless you really know what you're doing :)
    8701  */
    8702  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    8703  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    8704  for(;;)
    8705  {
    8706  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    8707  {
    8708  pAllocationInfo->memoryType = UINT32_MAX;
    8709  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
    8710  pAllocationInfo->offset = 0;
    8711  pAllocationInfo->size = hAllocation->GetSize();
    8712  pAllocationInfo->pMappedData = VMA_NULL;
    8713  pAllocationInfo->pUserData = hAllocation->GetUserData();
    8714  return;
    8715  }
    8716  else if(localLastUseFrameIndex == localCurrFrameIndex)
    8717  {
    8718  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    8719  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    8720  pAllocationInfo->offset = hAllocation->GetOffset();
    8721  pAllocationInfo->size = hAllocation->GetSize();
    8722  pAllocationInfo->pMappedData = VMA_NULL;
    8723  pAllocationInfo->pUserData = hAllocation->GetUserData();
    8724  return;
    8725  }
    8726  else // Last use time earlier than current time.
    8727  {
    8728  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    8729  {
    8730  localLastUseFrameIndex = localCurrFrameIndex;
    8731  }
    8732  }
    8733  }
    8734  }
    8735  else
    8736  {
    8737 #if VMA_STATS_STRING_ENABLED
    8738  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    8739  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    8740  for(;;)
    8741  {
    8742  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    8743  if(localLastUseFrameIndex == localCurrFrameIndex)
    8744  {
    8745  break;
    8746  }
    8747  else // Last use time earlier than current time.
    8748  {
    8749  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    8750  {
    8751  localLastUseFrameIndex = localCurrFrameIndex;
    8752  }
    8753  }
    8754  }
    8755 #endif
    8756 
    8757  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    8758  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    8759  pAllocationInfo->offset = hAllocation->GetOffset();
    8760  pAllocationInfo->size = hAllocation->GetSize();
    8761  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
    8762  pAllocationInfo->pUserData = hAllocation->GetUserData();
    8763  }
    8764 }
    8765 
    8766 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
    8767 {
    8768  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
    8769  if(hAllocation->CanBecomeLost())
    8770  {
    8771  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    8772  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    8773  for(;;)
    8774  {
    8775  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    8776  {
    8777  return false;
    8778  }
    8779  else if(localLastUseFrameIndex == localCurrFrameIndex)
    8780  {
    8781  return true;
    8782  }
    8783  else // Last use time earlier than current time.
    8784  {
    8785  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    8786  {
    8787  localLastUseFrameIndex = localCurrFrameIndex;
    8788  }
    8789  }
    8790  }
    8791  }
    8792  else
    8793  {
    8794 #if VMA_STATS_STRING_ENABLED
    8795  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    8796  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    8797  for(;;)
    8798  {
    8799  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    8800  if(localLastUseFrameIndex == localCurrFrameIndex)
    8801  {
    8802  break;
    8803  }
    8804  else // Last use time earlier than current time.
    8805  {
    8806  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    8807  {
    8808  localLastUseFrameIndex = localCurrFrameIndex;
    8809  }
    8810  }
    8811  }
    8812 #endif
    8813 
    8814  return true;
    8815  }
    8816 }
    8817 
    8818 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
    8819 {
    8820  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
    8821 
    8822  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
    8823 
    8824  if(newCreateInfo.maxBlockCount == 0)
    8825  {
    8826  newCreateInfo.maxBlockCount = SIZE_MAX;
    8827  }
    8828  if(newCreateInfo.blockSize == 0)
    8829  {
    8830  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
    8831  }
    8832 
    8833  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
    8834 
    8835  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
    8836  if(res != VK_SUCCESS)
    8837  {
    8838  vma_delete(this, *pPool);
    8839  *pPool = VMA_NULL;
    8840  return res;
    8841  }
    8842 
    8843  // Add to m_Pools.
    8844  {
    8845  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    8846  (*pPool)->SetId(m_NextPoolId++);
    8847  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
    8848  }
    8849 
    8850  return VK_SUCCESS;
    8851 }
    8852 
    8853 void VmaAllocator_T::DestroyPool(VmaPool pool)
    8854 {
    8855  // Remove from m_Pools.
    8856  {
    8857  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    8858  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
    8859  VMA_ASSERT(success && "Pool not found in Allocator.");
    8860  }
    8861 
    8862  vma_delete(this, pool);
    8863 }
    8864 
    8865 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
    8866 {
    8867  pool->m_BlockVector.GetPoolStats(pPoolStats);
    8868 }
    8869 
    8870 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
    8871 {
    8872  m_CurrentFrameIndex.store(frameIndex);
    8873 }
    8874 
    8875 void VmaAllocator_T::MakePoolAllocationsLost(
    8876  VmaPool hPool,
    8877  size_t* pLostAllocationCount)
    8878 {
    8879  hPool->m_BlockVector.MakePoolAllocationsLost(
    8880  m_CurrentFrameIndex.load(),
    8881  pLostAllocationCount);
    8882 }
    8883 
    8884 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
    8885 {
    8886  return hPool->m_BlockVector.CheckCorruption();
    8887 }
    8888 
    8889 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
    8890 {
    8891  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
    8892 
    8893  // Process default pools.
    8894  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    8895  {
    8896  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
    8897  {
    8898  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    8899  VMA_ASSERT(pBlockVector);
    8900  VkResult localRes = pBlockVector->CheckCorruption();
    8901  switch(localRes)
    8902  {
    8903  case VK_ERROR_FEATURE_NOT_PRESENT:
    8904  break;
    8905  case VK_SUCCESS:
    8906  finalRes = VK_SUCCESS;
    8907  break;
    8908  default:
    8909  return localRes;
    8910  }
    8911  }
    8912  }
    8913 
    8914  // Process custom pools.
    8915  {
    8916  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    8917  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    8918  {
    8919  if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0)
    8920  {
    8921  VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption();
    8922  switch(localRes)
    8923  {
    8924  case VK_ERROR_FEATURE_NOT_PRESENT:
    8925  break;
    8926  case VK_SUCCESS:
    8927  finalRes = VK_SUCCESS;
    8928  break;
    8929  default:
    8930  return localRes;
    8931  }
    8932  }
    8933  }
    8934  }
    8935 
    8936  return finalRes;
    8937 }
    8938 
    8939 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
    8940 {
    8941  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
    8942  (*pAllocation)->InitLost();
    8943 }
    8944 
    8945 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
    8946 {
    8947  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
    8948 
    8949  VkResult res;
    8950  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    8951  {
    8952  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    8953  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
    8954  {
    8955  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    8956  if(res == VK_SUCCESS)
    8957  {
    8958  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
    8959  }
    8960  }
    8961  else
    8962  {
    8963  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8964  }
    8965  }
    8966  else
    8967  {
    8968  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    8969  }
    8970 
    8971  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
    8972  {
    8973  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
    8974  }
    8975 
    8976  return res;
    8977 }
    8978 
    8979 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
    8980 {
    8981  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
    8982  {
    8983  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
    8984  }
    8985 
    8986  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
    8987 
    8988  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
    8989  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    8990  {
    8991  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    8992  m_HeapSizeLimit[heapIndex] += size;
    8993  }
    8994 }
    8995 
    8996 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
    8997 {
    8998  if(hAllocation->CanBecomeLost())
    8999  {
    9000  return VK_ERROR_MEMORY_MAP_FAILED;
    9001  }
    9002 
    9003  switch(hAllocation->GetType())
    9004  {
    9005  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9006  {
    9007  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9008  char *pBytes = VMA_NULL;
    9009  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
    9010  if(res == VK_SUCCESS)
    9011  {
    9012  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
    9013  hAllocation->BlockAllocMap();
    9014  }
    9015  return res;
    9016  }
    9017  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9018  return hAllocation->DedicatedAllocMap(this, ppData);
    9019  default:
    9020  VMA_ASSERT(0);
    9021  return VK_ERROR_MEMORY_MAP_FAILED;
    9022  }
    9023 }
    9024 
    9025 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
    9026 {
    9027  switch(hAllocation->GetType())
    9028  {
    9029  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9030  {
    9031  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9032  hAllocation->BlockAllocUnmap();
    9033  pBlock->Unmap(this, 1);
    9034  }
    9035  break;
    9036  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9037  hAllocation->DedicatedAllocUnmap(this);
    9038  break;
    9039  default:
    9040  VMA_ASSERT(0);
    9041  }
    9042 }
    9043 
    9044 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
    9045 {
    9046  VkResult res = VK_SUCCESS;
    9047  switch(hAllocation->GetType())
    9048  {
    9049  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9050  res = GetVulkanFunctions().vkBindBufferMemory(
    9051  m_hDevice,
    9052  hBuffer,
    9053  hAllocation->GetMemory(),
    9054  0); //memoryOffset
    9055  break;
    9056  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9057  {
    9058  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9059  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
    9060  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
    9061  break;
    9062  }
    9063  default:
    9064  VMA_ASSERT(0);
    9065  }
    9066  return res;
    9067 }
    9068 
    9069 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
    9070 {
    9071  VkResult res = VK_SUCCESS;
    9072  switch(hAllocation->GetType())
    9073  {
    9074  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9075  res = GetVulkanFunctions().vkBindImageMemory(
    9076  m_hDevice,
    9077  hImage,
    9078  hAllocation->GetMemory(),
    9079  0); //memoryOffset
    9080  break;
    9081  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9082  {
    9083  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9084  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
    9085  res = pBlock->BindImageMemory(this, hAllocation, hImage);
    9086  break;
    9087  }
    9088  default:
    9089  VMA_ASSERT(0);
    9090  }
    9091  return res;
    9092 }
    9093 
    9094 void VmaAllocator_T::FlushOrInvalidateAllocation(
    9095  VmaAllocation hAllocation,
    9096  VkDeviceSize offset, VkDeviceSize size,
    9097  VMA_CACHE_OPERATION op)
    9098 {
    9099  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
    9100  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
    9101  {
    9102  const VkDeviceSize allocationSize = hAllocation->GetSize();
    9103  VMA_ASSERT(offset <= allocationSize);
    9104 
    9105  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
    9106 
    9107  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
    9108  memRange.memory = hAllocation->GetMemory();
    9109 
    9110  switch(hAllocation->GetType())
    9111  {
    9112  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9113  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9114  if(size == VK_WHOLE_SIZE)
    9115  {
    9116  memRange.size = allocationSize - memRange.offset;
    9117  }
    9118  else
    9119  {
    9120  VMA_ASSERT(offset + size <= allocationSize);
    9121  memRange.size = VMA_MIN(
    9122  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
    9123  allocationSize - memRange.offset);
    9124  }
    9125  break;
    9126 
    9127  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9128  {
    9129  // 1. Still within this allocation.
    9130  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9131  if(size == VK_WHOLE_SIZE)
    9132  {
    9133  size = allocationSize - offset;
    9134  }
    9135  else
    9136  {
    9137  VMA_ASSERT(offset + size <= allocationSize);
    9138  }
    9139  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
    9140 
    9141  // 2. Adjust to whole block.
    9142  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
    9143  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
    9144  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize();
    9145  memRange.offset += allocationOffset;
    9146  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
    9147 
    9148  break;
    9149  }
    9150 
    9151  default:
    9152  VMA_ASSERT(0);
    9153  }
    9154 
    9155  switch(op)
    9156  {
    9157  case VMA_CACHE_FLUSH:
    9158  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9159  break;
    9160  case VMA_CACHE_INVALIDATE:
    9161  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9162  break;
    9163  default:
    9164  VMA_ASSERT(0);
    9165  }
    9166  }
    9167  // else: Just ignore this call.
    9168 }
    9169 
    9170 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
    9171 {
    9172  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
    9173 
    9174  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9175  {
    9176  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9177  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    9178  VMA_ASSERT(pDedicatedAllocations);
    9179  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
    9180  VMA_ASSERT(success);
    9181  }
    9182 
    9183  VkDeviceMemory hMemory = allocation->GetMemory();
    9184 
    9185  if(allocation->GetMappedData() != VMA_NULL)
    9186  {
    9187  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
    9188  }
    9189 
    9190  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
    9191 
    9192  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
    9193 }
    9194 
    9195 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
    9196 {
    9197  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
    9198  !hAllocation->CanBecomeLost() &&
    9199  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9200  {
    9201  void* pData = VMA_NULL;
    9202  VkResult res = Map(hAllocation, &pData);
    9203  if(res == VK_SUCCESS)
    9204  {
    9205  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
    9206  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
    9207  Unmap(hAllocation);
    9208  }
    9209  else
    9210  {
    9211  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
    9212  }
    9213  }
    9214 }
    9215 
    9216 #if VMA_STATS_STRING_ENABLED
    9217 
    9218 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
    9219 {
    9220  bool dedicatedAllocationsStarted = false;
    9221  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9222  {
    9223  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9224  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9225  VMA_ASSERT(pDedicatedAllocVector);
    9226  if(pDedicatedAllocVector->empty() == false)
    9227  {
    9228  if(dedicatedAllocationsStarted == false)
    9229  {
    9230  dedicatedAllocationsStarted = true;
    9231  json.WriteString("DedicatedAllocations");
    9232  json.BeginObject();
    9233  }
    9234 
    9235  json.BeginString("Type ");
    9236  json.ContinueString(memTypeIndex);
    9237  json.EndString();
    9238 
    9239  json.BeginArray();
    9240 
    9241  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
    9242  {
    9243  json.BeginObject(true);
    9244  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
    9245  hAlloc->PrintParameters(json);
    9246  json.EndObject();
    9247  }
    9248 
    9249  json.EndArray();
    9250  }
    9251  }
    9252  if(dedicatedAllocationsStarted)
    9253  {
    9254  json.EndObject();
    9255  }
    9256 
    9257  {
    9258  bool allocationsStarted = false;
    9259  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9260  {
    9261  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
    9262  {
    9263  if(allocationsStarted == false)
    9264  {
    9265  allocationsStarted = true;
    9266  json.WriteString("DefaultPools");
    9267  json.BeginObject();
    9268  }
    9269 
    9270  json.BeginString("Type ");
    9271  json.ContinueString(memTypeIndex);
    9272  json.EndString();
    9273 
    9274  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
    9275  }
    9276  }
    9277  if(allocationsStarted)
    9278  {
    9279  json.EndObject();
    9280  }
    9281  }
    9282 
    9283  {
    9284  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9285  const size_t poolCount = m_Pools.size();
    9286  if(poolCount > 0)
    9287  {
    9288  json.WriteString("Pools");
    9289  json.BeginObject();
    9290  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
    9291  {
    9292  json.BeginString();
    9293  json.ContinueString(m_Pools[poolIndex]->GetId());
    9294  json.EndString();
    9295 
    9296  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
    9297  }
    9298  json.EndObject();
    9299  }
    9300  }
    9301 }
    9302 
    9303 #endif // #if VMA_STATS_STRING_ENABLED
    9304 
    9305 static VkResult AllocateMemoryForImage(
    9306  VmaAllocator allocator,
    9307  VkImage image,
    9308  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    9309  VmaSuballocationType suballocType,
    9310  VmaAllocation* pAllocation)
    9311 {
    9312  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
    9313 
    9314  VkMemoryRequirements vkMemReq = {};
    9315  bool requiresDedicatedAllocation = false;
    9316  bool prefersDedicatedAllocation = false;
    9317  allocator->GetImageMemoryRequirements(image, vkMemReq,
    9318  requiresDedicatedAllocation, prefersDedicatedAllocation);
    9319 
    9320  return allocator->AllocateMemory(
    9321  vkMemReq,
    9322  requiresDedicatedAllocation,
    9323  prefersDedicatedAllocation,
    9324  VK_NULL_HANDLE, // dedicatedBuffer
    9325  image, // dedicatedImage
    9326  *pAllocationCreateInfo,
    9327  suballocType,
    9328  pAllocation);
    9329 }
    9330 
    9332 // Public interface
    9333 
    9334 VkResult vmaCreateAllocator(
    9335  const VmaAllocatorCreateInfo* pCreateInfo,
    9336  VmaAllocator* pAllocator)
    9337 {
    9338  VMA_ASSERT(pCreateInfo && pAllocator);
    9339  VMA_DEBUG_LOG("vmaCreateAllocator");
    9340  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
    9341  return VK_SUCCESS;
    9342 }
    9343 
    9344 void vmaDestroyAllocator(
    9345  VmaAllocator allocator)
    9346 {
    9347  if(allocator != VK_NULL_HANDLE)
    9348  {
    9349  VMA_DEBUG_LOG("vmaDestroyAllocator");
    9350  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
    9351  vma_delete(&allocationCallbacks, allocator);
    9352  }
    9353 }
    9354 
    9355 void vmaGetPhysicalDeviceProperties(
    9356  VmaAllocator allocator,
    9357  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
    9358 {
    9359  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
    9360  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
    9361 }
    9362 
    9363 void vmaGetMemoryProperties(
    9364  VmaAllocator allocator,
    9365  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
    9366 {
    9367  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
    9368  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
    9369 }
    9370 
    9371 void vmaGetMemoryTypeProperties(
    9372  VmaAllocator allocator,
    9373  uint32_t memoryTypeIndex,
    9374  VkMemoryPropertyFlags* pFlags)
    9375 {
    9376  VMA_ASSERT(allocator && pFlags);
    9377  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
    9378  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
    9379 }
    9380 
    9381 void vmaSetCurrentFrameIndex(
    9382  VmaAllocator allocator,
    9383  uint32_t frameIndex)
    9384 {
    9385  VMA_ASSERT(allocator);
    9386  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
    9387 
    9388  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9389 
    9390  allocator->SetCurrentFrameIndex(frameIndex);
    9391 }
    9392 
    9393 void vmaCalculateStats(
    9394  VmaAllocator allocator,
    9395  VmaStats* pStats)
    9396 {
    9397  VMA_ASSERT(allocator && pStats);
    9398  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9399  allocator->CalculateStats(pStats);
    9400 }
    9401 
    9402 #if VMA_STATS_STRING_ENABLED
    9403 
    9404 void vmaBuildStatsString(
    9405  VmaAllocator allocator,
    9406  char** ppStatsString,
    9407  VkBool32 detailedMap)
    9408 {
    9409  VMA_ASSERT(allocator && ppStatsString);
    9410  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9411 
    9412  VmaStringBuilder sb(allocator);
    9413  {
    9414  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
    9415  json.BeginObject();
    9416 
    9417  VmaStats stats;
    9418  allocator->CalculateStats(&stats);
    9419 
    9420  json.WriteString("Total");
    9421  VmaPrintStatInfo(json, stats.total);
    9422 
    9423  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
    9424  {
    9425  json.BeginString("Heap ");
    9426  json.ContinueString(heapIndex);
    9427  json.EndString();
    9428  json.BeginObject();
    9429 
    9430  json.WriteString("Size");
    9431  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
    9432 
    9433  json.WriteString("Flags");
    9434  json.BeginArray(true);
    9435  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
    9436  {
    9437  json.WriteString("DEVICE_LOCAL");
    9438  }
    9439  json.EndArray();
    9440 
    9441  if(stats.memoryHeap[heapIndex].blockCount > 0)
    9442  {
    9443  json.WriteString("Stats");
    9444  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
    9445  }
    9446 
    9447  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
    9448  {
    9449  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
    9450  {
    9451  json.BeginString("Type ");
    9452  json.ContinueString(typeIndex);
    9453  json.EndString();
    9454 
    9455  json.BeginObject();
    9456 
    9457  json.WriteString("Flags");
    9458  json.BeginArray(true);
    9459  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
    9460  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
    9461  {
    9462  json.WriteString("DEVICE_LOCAL");
    9463  }
    9464  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9465  {
    9466  json.WriteString("HOST_VISIBLE");
    9467  }
    9468  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
    9469  {
    9470  json.WriteString("HOST_COHERENT");
    9471  }
    9472  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
    9473  {
    9474  json.WriteString("HOST_CACHED");
    9475  }
    9476  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
    9477  {
    9478  json.WriteString("LAZILY_ALLOCATED");
    9479  }
    9480  json.EndArray();
    9481 
    9482  if(stats.memoryType[typeIndex].blockCount > 0)
    9483  {
    9484  json.WriteString("Stats");
    9485  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
    9486  }
    9487 
    9488  json.EndObject();
    9489  }
    9490  }
    9491 
    9492  json.EndObject();
    9493  }
    9494  if(detailedMap == VK_TRUE)
    9495  {
    9496  allocator->PrintDetailedMap(json);
    9497  }
    9498 
    9499  json.EndObject();
    9500  }
    9501 
    9502  const size_t len = sb.GetLength();
    9503  char* const pChars = vma_new_array(allocator, char, len + 1);
    9504  if(len > 0)
    9505  {
    9506  memcpy(pChars, sb.GetData(), len);
    9507  }
    9508  pChars[len] = '\0';
    9509  *ppStatsString = pChars;
    9510 }
    9511 
    9512 void vmaFreeStatsString(
    9513  VmaAllocator allocator,
    9514  char* pStatsString)
    9515 {
    9516  if(pStatsString != VMA_NULL)
    9517  {
    9518  VMA_ASSERT(allocator);
    9519  size_t len = strlen(pStatsString);
    9520  vma_delete_array(allocator, pStatsString, len + 1);
    9521  }
    9522 }
    9523 
    9524 #endif // #if VMA_STATS_STRING_ENABLED
    9525 
    9526 /*
    9527 This function is not protected by any mutex because it just reads immutable data.
    9528 */
    9529 VkResult vmaFindMemoryTypeIndex(
    9530  VmaAllocator allocator,
    9531  uint32_t memoryTypeBits,
    9532  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    9533  uint32_t* pMemoryTypeIndex)
    9534 {
    9535  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    9536  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    9537  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    9538 
    9539  if(pAllocationCreateInfo->memoryTypeBits != 0)
    9540  {
    9541  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
    9542  }
    9543 
    9544  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
    9545  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
    9546 
    9547  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    9548  if(mapped)
    9549  {
    9550  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    9551  }
    9552 
    9553  // Convert usage to requiredFlags and preferredFlags.
    9554  switch(pAllocationCreateInfo->usage)
    9555  {
    9556  case VMA_MEMORY_USAGE_UNKNOWN:
    9557  break;
    9558  case VMA_MEMORY_USAGE_GPU_ONLY:
    9559  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    9560  {
    9561  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    9562  }
    9563  break;
    9564  case VMA_MEMORY_USAGE_CPU_ONLY:
    9565  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    9566  break;
    9567  case VMA_MEMORY_USAGE_CPU_TO_GPU:
    9568  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    9569  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    9570  {
    9571  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    9572  }
    9573  break;
    9574  case VMA_MEMORY_USAGE_GPU_TO_CPU:
    9575  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    9576  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
    9577  break;
    9578  default:
    9579  break;
    9580  }
    9581 
    9582  *pMemoryTypeIndex = UINT32_MAX;
    9583  uint32_t minCost = UINT32_MAX;
    9584  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
    9585  memTypeIndex < allocator->GetMemoryTypeCount();
    9586  ++memTypeIndex, memTypeBit <<= 1)
    9587  {
    9588  // This memory type is acceptable according to memoryTypeBits bitmask.
    9589  if((memTypeBit & memoryTypeBits) != 0)
    9590  {
    9591  const VkMemoryPropertyFlags currFlags =
    9592  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
    9593  // This memory type contains requiredFlags.
    9594  if((requiredFlags & ~currFlags) == 0)
    9595  {
    9596  // Calculate cost as number of bits from preferredFlags not present in this memory type.
    9597  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
    9598  // Remember memory type with lowest cost.
    9599  if(currCost < minCost)
    9600  {
    9601  *pMemoryTypeIndex = memTypeIndex;
    9602  if(currCost == 0)
    9603  {
    9604  return VK_SUCCESS;
    9605  }
    9606  minCost = currCost;
    9607  }
    9608  }
    9609  }
    9610  }
    9611  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
    9612 }
    9613 
    9614 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    9615  VmaAllocator allocator,
    9616  const VkBufferCreateInfo* pBufferCreateInfo,
    9617  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    9618  uint32_t* pMemoryTypeIndex)
    9619 {
    9620  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    9621  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
    9622  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    9623  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    9624 
    9625  const VkDevice hDev = allocator->m_hDevice;
    9626  VkBuffer hBuffer = VK_NULL_HANDLE;
    9627  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
    9628  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
    9629  if(res == VK_SUCCESS)
    9630  {
    9631  VkMemoryRequirements memReq = {};
    9632  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
    9633  hDev, hBuffer, &memReq);
    9634 
    9635  res = vmaFindMemoryTypeIndex(
    9636  allocator,
    9637  memReq.memoryTypeBits,
    9638  pAllocationCreateInfo,
    9639  pMemoryTypeIndex);
    9640 
    9641  allocator->GetVulkanFunctions().vkDestroyBuffer(
    9642  hDev, hBuffer, allocator->GetAllocationCallbacks());
    9643  }
    9644  return res;
    9645 }
    9646 
    9647 VkResult vmaFindMemoryTypeIndexForImageInfo(
    9648  VmaAllocator allocator,
    9649  const VkImageCreateInfo* pImageCreateInfo,
    9650  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    9651  uint32_t* pMemoryTypeIndex)
    9652 {
    9653  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    9654  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
    9655  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    9656  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    9657 
    9658  const VkDevice hDev = allocator->m_hDevice;
    9659  VkImage hImage = VK_NULL_HANDLE;
    9660  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
    9661  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
    9662  if(res == VK_SUCCESS)
    9663  {
    9664  VkMemoryRequirements memReq = {};
    9665  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
    9666  hDev, hImage, &memReq);
    9667 
    9668  res = vmaFindMemoryTypeIndex(
    9669  allocator,
    9670  memReq.memoryTypeBits,
    9671  pAllocationCreateInfo,
    9672  pMemoryTypeIndex);
    9673 
    9674  allocator->GetVulkanFunctions().vkDestroyImage(
    9675  hDev, hImage, allocator->GetAllocationCallbacks());
    9676  }
    9677  return res;
    9678 }
    9679 
    9680 VkResult vmaCreatePool(
    9681  VmaAllocator allocator,
    9682  const VmaPoolCreateInfo* pCreateInfo,
    9683  VmaPool* pPool)
    9684 {
    9685  VMA_ASSERT(allocator && pCreateInfo && pPool);
    9686 
    9687  VMA_DEBUG_LOG("vmaCreatePool");
    9688 
    9689  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9690 
    9691  return allocator->CreatePool(pCreateInfo, pPool);
    9692 }
    9693 
    9694 void vmaDestroyPool(
    9695  VmaAllocator allocator,
    9696  VmaPool pool)
    9697 {
    9698  VMA_ASSERT(allocator);
    9699 
    9700  if(pool == VK_NULL_HANDLE)
    9701  {
    9702  return;
    9703  }
    9704 
    9705  VMA_DEBUG_LOG("vmaDestroyPool");
    9706 
    9707  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9708 
    9709  allocator->DestroyPool(pool);
    9710 }
    9711 
    9712 void vmaGetPoolStats(
    9713  VmaAllocator allocator,
    9714  VmaPool pool,
    9715  VmaPoolStats* pPoolStats)
    9716 {
    9717  VMA_ASSERT(allocator && pool && pPoolStats);
    9718 
    9719  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9720 
    9721  allocator->GetPoolStats(pool, pPoolStats);
    9722 }
    9723 
    9724 void vmaMakePoolAllocationsLost(
    9725  VmaAllocator allocator,
    9726  VmaPool pool,
    9727  size_t* pLostAllocationCount)
    9728 {
    9729  VMA_ASSERT(allocator && pool);
    9730 
    9731  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9732 
    9733  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
    9734 }
    9735 
    9736 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
    9737 {
    9738  VMA_ASSERT(allocator && pool);
    9739 
    9740  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9741 
    9742  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
    9743 
    9744  return allocator->CheckPoolCorruption(pool);
    9745 }
    9746 
    9747 VkResult vmaAllocateMemory(
    9748  VmaAllocator allocator,
    9749  const VkMemoryRequirements* pVkMemoryRequirements,
    9750  const VmaAllocationCreateInfo* pCreateInfo,
    9751  VmaAllocation* pAllocation,
    9752  VmaAllocationInfo* pAllocationInfo)
    9753 {
    9754  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
    9755 
    9756  VMA_DEBUG_LOG("vmaAllocateMemory");
    9757 
    9758  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9759 
    9760  VkResult result = allocator->AllocateMemory(
    9761  *pVkMemoryRequirements,
    9762  false, // requiresDedicatedAllocation
    9763  false, // prefersDedicatedAllocation
    9764  VK_NULL_HANDLE, // dedicatedBuffer
    9765  VK_NULL_HANDLE, // dedicatedImage
    9766  *pCreateInfo,
    9767  VMA_SUBALLOCATION_TYPE_UNKNOWN,
    9768  pAllocation);
    9769 
    9770  if(pAllocationInfo && result == VK_SUCCESS)
    9771  {
    9772  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    9773  }
    9774 
    9775  return result;
    9776 }
    9777 
    9778 VkResult vmaAllocateMemoryForBuffer(
    9779  VmaAllocator allocator,
    9780  VkBuffer buffer,
    9781  const VmaAllocationCreateInfo* pCreateInfo,
    9782  VmaAllocation* pAllocation,
    9783  VmaAllocationInfo* pAllocationInfo)
    9784 {
    9785  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    9786 
    9787  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
    9788 
    9789  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9790 
    9791  VkMemoryRequirements vkMemReq = {};
    9792  bool requiresDedicatedAllocation = false;
    9793  bool prefersDedicatedAllocation = false;
    9794  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
    9795  requiresDedicatedAllocation,
    9796  prefersDedicatedAllocation);
    9797 
    9798  VkResult result = allocator->AllocateMemory(
    9799  vkMemReq,
    9800  requiresDedicatedAllocation,
    9801  prefersDedicatedAllocation,
    9802  buffer, // dedicatedBuffer
    9803  VK_NULL_HANDLE, // dedicatedImage
    9804  *pCreateInfo,
    9805  VMA_SUBALLOCATION_TYPE_BUFFER,
    9806  pAllocation);
    9807 
    9808  if(pAllocationInfo && result == VK_SUCCESS)
    9809  {
    9810  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    9811  }
    9812 
    9813  return result;
    9814 }
    9815 
    9816 VkResult vmaAllocateMemoryForImage(
    9817  VmaAllocator allocator,
    9818  VkImage image,
    9819  const VmaAllocationCreateInfo* pCreateInfo,
    9820  VmaAllocation* pAllocation,
    9821  VmaAllocationInfo* pAllocationInfo)
    9822 {
    9823  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    9824 
    9825  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
    9826 
    9827  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9828 
    9829  VkResult result = AllocateMemoryForImage(
    9830  allocator,
    9831  image,
    9832  pCreateInfo,
    9833  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
    9834  pAllocation);
    9835 
    9836  if(pAllocationInfo && result == VK_SUCCESS)
    9837  {
    9838  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    9839  }
    9840 
    9841  return result;
    9842 }
    9843 
    9844 void vmaFreeMemory(
    9845  VmaAllocator allocator,
    9846  VmaAllocation allocation)
    9847 {
    9848  VMA_ASSERT(allocator);
    9849  VMA_DEBUG_LOG("vmaFreeMemory");
    9850  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9851  if(allocation != VK_NULL_HANDLE)
    9852  {
    9853  allocator->FreeMemory(allocation);
    9854  }
    9855 }
    9856 
    9857 void vmaGetAllocationInfo(
    9858  VmaAllocator allocator,
    9859  VmaAllocation allocation,
    9860  VmaAllocationInfo* pAllocationInfo)
    9861 {
    9862  VMA_ASSERT(allocator && allocation && pAllocationInfo);
    9863 
    9864  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9865 
    9866  allocator->GetAllocationInfo(allocation, pAllocationInfo);
    9867 }
    9868 
    9869 VkBool32 vmaTouchAllocation(
    9870  VmaAllocator allocator,
    9871  VmaAllocation allocation)
    9872 {
    9873  VMA_ASSERT(allocator && allocation);
    9874 
    9875  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9876 
    9877  return allocator->TouchAllocation(allocation);
    9878 }
    9879 
    9880 void vmaSetAllocationUserData(
    9881  VmaAllocator allocator,
    9882  VmaAllocation allocation,
    9883  void* pUserData)
    9884 {
    9885  VMA_ASSERT(allocator && allocation);
    9886 
    9887  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9888 
    9889  allocation->SetUserData(allocator, pUserData);
    9890 }
    9891 
    9892 void vmaCreateLostAllocation(
    9893  VmaAllocator allocator,
    9894  VmaAllocation* pAllocation)
    9895 {
    9896  VMA_ASSERT(allocator && pAllocation);
    9897 
    9898  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
    9899 
    9900  allocator->CreateLostAllocation(pAllocation);
    9901 }
    9902 
    9903 VkResult vmaMapMemory(
    9904  VmaAllocator allocator,
    9905  VmaAllocation allocation,
    9906  void** ppData)
    9907 {
    9908  VMA_ASSERT(allocator && allocation && ppData);
    9909 
    9910  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9911 
    9912  return allocator->Map(allocation, ppData);
    9913 }
    9914 
    9915 void vmaUnmapMemory(
    9916  VmaAllocator allocator,
    9917  VmaAllocation allocation)
    9918 {
    9919  VMA_ASSERT(allocator && allocation);
    9920 
    9921  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9922 
    9923  allocator->Unmap(allocation);
    9924 }
    9925 
    9926 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    9927 {
    9928  VMA_ASSERT(allocator && allocation);
    9929 
    9930  VMA_DEBUG_LOG("vmaFlushAllocation");
    9931 
    9932  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9933 
    9934  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
    9935 }
    9936 
    9937 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    9938 {
    9939  VMA_ASSERT(allocator && allocation);
    9940 
    9941  VMA_DEBUG_LOG("vmaInvalidateAllocation");
    9942 
    9943  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9944 
    9945  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
    9946 }
    9947 
    9948 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
    9949 {
    9950  VMA_ASSERT(allocator);
    9951 
    9952  VMA_DEBUG_LOG("vmaCheckCorruption");
    9953 
    9954  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9955 
    9956  return allocator->CheckCorruption(memoryTypeBits);
    9957 }
    9958 
    9959 VkResult vmaDefragment(
    9960  VmaAllocator allocator,
    9961  VmaAllocation* pAllocations,
    9962  size_t allocationCount,
    9963  VkBool32* pAllocationsChanged,
    9964  const VmaDefragmentationInfo *pDefragmentationInfo,
    9965  VmaDefragmentationStats* pDefragmentationStats)
    9966 {
    9967  VMA_ASSERT(allocator && pAllocations);
    9968 
    9969  VMA_DEBUG_LOG("vmaDefragment");
    9970 
    9971  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9972 
    9973  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
    9974 }
    9975 
    9976 VkResult vmaBindBufferMemory(
    9977  VmaAllocator allocator,
    9978  VmaAllocation allocation,
    9979  VkBuffer buffer)
    9980 {
    9981  VMA_ASSERT(allocator && allocation && buffer);
    9982 
    9983  VMA_DEBUG_LOG("vmaBindBufferMemory");
    9984 
    9985  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9986 
    9987  return allocator->BindBufferMemory(allocation, buffer);
    9988 }
    9989 
    9990 VkResult vmaBindImageMemory(
    9991  VmaAllocator allocator,
    9992  VmaAllocation allocation,
    9993  VkImage image)
    9994 {
    9995  VMA_ASSERT(allocator && allocation && image);
    9996 
    9997  VMA_DEBUG_LOG("vmaBindImageMemory");
    9998 
    9999  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10000 
    10001  return allocator->BindImageMemory(allocation, image);
    10002 }
    10003 
    10004 VkResult vmaCreateBuffer(
    10005  VmaAllocator allocator,
    10006  const VkBufferCreateInfo* pBufferCreateInfo,
    10007  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10008  VkBuffer* pBuffer,
    10009  VmaAllocation* pAllocation,
    10010  VmaAllocationInfo* pAllocationInfo)
    10011 {
    10012  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
    10013 
    10014  VMA_DEBUG_LOG("vmaCreateBuffer");
    10015 
    10016  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10017 
    10018  *pBuffer = VK_NULL_HANDLE;
    10019  *pAllocation = VK_NULL_HANDLE;
    10020 
    10021  // 1. Create VkBuffer.
    10022  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
    10023  allocator->m_hDevice,
    10024  pBufferCreateInfo,
    10025  allocator->GetAllocationCallbacks(),
    10026  pBuffer);
    10027  if(res >= 0)
    10028  {
    10029  // 2. vkGetBufferMemoryRequirements.
    10030  VkMemoryRequirements vkMemReq = {};
    10031  bool requiresDedicatedAllocation = false;
    10032  bool prefersDedicatedAllocation = false;
    10033  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
    10034  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10035 
    10036  // Make sure alignment requirements for specific buffer usages reported
    10037  // in Physical Device Properties are included in alignment reported by memory requirements.
    10038  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
    10039  {
    10040  VMA_ASSERT(vkMemReq.alignment %
    10041  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
    10042  }
    10043  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
    10044  {
    10045  VMA_ASSERT(vkMemReq.alignment %
    10046  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
    10047  }
    10048  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
    10049  {
    10050  VMA_ASSERT(vkMemReq.alignment %
    10051  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
    10052  }
    10053 
    10054  // 3. Allocate memory using allocator.
    10055  res = allocator->AllocateMemory(
    10056  vkMemReq,
    10057  requiresDedicatedAllocation,
    10058  prefersDedicatedAllocation,
    10059  *pBuffer, // dedicatedBuffer
    10060  VK_NULL_HANDLE, // dedicatedImage
    10061  *pAllocationCreateInfo,
    10062  VMA_SUBALLOCATION_TYPE_BUFFER,
    10063  pAllocation);
    10064  if(res >= 0)
    10065  {
    10066  // 3. Bind buffer with memory.
    10067  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
    10068  if(res >= 0)
    10069  {
    10070  // All steps succeeded.
    10071  #if VMA_STATS_STRING_ENABLED
    10072  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
    10073  #endif
    10074  if(pAllocationInfo != VMA_NULL)
    10075  {
    10076  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10077  }
    10078  return VK_SUCCESS;
    10079  }
    10080  allocator->FreeMemory(*pAllocation);
    10081  *pAllocation = VK_NULL_HANDLE;
    10082  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10083  *pBuffer = VK_NULL_HANDLE;
    10084  return res;
    10085  }
    10086  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10087  *pBuffer = VK_NULL_HANDLE;
    10088  return res;
    10089  }
    10090  return res;
    10091 }
    10092 
    10093 void vmaDestroyBuffer(
    10094  VmaAllocator allocator,
    10095  VkBuffer buffer,
    10096  VmaAllocation allocation)
    10097 {
    10098  VMA_ASSERT(allocator);
    10099  VMA_DEBUG_LOG("vmaDestroyBuffer");
    10100  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10101  if(buffer != VK_NULL_HANDLE)
    10102  {
    10103  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
    10104  }
    10105  if(allocation != VK_NULL_HANDLE)
    10106  {
    10107  allocator->FreeMemory(allocation);
    10108  }
    10109 }
    10110 
    10111 VkResult vmaCreateImage(
    10112  VmaAllocator allocator,
    10113  const VkImageCreateInfo* pImageCreateInfo,
    10114  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10115  VkImage* pImage,
    10116  VmaAllocation* pAllocation,
    10117  VmaAllocationInfo* pAllocationInfo)
    10118 {
    10119  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
    10120 
    10121  VMA_DEBUG_LOG("vmaCreateImage");
    10122 
    10123  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10124 
    10125  *pImage = VK_NULL_HANDLE;
    10126  *pAllocation = VK_NULL_HANDLE;
    10127 
    10128  // 1. Create VkImage.
    10129  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
    10130  allocator->m_hDevice,
    10131  pImageCreateInfo,
    10132  allocator->GetAllocationCallbacks(),
    10133  pImage);
    10134  if(res >= 0)
    10135  {
    10136  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
    10137  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
    10138  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
    10139 
    10140  // 2. Allocate memory using allocator.
    10141  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
    10142  if(res >= 0)
    10143  {
    10144  // 3. Bind image with memory.
    10145  res = allocator->BindImageMemory(*pAllocation, *pImage);
    10146  if(res >= 0)
    10147  {
    10148  // All steps succeeded.
    10149  #if VMA_STATS_STRING_ENABLED
    10150  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
    10151  #endif
    10152  if(pAllocationInfo != VMA_NULL)
    10153  {
    10154  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10155  }
    10156  return VK_SUCCESS;
    10157  }
    10158  allocator->FreeMemory(*pAllocation);
    10159  *pAllocation = VK_NULL_HANDLE;
    10160  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    10161  *pImage = VK_NULL_HANDLE;
    10162  return res;
    10163  }
    10164  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    10165  *pImage = VK_NULL_HANDLE;
    10166  return res;
    10167  }
    10168  return res;
    10169 }
    10170 
    10171 void vmaDestroyImage(
    10172  VmaAllocator allocator,
    10173  VkImage image,
    10174  VmaAllocation allocation)
    10175 {
    10176  VMA_ASSERT(allocator);
    10177  VMA_DEBUG_LOG("vmaDestroyImage");
    10178  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10179  if(image != VK_NULL_HANDLE)
    10180  {
    10181  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
    10182  }
    10183  if(allocation != VK_NULL_HANDLE)
    10184  {
    10185  allocator->FreeMemory(allocation);
    10186  }
    10187 }
    10188 
    10189 #endif // #ifdef VMA_IMPLEMENTATION
    VmaVulkanFunctions::vkGetPhysicalDeviceProperties
    PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
    Definition: vk_mem_alloc.h:1271
    -
    VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
    Set this flag if the allocation should have its own memory block.
    Definition: vk_mem_alloc.h:1537
    +Go to the documentation of this file.
    1 //
    2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
    3 //
    4 // Permission is hereby granted, free of charge, to any person obtaining a copy
    5 // of this software and associated documentation files (the "Software"), to deal
    6 // in the Software without restriction, including without limitation the rights
    7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    8 // copies of the Software, and to permit persons to whom the Software is
    9 // furnished to do so, subject to the following conditions:
    10 //
    11 // The above copyright notice and this permission notice shall be included in
    12 // all copies or substantial portions of the Software.
    13 //
    14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    20 // THE SOFTWARE.
    21 //
    22 
    23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
    24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
    25 
    26 #ifdef __cplusplus
    27 extern "C" {
    28 #endif
    29 
    1211 #include <vulkan/vulkan.h>
    1212 
    1213 #if !defined(VMA_DEDICATED_ALLOCATION)
    1214  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
    1215  #define VMA_DEDICATED_ALLOCATION 1
    1216  #else
    1217  #define VMA_DEDICATED_ALLOCATION 0
    1218  #endif
    1219 #endif
    1220 
    1230 VK_DEFINE_HANDLE(VmaAllocator)
    1231 
    1232 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
    1234  VmaAllocator allocator,
    1235  uint32_t memoryType,
    1236  VkDeviceMemory memory,
    1237  VkDeviceSize size);
    1239 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
    1240  VmaAllocator allocator,
    1241  uint32_t memoryType,
    1242  VkDeviceMemory memory,
    1243  VkDeviceSize size);
    1244 
    1252 typedef struct VmaDeviceMemoryCallbacks {
    1254  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
    1256  PFN_vmaFreeDeviceMemoryFunction pfnFree;
    1257 } VmaDeviceMemoryCallbacks;
    1258 
    1260 typedef enum VmaAllocatorCreateFlagBits {
    1265  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
    1287  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
    1288 
    1289  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1290 } VmaAllocatorCreateFlagBits;
    1291 typedef VkFlags VmaAllocatorCreateFlags;
    1292 
    1297 typedef struct VmaVulkanFunctions {
    1298  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
    1299  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
    1300  PFN_vkAllocateMemory vkAllocateMemory;
    1301  PFN_vkFreeMemory vkFreeMemory;
    1302  PFN_vkMapMemory vkMapMemory;
    1303  PFN_vkUnmapMemory vkUnmapMemory;
    1304  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
    1305  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
    1306  PFN_vkBindBufferMemory vkBindBufferMemory;
    1307  PFN_vkBindImageMemory vkBindImageMemory;
    1308  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
    1309  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
    1310  PFN_vkCreateBuffer vkCreateBuffer;
    1311  PFN_vkDestroyBuffer vkDestroyBuffer;
    1312  PFN_vkCreateImage vkCreateImage;
    1313  PFN_vkDestroyImage vkDestroyImage;
    1314 #if VMA_DEDICATED_ALLOCATION
    1315  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
    1316  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
    1317 #endif
    1318 } VmaVulkanFunctions;
    1319 
    1321 typedef enum VmaRecordFlagBits {
    1327  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
    1328 
    1329  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1330 } VmaRecordFlagBits;
    1331 typedef VkFlags VmaRecordFlags;
    1332 
    1333 /*
    1334 Define this macro to 0/1 to disable/enable support for recording functionality,
    1335 available through VmaAllocatorCreateInfo::pRecordSettings.
    1336 */
    1337 #ifndef VMA_RECORDING_ENABLED
    1338  #ifdef _WIN32
    1339  #define VMA_RECORDING_ENABLED 1
    1340  #else
    1341  #define VMA_RECORDING_ENABLED 0
    1342  #endif
    1343 #endif
    1344 
    1346 typedef struct VmaRecordSettings
    1347 {
    1349  VmaRecordFlags flags;
    1357  const char* pFilePath;
    1358 } VmaRecordSettings;
    1359 
    1361 typedef struct VmaAllocatorCreateInfo
    1362 {
    1364  VmaAllocatorCreateFlags flags;
    1366 
    1367  VkPhysicalDevice physicalDevice;
    1369 
    1370  VkDevice device;
    1372 
    1373  VkDeviceSize preferredLargeHeapBlockSize;
    1375 
    1376  const VkAllocationCallbacks* pAllocationCallbacks;
    1378 
    1379  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
    1393  uint32_t frameInUseCount;
    1417  const VkDeviceSize* pHeapSizeLimit;
    1429  const VmaVulkanFunctions* pVulkanFunctions;
    1436  const VmaRecordSettings* pRecordSettings;
    1437 } VmaAllocatorCreateInfo;
    1438 
    1440 VkResult vmaCreateAllocator(
    1441  const VmaAllocatorCreateInfo* pCreateInfo,
    1442  VmaAllocator* pAllocator);
    1443 
    1445 void vmaDestroyAllocator(
    1446  VmaAllocator allocator);
    1447 
    1452 void vmaGetPhysicalDeviceProperties(
    1453  VmaAllocator allocator,
    1454  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
    1455 
    1460 void vmaGetMemoryProperties(
    1461  VmaAllocator allocator,
    1462  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
    1463 
    1470 void vmaGetMemoryTypeProperties(
    1471  VmaAllocator allocator,
    1472  uint32_t memoryTypeIndex,
    1473  VkMemoryPropertyFlags* pFlags);
    1474 
    1483 void vmaSetCurrentFrameIndex(
    1484  VmaAllocator allocator,
    1485  uint32_t frameIndex);
    1486 
    1489 typedef struct VmaStatInfo
    1490 {
    1492  uint32_t blockCount;
    1494  uint32_t allocationCount;
    1496  uint32_t unusedRangeCount;
    1498  VkDeviceSize usedBytes;
    1500  VkDeviceSize unusedBytes;
    1501  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
    1502  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
    1503 } VmaStatInfo;
    1504 
    1506 typedef struct VmaStats
    1507 {
    1508  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
    1509  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
    1510  VmaStatInfo total;
    1511 } VmaStats;
    1512 
    1514 void vmaCalculateStats(
    1515  VmaAllocator allocator,
    1516  VmaStats* pStats);
    1517 
    1518 #define VMA_STATS_STRING_ENABLED 1
    1519 
    1520 #if VMA_STATS_STRING_ENABLED
    1521 
    1523 
    1525 void vmaBuildStatsString(
    1526  VmaAllocator allocator,
    1527  char** ppStatsString,
    1528  VkBool32 detailedMap);
    1529 
    1530 void vmaFreeStatsString(
    1531  VmaAllocator allocator,
    1532  char* pStatsString);
    1533 
    1534 #endif // #if VMA_STATS_STRING_ENABLED
    1535 
    1544 VK_DEFINE_HANDLE(VmaPool)
    1545 
    1546 typedef enum VmaMemoryUsage
    1547 {
    1551  VMA_MEMORY_USAGE_UNKNOWN = 0,
    1568  VMA_MEMORY_USAGE_GPU_ONLY = 1,
    1578  VMA_MEMORY_USAGE_CPU_ONLY = 2,
    1585  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
    1594  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
    1595  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
    1596 } VmaMemoryUsage;
    1597 
    1599 typedef enum VmaAllocationCreateFlagBits {
    1611  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
    1612 
    1622  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
    1635  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
    1648  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
    1655  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
    1661  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
    1662 
    1663  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1664 } VmaAllocationCreateFlagBits;
    1665 typedef VkFlags VmaAllocationCreateFlags;
    1666 
    1667 typedef struct VmaAllocationCreateInfo
    1668 {
    1670  VmaAllocationCreateFlags flags;
    1676  VmaMemoryUsage usage;
    1681  VkMemoryPropertyFlags requiredFlags;
    1686  VkMemoryPropertyFlags preferredFlags;
    1694  uint32_t memoryTypeBits;
    1700  VmaPool pool;
    1707  void* pUserData;
    1708 } VmaAllocationCreateInfo;
    1709 
    1726 VkResult vmaFindMemoryTypeIndex(
    1727  VmaAllocator allocator,
    1728  uint32_t memoryTypeBits,
    1729  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1730  uint32_t* pMemoryTypeIndex);
    1731 
    1744 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    1745  VmaAllocator allocator,
    1746  const VkBufferCreateInfo* pBufferCreateInfo,
    1747  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1748  uint32_t* pMemoryTypeIndex);
    1749 
    1762 VkResult vmaFindMemoryTypeIndexForImageInfo(
    1763  VmaAllocator allocator,
    1764  const VkImageCreateInfo* pImageCreateInfo,
    1765  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1766  uint32_t* pMemoryTypeIndex);
    1767 
    1769 typedef enum VmaPoolCreateFlagBits {
    1787  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
    1788 
    1789  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1790 } VmaPoolCreateFlagBits;
    1791 typedef VkFlags VmaPoolCreateFlags;
    1792 
    1795 typedef struct VmaPoolCreateInfo {
    1798  uint32_t memoryTypeIndex;
    1801  VmaPoolCreateFlags flags;
    1806  VkDeviceSize blockSize;
    1811  size_t minBlockCount;
    1819  size_t maxBlockCount;
    1833  uint32_t frameInUseCount;
    1834 } VmaPoolCreateInfo;
    1835 
    1838 typedef struct VmaPoolStats {
    1841  VkDeviceSize size;
    1844  VkDeviceSize unusedSize;
    1847  size_t allocationCount;
    1850  size_t unusedRangeCount;
    1857  VkDeviceSize unusedRangeSizeMax;
    1858 } VmaPoolStats;
    1859 
    1866 VkResult vmaCreatePool(
    1867  VmaAllocator allocator,
    1868  const VmaPoolCreateInfo* pCreateInfo,
    1869  VmaPool* pPool);
    1870 
    1873 void vmaDestroyPool(
    1874  VmaAllocator allocator,
    1875  VmaPool pool);
    1876 
    1883 void vmaGetPoolStats(
    1884  VmaAllocator allocator,
    1885  VmaPool pool,
    1886  VmaPoolStats* pPoolStats);
    1887 
    1894 void vmaMakePoolAllocationsLost(
    1895  VmaAllocator allocator,
    1896  VmaPool pool,
    1897  size_t* pLostAllocationCount);
    1898 
    1913 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
    1914 
    1939 VK_DEFINE_HANDLE(VmaAllocation)
    1940 
    1941 
    1943 typedef struct VmaAllocationInfo {
    1948  uint32_t memoryType;
    1957  VkDeviceMemory deviceMemory;
    1962  VkDeviceSize offset;
    1967  VkDeviceSize size;
    1976  void* pMappedData;
    1981  void* pUserData;
    1982 } VmaAllocationInfo;
    1983 
    1994 VkResult vmaAllocateMemory(
    1995  VmaAllocator allocator,
    1996  const VkMemoryRequirements* pVkMemoryRequirements,
    1997  const VmaAllocationCreateInfo* pCreateInfo,
    1998  VmaAllocation* pAllocation,
    1999  VmaAllocationInfo* pAllocationInfo);
    2000 
    2007 VkResult vmaAllocateMemoryForBuffer(
    2008  VmaAllocator allocator,
    2009  VkBuffer buffer,
    2010  const VmaAllocationCreateInfo* pCreateInfo,
    2011  VmaAllocation* pAllocation,
    2012  VmaAllocationInfo* pAllocationInfo);
    2013 
    2015 VkResult vmaAllocateMemoryForImage(
    2016  VmaAllocator allocator,
    2017  VkImage image,
    2018  const VmaAllocationCreateInfo* pCreateInfo,
    2019  VmaAllocation* pAllocation,
    2020  VmaAllocationInfo* pAllocationInfo);
    2021 
    2023 void vmaFreeMemory(
    2024  VmaAllocator allocator,
    2025  VmaAllocation allocation);
    2026 
    2043 void vmaGetAllocationInfo(
    2044  VmaAllocator allocator,
    2045  VmaAllocation allocation,
    2046  VmaAllocationInfo* pAllocationInfo);
    2047 
    2062 VkBool32 vmaTouchAllocation(
    2063  VmaAllocator allocator,
    2064  VmaAllocation allocation);
    2065 
    2079 void vmaSetAllocationUserData(
    2080  VmaAllocator allocator,
    2081  VmaAllocation allocation,
    2082  void* pUserData);
    2083 
    2094 void vmaCreateLostAllocation(
    2095  VmaAllocator allocator,
    2096  VmaAllocation* pAllocation);
    2097 
    2132 VkResult vmaMapMemory(
    2133  VmaAllocator allocator,
    2134  VmaAllocation allocation,
    2135  void** ppData);
    2136 
    2141 void vmaUnmapMemory(
    2142  VmaAllocator allocator,
    2143  VmaAllocation allocation);
    2144 
    2157 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2158 
    2171 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2172 
    2189 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
    2190 
    2192 typedef struct VmaDefragmentationInfo {
    2197  VkDeviceSize maxBytesToMove;
    2202  uint32_t maxAllocationsToMove;
    2203 } VmaDefragmentationInfo;
    2204 
    2206 typedef struct VmaDefragmentationStats {
    2208  VkDeviceSize bytesMoved;
    2210  VkDeviceSize bytesFreed;
    2212  uint32_t allocationsMoved;
    2214  uint32_t deviceMemoryBlocksFreed;
    2215 } VmaDefragmentationStats;
    2216 
    2299 VkResult vmaDefragment(
    2300  VmaAllocator allocator,
    2301  VmaAllocation* pAllocations,
    2302  size_t allocationCount,
    2303  VkBool32* pAllocationsChanged,
    2304  const VmaDefragmentationInfo *pDefragmentationInfo,
    2305  VmaDefragmentationStats* pDefragmentationStats);
    2306 
    2319 VkResult vmaBindBufferMemory(
    2320  VmaAllocator allocator,
    2321  VmaAllocation allocation,
    2322  VkBuffer buffer);
    2323 
    2336 VkResult vmaBindImageMemory(
    2337  VmaAllocator allocator,
    2338  VmaAllocation allocation,
    2339  VkImage image);
    2340 
    2367 VkResult vmaCreateBuffer(
    2368  VmaAllocator allocator,
    2369  const VkBufferCreateInfo* pBufferCreateInfo,
    2370  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2371  VkBuffer* pBuffer,
    2372  VmaAllocation* pAllocation,
    2373  VmaAllocationInfo* pAllocationInfo);
    2374 
    2386 void vmaDestroyBuffer(
    2387  VmaAllocator allocator,
    2388  VkBuffer buffer,
    2389  VmaAllocation allocation);
    2390 
    2392 VkResult vmaCreateImage(
    2393  VmaAllocator allocator,
    2394  const VkImageCreateInfo* pImageCreateInfo,
    2395  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2396  VkImage* pImage,
    2397  VmaAllocation* pAllocation,
    2398  VmaAllocationInfo* pAllocationInfo);
    2399 
    2411 void vmaDestroyImage(
    2412  VmaAllocator allocator,
    2413  VkImage image,
    2414  VmaAllocation allocation);
    2415 
    2416 #ifdef __cplusplus
    2417 }
    2418 #endif
    2419 
    2420 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
    2421 
    2422 // For Visual Studio IntelliSense.
    2423 #if defined(__cplusplus) && defined(__INTELLISENSE__)
    2424 #define VMA_IMPLEMENTATION
    2425 #endif
    2426 
    2427 #ifdef VMA_IMPLEMENTATION
    2428 #undef VMA_IMPLEMENTATION
    2429 
    2430 #include <cstdint>
    2431 #include <cstdlib>
    2432 #include <cstring>
    2433 
    2434 /*******************************************************************************
    2435 CONFIGURATION SECTION
    2436 
    2437 Define some of these macros before each #include of this header or change them
    2438 here if you need other then default behavior depending on your environment.
    2439 */
    2440 
    2441 /*
    2442 Define this macro to 1 to make the library fetch pointers to Vulkan functions
    2443 internally, like:
    2444 
    2445  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    2446 
    2447 Define to 0 if you are going to provide you own pointers to Vulkan functions via
    2448 VmaAllocatorCreateInfo::pVulkanFunctions.
    2449 */
    2450 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
    2451 #define VMA_STATIC_VULKAN_FUNCTIONS 1
    2452 #endif
    2453 
    2454 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
    2455 //#define VMA_USE_STL_CONTAINERS 1
    2456 
    2457 /* Set this macro to 1 to make the library including and using STL containers:
    2458 std::pair, std::vector, std::list, std::unordered_map.
    2459 
    2460 Set it to 0 or undefined to make the library using its own implementation of
    2461 the containers.
    2462 */
    2463 #if VMA_USE_STL_CONTAINERS
    2464  #define VMA_USE_STL_VECTOR 1
    2465  #define VMA_USE_STL_UNORDERED_MAP 1
    2466  #define VMA_USE_STL_LIST 1
    2467 #endif
    2468 
    2469 #if VMA_USE_STL_VECTOR
    2470  #include <vector>
    2471 #endif
    2472 
    2473 #if VMA_USE_STL_UNORDERED_MAP
    2474  #include <unordered_map>
    2475 #endif
    2476 
    2477 #if VMA_USE_STL_LIST
    2478  #include <list>
    2479 #endif
    2480 
    2481 /*
    2482 Following headers are used in this CONFIGURATION section only, so feel free to
    2483 remove them if not needed.
    2484 */
    2485 #include <cassert> // for assert
    2486 #include <algorithm> // for min, max
    2487 #include <mutex> // for std::mutex
    2488 #include <atomic> // for std::atomic
    2489 
    2490 #ifndef VMA_NULL
    2491  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
    2492  #define VMA_NULL nullptr
    2493 #endif
    2494 
    2495 #if defined(__APPLE__) || defined(__ANDROID__)
    2496 #include <cstdlib>
    2497 void *aligned_alloc(size_t alignment, size_t size)
    2498 {
    2499  // alignment must be >= sizeof(void*)
    2500  if(alignment < sizeof(void*))
    2501  {
    2502  alignment = sizeof(void*);
    2503  }
    2504 
    2505  void *pointer;
    2506  if(posix_memalign(&pointer, alignment, size) == 0)
    2507  return pointer;
    2508  return VMA_NULL;
    2509 }
    2510 #endif
    2511 
    2512 // If your compiler is not compatible with C++11 and definition of
    2513 // aligned_alloc() function is missing, uncommeting following line may help:
    2514 
    2515 //#include <malloc.h>
    2516 
    2517 // Normal assert to check for programmer's errors, especially in Debug configuration.
    2518 #ifndef VMA_ASSERT
    2519  #ifdef _DEBUG
    2520  #define VMA_ASSERT(expr) assert(expr)
    2521  #else
    2522  #define VMA_ASSERT(expr)
    2523  #endif
    2524 #endif
    2525 
    2526 // Assert that will be called very often, like inside data structures e.g. operator[].
    2527 // Making it non-empty can make program slow.
    2528 #ifndef VMA_HEAVY_ASSERT
    2529  #ifdef _DEBUG
    2530  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
    2531  #else
    2532  #define VMA_HEAVY_ASSERT(expr)
    2533  #endif
    2534 #endif
    2535 
    2536 #ifndef VMA_ALIGN_OF
    2537  #define VMA_ALIGN_OF(type) (__alignof(type))
    2538 #endif
    2539 
    2540 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
    2541  #if defined(_WIN32)
    2542  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
    2543  #else
    2544  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
    2545  #endif
    2546 #endif
    2547 
    2548 #ifndef VMA_SYSTEM_FREE
    2549  #if defined(_WIN32)
    2550  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
    2551  #else
    2552  #define VMA_SYSTEM_FREE(ptr) free(ptr)
    2553  #endif
    2554 #endif
    2555 
    2556 #ifndef VMA_MIN
    2557  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
    2558 #endif
    2559 
    2560 #ifndef VMA_MAX
    2561  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
    2562 #endif
    2563 
    2564 #ifndef VMA_SWAP
    2565  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
    2566 #endif
    2567 
    2568 #ifndef VMA_SORT
    2569  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
    2570 #endif
    2571 
    2572 #ifndef VMA_DEBUG_LOG
    2573  #define VMA_DEBUG_LOG(format, ...)
    2574  /*
    2575  #define VMA_DEBUG_LOG(format, ...) do { \
    2576  printf(format, __VA_ARGS__); \
    2577  printf("\n"); \
    2578  } while(false)
    2579  */
    2580 #endif
    2581 
    2582 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
    2583 #if VMA_STATS_STRING_ENABLED
    2584  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
    2585  {
    2586  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
    2587  }
    2588  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
    2589  {
    2590  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
    2591  }
    2592  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
    2593  {
    2594  snprintf(outStr, strLen, "%p", ptr);
    2595  }
    2596 #endif
    2597 
    2598 #ifndef VMA_MUTEX
    2599  class VmaMutex
    2600  {
    2601  public:
    2602  VmaMutex() { }
    2603  ~VmaMutex() { }
    2604  void Lock() { m_Mutex.lock(); }
    2605  void Unlock() { m_Mutex.unlock(); }
    2606  private:
    2607  std::mutex m_Mutex;
    2608  };
    2609  #define VMA_MUTEX VmaMutex
    2610 #endif
    2611 
    2612 /*
    2613 If providing your own implementation, you need to implement a subset of std::atomic:
    2614 
    2615 - Constructor(uint32_t desired)
    2616 - uint32_t load() const
    2617 - void store(uint32_t desired)
    2618 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
    2619 */
    2620 #ifndef VMA_ATOMIC_UINT32
    2621  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
    2622 #endif
    2623 
    2624 #ifndef VMA_BEST_FIT
    2625 
    2637  #define VMA_BEST_FIT (1)
    2638 #endif
    2639 
    2640 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
    2641 
    2645  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
    2646 #endif
    2647 
    2648 #ifndef VMA_DEBUG_ALIGNMENT
    2649 
    2653  #define VMA_DEBUG_ALIGNMENT (1)
    2654 #endif
    2655 
    2656 #ifndef VMA_DEBUG_MARGIN
    2657 
    2661  #define VMA_DEBUG_MARGIN (0)
    2662 #endif
    2663 
    2664 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
    2665 
    2669  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
    2670 #endif
    2671 
    2672 #ifndef VMA_DEBUG_DETECT_CORRUPTION
    2673 
    2678  #define VMA_DEBUG_DETECT_CORRUPTION (0)
    2679 #endif
    2680 
    2681 #ifndef VMA_DEBUG_GLOBAL_MUTEX
    2682 
    2686  #define VMA_DEBUG_GLOBAL_MUTEX (0)
    2687 #endif
    2688 
    2689 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
    2690 
    2694  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
    2695 #endif
    2696 
    2697 #ifndef VMA_SMALL_HEAP_MAX_SIZE
    2698  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
    2700 #endif
    2701 
    2702 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
    2703  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
    2705 #endif
    2706 
    2707 #ifndef VMA_CLASS_NO_COPY
    2708  #define VMA_CLASS_NO_COPY(className) \
    2709  private: \
    2710  className(const className&) = delete; \
    2711  className& operator=(const className&) = delete;
    2712 #endif
    2713 
    2714 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
    2715 
    2716 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
    2717 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
    2718 
    2719 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
    2720 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
    2721 
    2722 /*******************************************************************************
    2723 END OF CONFIGURATION
    2724 */
    2725 
    2726 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
    2727  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
    2728 
    2729 // Returns number of bits set to 1 in (v).
    2730 static inline uint32_t VmaCountBitsSet(uint32_t v)
    2731 {
    2732  uint32_t c = v - ((v >> 1) & 0x55555555);
    2733  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
    2734  c = ((c >> 4) + c) & 0x0F0F0F0F;
    2735  c = ((c >> 8) + c) & 0x00FF00FF;
    2736  c = ((c >> 16) + c) & 0x0000FFFF;
    2737  return c;
    2738 }
    2739 
    2740 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
    2741 // Use types like uint32_t, uint64_t as T.
    2742 template <typename T>
    2743 static inline T VmaAlignUp(T val, T align)
    2744 {
    2745  return (val + align - 1) / align * align;
    2746 }
    2747 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
    2748 // Use types like uint32_t, uint64_t as T.
    2749 template <typename T>
    2750 static inline T VmaAlignDown(T val, T align)
    2751 {
    2752  return val / align * align;
    2753 }
    2754 
    2755 // Division with mathematical rounding to nearest number.
    2756 template <typename T>
    2757 inline T VmaRoundDiv(T x, T y)
    2758 {
    2759  return (x + (y / (T)2)) / y;
    2760 }
    2761 
    2762 static inline bool VmaStrIsEmpty(const char* pStr)
    2763 {
    2764  return pStr == VMA_NULL || *pStr == '\0';
    2765 }
    2766 
    2767 #ifndef VMA_SORT
    2768 
    2769 template<typename Iterator, typename Compare>
    2770 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
    2771 {
    2772  Iterator centerValue = end; --centerValue;
    2773  Iterator insertIndex = beg;
    2774  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
    2775  {
    2776  if(cmp(*memTypeIndex, *centerValue))
    2777  {
    2778  if(insertIndex != memTypeIndex)
    2779  {
    2780  VMA_SWAP(*memTypeIndex, *insertIndex);
    2781  }
    2782  ++insertIndex;
    2783  }
    2784  }
    2785  if(insertIndex != centerValue)
    2786  {
    2787  VMA_SWAP(*insertIndex, *centerValue);
    2788  }
    2789  return insertIndex;
    2790 }
    2791 
    2792 template<typename Iterator, typename Compare>
    2793 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
    2794 {
    2795  if(beg < end)
    2796  {
    2797  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
    2798  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
    2799  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
    2800  }
    2801 }
    2802 
    2803 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
    2804 
    2805 #endif // #ifndef VMA_SORT
    2806 
    2807 /*
    2808 Returns true if two memory blocks occupy overlapping pages.
    2809 ResourceA must be in less memory offset than ResourceB.
    2810 
    2811 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
    2812 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
    2813 */
    2814 static inline bool VmaBlocksOnSamePage(
    2815  VkDeviceSize resourceAOffset,
    2816  VkDeviceSize resourceASize,
    2817  VkDeviceSize resourceBOffset,
    2818  VkDeviceSize pageSize)
    2819 {
    2820  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
    2821  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
    2822  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
    2823  VkDeviceSize resourceBStart = resourceBOffset;
    2824  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
    2825  return resourceAEndPage == resourceBStartPage;
    2826 }
    2827 
    2828 enum VmaSuballocationType
    2829 {
    2830  VMA_SUBALLOCATION_TYPE_FREE = 0,
    2831  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
    2832  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
    2833  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
    2834  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
    2835  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
    2836  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
    2837 };
    2838 
    2839 /*
    2840 Returns true if given suballocation types could conflict and must respect
    2841 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
    2842 or linear image and another one is optimal image. If type is unknown, behave
    2843 conservatively.
    2844 */
    2845 static inline bool VmaIsBufferImageGranularityConflict(
    2846  VmaSuballocationType suballocType1,
    2847  VmaSuballocationType suballocType2)
    2848 {
    2849  if(suballocType1 > suballocType2)
    2850  {
    2851  VMA_SWAP(suballocType1, suballocType2);
    2852  }
    2853 
    2854  switch(suballocType1)
    2855  {
    2856  case VMA_SUBALLOCATION_TYPE_FREE:
    2857  return false;
    2858  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
    2859  return true;
    2860  case VMA_SUBALLOCATION_TYPE_BUFFER:
    2861  return
    2862  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2863  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2864  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
    2865  return
    2866  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2867  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
    2868  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2869  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
    2870  return
    2871  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2872  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
    2873  return false;
    2874  default:
    2875  VMA_ASSERT(0);
    2876  return true;
    2877  }
    2878 }
    2879 
    2880 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
    2881 {
    2882  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
    2883  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2884  for(size_t i = 0; i < numberCount; ++i, ++pDst)
    2885  {
    2886  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
    2887  }
    2888 }
    2889 
    2890 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
    2891 {
    2892  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
    2893  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2894  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
    2895  {
    2896  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
    2897  {
    2898  return false;
    2899  }
    2900  }
    2901  return true;
    2902 }
    2903 
    2904 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
    2905 struct VmaMutexLock
    2906 {
    2907  VMA_CLASS_NO_COPY(VmaMutexLock)
    2908 public:
    2909  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
    2910  m_pMutex(useMutex ? &mutex : VMA_NULL)
    2911  {
    2912  if(m_pMutex)
    2913  {
    2914  m_pMutex->Lock();
    2915  }
    2916  }
    2917 
    2918  ~VmaMutexLock()
    2919  {
    2920  if(m_pMutex)
    2921  {
    2922  m_pMutex->Unlock();
    2923  }
    2924  }
    2925 
    2926 private:
    2927  VMA_MUTEX* m_pMutex;
    2928 };
    2929 
    2930 #if VMA_DEBUG_GLOBAL_MUTEX
    2931  static VMA_MUTEX gDebugGlobalMutex;
    2932  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
    2933 #else
    2934  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
    2935 #endif
    2936 
    2937 // Minimum size of a free suballocation to register it in the free suballocation collection.
    2938 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
    2939 
    2940 /*
    2941 Performs binary search and returns iterator to first element that is greater or
    2942 equal to (key), according to comparison (cmp).
    2943 
    2944 Cmp should return true if first argument is less than second argument.
    2945 
    2946 Returned value is the found element, if present in the collection or place where
    2947 new element with value (key) should be inserted.
    2948 */
    2949 template <typename IterT, typename KeyT, typename CmpT>
    2950 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
    2951 {
    2952  size_t down = 0, up = (end - beg);
    2953  while(down < up)
    2954  {
    2955  const size_t mid = (down + up) / 2;
    2956  if(cmp(*(beg+mid), key))
    2957  {
    2958  down = mid + 1;
    2959  }
    2960  else
    2961  {
    2962  up = mid;
    2963  }
    2964  }
    2965  return beg + down;
    2966 }
    2967 
    2969 // Memory allocation
    2970 
    2971 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
    2972 {
    2973  if((pAllocationCallbacks != VMA_NULL) &&
    2974  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
    2975  {
    2976  return (*pAllocationCallbacks->pfnAllocation)(
    2977  pAllocationCallbacks->pUserData,
    2978  size,
    2979  alignment,
    2980  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    2981  }
    2982  else
    2983  {
    2984  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
    2985  }
    2986 }
    2987 
    2988 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
    2989 {
    2990  if((pAllocationCallbacks != VMA_NULL) &&
    2991  (pAllocationCallbacks->pfnFree != VMA_NULL))
    2992  {
    2993  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
    2994  }
    2995  else
    2996  {
    2997  VMA_SYSTEM_FREE(ptr);
    2998  }
    2999 }
    3000 
    3001 template<typename T>
    3002 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
    3003 {
    3004  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
    3005 }
    3006 
    3007 template<typename T>
    3008 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
    3009 {
    3010  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
    3011 }
    3012 
    3013 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
    3014 
    3015 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
    3016 
    3017 template<typename T>
    3018 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
    3019 {
    3020  ptr->~T();
    3021  VmaFree(pAllocationCallbacks, ptr);
    3022 }
    3023 
    3024 template<typename T>
    3025 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
    3026 {
    3027  if(ptr != VMA_NULL)
    3028  {
    3029  for(size_t i = count; i--; )
    3030  {
    3031  ptr[i].~T();
    3032  }
    3033  VmaFree(pAllocationCallbacks, ptr);
    3034  }
    3035 }
    3036 
    3037 // STL-compatible allocator.
    3038 template<typename T>
    3039 class VmaStlAllocator
    3040 {
    3041 public:
    3042  const VkAllocationCallbacks* const m_pCallbacks;
    3043  typedef T value_type;
    3044 
    3045  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
    3046  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
    3047 
    3048  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
    3049  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
    3050 
    3051  template<typename U>
    3052  bool operator==(const VmaStlAllocator<U>& rhs) const
    3053  {
    3054  return m_pCallbacks == rhs.m_pCallbacks;
    3055  }
    3056  template<typename U>
    3057  bool operator!=(const VmaStlAllocator<U>& rhs) const
    3058  {
    3059  return m_pCallbacks != rhs.m_pCallbacks;
    3060  }
    3061 
    3062  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
    3063 };
    3064 
    3065 #if VMA_USE_STL_VECTOR
    3066 
    3067 #define VmaVector std::vector
    3068 
    3069 template<typename T, typename allocatorT>
    3070 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
    3071 {
    3072  vec.insert(vec.begin() + index, item);
    3073 }
    3074 
    3075 template<typename T, typename allocatorT>
    3076 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
    3077 {
    3078  vec.erase(vec.begin() + index);
    3079 }
    3080 
    3081 #else // #if VMA_USE_STL_VECTOR
    3082 
    3083 /* Class with interface compatible with subset of std::vector.
    3084 T must be POD because constructors and destructors are not called and memcpy is
    3085 used for these objects. */
    3086 template<typename T, typename AllocatorT>
    3087 class VmaVector
    3088 {
    3089 public:
    3090  typedef T value_type;
    3091 
    3092  VmaVector(const AllocatorT& allocator) :
    3093  m_Allocator(allocator),
    3094  m_pArray(VMA_NULL),
    3095  m_Count(0),
    3096  m_Capacity(0)
    3097  {
    3098  }
    3099 
    3100  VmaVector(size_t count, const AllocatorT& allocator) :
    3101  m_Allocator(allocator),
    3102  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
    3103  m_Count(count),
    3104  m_Capacity(count)
    3105  {
    3106  }
    3107 
    3108  VmaVector(const VmaVector<T, AllocatorT>& src) :
    3109  m_Allocator(src.m_Allocator),
    3110  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
    3111  m_Count(src.m_Count),
    3112  m_Capacity(src.m_Count)
    3113  {
    3114  if(m_Count != 0)
    3115  {
    3116  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
    3117  }
    3118  }
    3119 
    3120  ~VmaVector()
    3121  {
    3122  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3123  }
    3124 
    3125  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
    3126  {
    3127  if(&rhs != this)
    3128  {
    3129  resize(rhs.m_Count);
    3130  if(m_Count != 0)
    3131  {
    3132  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
    3133  }
    3134  }
    3135  return *this;
    3136  }
    3137 
    3138  bool empty() const { return m_Count == 0; }
    3139  size_t size() const { return m_Count; }
    3140  T* data() { return m_pArray; }
    3141  const T* data() const { return m_pArray; }
    3142 
    3143  T& operator[](size_t index)
    3144  {
    3145  VMA_HEAVY_ASSERT(index < m_Count);
    3146  return m_pArray[index];
    3147  }
    3148  const T& operator[](size_t index) const
    3149  {
    3150  VMA_HEAVY_ASSERT(index < m_Count);
    3151  return m_pArray[index];
    3152  }
    3153 
    3154  T& front()
    3155  {
    3156  VMA_HEAVY_ASSERT(m_Count > 0);
    3157  return m_pArray[0];
    3158  }
    3159  const T& front() const
    3160  {
    3161  VMA_HEAVY_ASSERT(m_Count > 0);
    3162  return m_pArray[0];
    3163  }
    3164  T& back()
    3165  {
    3166  VMA_HEAVY_ASSERT(m_Count > 0);
    3167  return m_pArray[m_Count - 1];
    3168  }
    3169  const T& back() const
    3170  {
    3171  VMA_HEAVY_ASSERT(m_Count > 0);
    3172  return m_pArray[m_Count - 1];
    3173  }
    3174 
    3175  void reserve(size_t newCapacity, bool freeMemory = false)
    3176  {
    3177  newCapacity = VMA_MAX(newCapacity, m_Count);
    3178 
    3179  if((newCapacity < m_Capacity) && !freeMemory)
    3180  {
    3181  newCapacity = m_Capacity;
    3182  }
    3183 
    3184  if(newCapacity != m_Capacity)
    3185  {
    3186  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
    3187  if(m_Count != 0)
    3188  {
    3189  memcpy(newArray, m_pArray, m_Count * sizeof(T));
    3190  }
    3191  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3192  m_Capacity = newCapacity;
    3193  m_pArray = newArray;
    3194  }
    3195  }
    3196 
    3197  void resize(size_t newCount, bool freeMemory = false)
    3198  {
    3199  size_t newCapacity = m_Capacity;
    3200  if(newCount > m_Capacity)
    3201  {
    3202  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
    3203  }
    3204  else if(freeMemory)
    3205  {
    3206  newCapacity = newCount;
    3207  }
    3208 
    3209  if(newCapacity != m_Capacity)
    3210  {
    3211  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
    3212  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
    3213  if(elementsToCopy != 0)
    3214  {
    3215  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
    3216  }
    3217  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3218  m_Capacity = newCapacity;
    3219  m_pArray = newArray;
    3220  }
    3221 
    3222  m_Count = newCount;
    3223  }
    3224 
    3225  void clear(bool freeMemory = false)
    3226  {
    3227  resize(0, freeMemory);
    3228  }
    3229 
    3230  void insert(size_t index, const T& src)
    3231  {
    3232  VMA_HEAVY_ASSERT(index <= m_Count);
    3233  const size_t oldCount = size();
    3234  resize(oldCount + 1);
    3235  if(index < oldCount)
    3236  {
    3237  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
    3238  }
    3239  m_pArray[index] = src;
    3240  }
    3241 
    3242  void remove(size_t index)
    3243  {
    3244  VMA_HEAVY_ASSERT(index < m_Count);
    3245  const size_t oldCount = size();
    3246  if(index < oldCount - 1)
    3247  {
    3248  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
    3249  }
    3250  resize(oldCount - 1);
    3251  }
    3252 
    3253  void push_back(const T& src)
    3254  {
    3255  const size_t newIndex = size();
    3256  resize(newIndex + 1);
    3257  m_pArray[newIndex] = src;
    3258  }
    3259 
    3260  void pop_back()
    3261  {
    3262  VMA_HEAVY_ASSERT(m_Count > 0);
    3263  resize(size() - 1);
    3264  }
    3265 
    3266  void push_front(const T& src)
    3267  {
    3268  insert(0, src);
    3269  }
    3270 
    3271  void pop_front()
    3272  {
    3273  VMA_HEAVY_ASSERT(m_Count > 0);
    3274  remove(0);
    3275  }
    3276 
    3277  typedef T* iterator;
    3278 
    3279  iterator begin() { return m_pArray; }
    3280  iterator end() { return m_pArray + m_Count; }
    3281 
    3282 private:
    3283  AllocatorT m_Allocator;
    3284  T* m_pArray;
    3285  size_t m_Count;
    3286  size_t m_Capacity;
    3287 };
    3288 
    3289 template<typename T, typename allocatorT>
    3290 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
    3291 {
    3292  vec.insert(index, item);
    3293 }
    3294 
    3295 template<typename T, typename allocatorT>
    3296 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
    3297 {
    3298  vec.remove(index);
    3299 }
    3300 
    3301 #endif // #if VMA_USE_STL_VECTOR
    3302 
    3303 template<typename CmpLess, typename VectorT>
    3304 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
    3305 {
    3306  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    3307  vector.data(),
    3308  vector.data() + vector.size(),
    3309  value,
    3310  CmpLess()) - vector.data();
    3311  VmaVectorInsert(vector, indexToInsert, value);
    3312  return indexToInsert;
    3313 }
    3314 
    3315 template<typename CmpLess, typename VectorT>
    3316 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
    3317 {
    3318  CmpLess comparator;
    3319  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3320  vector.begin(),
    3321  vector.end(),
    3322  value,
    3323  comparator);
    3324  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
    3325  {
    3326  size_t indexToRemove = it - vector.begin();
    3327  VmaVectorRemove(vector, indexToRemove);
    3328  return true;
    3329  }
    3330  return false;
    3331 }
    3332 
    3333 template<typename CmpLess, typename VectorT>
    3334 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
    3335 {
    3336  CmpLess comparator;
    3337  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3338  vector.data(),
    3339  vector.data() + vector.size(),
    3340  value,
    3341  comparator);
    3342  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
    3343  {
    3344  return it - vector.begin();
    3345  }
    3346  else
    3347  {
    3348  return vector.size();
    3349  }
    3350 }
    3351 
    3353 // class VmaPoolAllocator
    3354 
    3355 /*
    3356 Allocator for objects of type T using a list of arrays (pools) to speed up
    3357 allocation. Number of elements that can be allocated is not bounded because
    3358 allocator can create multiple blocks.
    3359 */
    3360 template<typename T>
    3361 class VmaPoolAllocator
    3362 {
    3363  VMA_CLASS_NO_COPY(VmaPoolAllocator)
    3364 public:
    3365  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
    3366  ~VmaPoolAllocator();
    3367  void Clear();
    3368  T* Alloc();
    3369  void Free(T* ptr);
    3370 
    3371 private:
    3372  union Item
    3373  {
    3374  uint32_t NextFreeIndex;
    3375  T Value;
    3376  };
    3377 
    3378  struct ItemBlock
    3379  {
    3380  Item* pItems;
    3381  uint32_t FirstFreeIndex;
    3382  };
    3383 
    3384  const VkAllocationCallbacks* m_pAllocationCallbacks;
    3385  size_t m_ItemsPerBlock;
    3386  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
    3387 
    3388  ItemBlock& CreateNewBlock();
    3389 };
    3390 
    3391 template<typename T>
    3392 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
    3393  m_pAllocationCallbacks(pAllocationCallbacks),
    3394  m_ItemsPerBlock(itemsPerBlock),
    3395  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
    3396 {
    3397  VMA_ASSERT(itemsPerBlock > 0);
    3398 }
    3399 
    3400 template<typename T>
    3401 VmaPoolAllocator<T>::~VmaPoolAllocator()
    3402 {
    3403  Clear();
    3404 }
    3405 
    3406 template<typename T>
    3407 void VmaPoolAllocator<T>::Clear()
    3408 {
    3409  for(size_t i = m_ItemBlocks.size(); i--; )
    3410  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
    3411  m_ItemBlocks.clear();
    3412 }
    3413 
    3414 template<typename T>
    3415 T* VmaPoolAllocator<T>::Alloc()
    3416 {
    3417  for(size_t i = m_ItemBlocks.size(); i--; )
    3418  {
    3419  ItemBlock& block = m_ItemBlocks[i];
    3420  // This block has some free items: Use first one.
    3421  if(block.FirstFreeIndex != UINT32_MAX)
    3422  {
    3423  Item* const pItem = &block.pItems[block.FirstFreeIndex];
    3424  block.FirstFreeIndex = pItem->NextFreeIndex;
    3425  return &pItem->Value;
    3426  }
    3427  }
    3428 
    3429  // No block has free item: Create new one and use it.
    3430  ItemBlock& newBlock = CreateNewBlock();
    3431  Item* const pItem = &newBlock.pItems[0];
    3432  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
    3433  return &pItem->Value;
    3434 }
    3435 
    3436 template<typename T>
    3437 void VmaPoolAllocator<T>::Free(T* ptr)
    3438 {
    3439  // Search all memory blocks to find ptr.
    3440  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
    3441  {
    3442  ItemBlock& block = m_ItemBlocks[i];
    3443 
    3444  // Casting to union.
    3445  Item* pItemPtr;
    3446  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
    3447 
    3448  // Check if pItemPtr is in address range of this block.
    3449  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
    3450  {
    3451  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
    3452  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
    3453  block.FirstFreeIndex = index;
    3454  return;
    3455  }
    3456  }
    3457  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
    3458 }
    3459 
    3460 template<typename T>
    3461 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
    3462 {
    3463  ItemBlock newBlock = {
    3464  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
    3465 
    3466  m_ItemBlocks.push_back(newBlock);
    3467 
    3468  // Setup singly-linked list of all free items in this block.
    3469  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
    3470  newBlock.pItems[i].NextFreeIndex = i + 1;
    3471  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
    3472  return m_ItemBlocks.back();
    3473 }
    3474 
    3476 // class VmaRawList, VmaList
    3477 
    3478 #if VMA_USE_STL_LIST
    3479 
    3480 #define VmaList std::list
    3481 
    3482 #else // #if VMA_USE_STL_LIST
    3483 
    3484 template<typename T>
    3485 struct VmaListItem
    3486 {
    3487  VmaListItem* pPrev;
    3488  VmaListItem* pNext;
    3489  T Value;
    3490 };
    3491 
    3492 // Doubly linked list.
    3493 template<typename T>
    3494 class VmaRawList
    3495 {
    3496  VMA_CLASS_NO_COPY(VmaRawList)
    3497 public:
    3498  typedef VmaListItem<T> ItemType;
    3499 
    3500  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
    3501  ~VmaRawList();
    3502  void Clear();
    3503 
    3504  size_t GetCount() const { return m_Count; }
    3505  bool IsEmpty() const { return m_Count == 0; }
    3506 
    3507  ItemType* Front() { return m_pFront; }
    3508  const ItemType* Front() const { return m_pFront; }
    3509  ItemType* Back() { return m_pBack; }
    3510  const ItemType* Back() const { return m_pBack; }
    3511 
    3512  ItemType* PushBack();
    3513  ItemType* PushFront();
    3514  ItemType* PushBack(const T& value);
    3515  ItemType* PushFront(const T& value);
    3516  void PopBack();
    3517  void PopFront();
    3518 
    3519  // Item can be null - it means PushBack.
    3520  ItemType* InsertBefore(ItemType* pItem);
    3521  // Item can be null - it means PushFront.
    3522  ItemType* InsertAfter(ItemType* pItem);
    3523 
    3524  ItemType* InsertBefore(ItemType* pItem, const T& value);
    3525  ItemType* InsertAfter(ItemType* pItem, const T& value);
    3526 
    3527  void Remove(ItemType* pItem);
    3528 
    3529 private:
    3530  const VkAllocationCallbacks* const m_pAllocationCallbacks;
    3531  VmaPoolAllocator<ItemType> m_ItemAllocator;
    3532  ItemType* m_pFront;
    3533  ItemType* m_pBack;
    3534  size_t m_Count;
    3535 };
    3536 
    3537 template<typename T>
    3538 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
    3539  m_pAllocationCallbacks(pAllocationCallbacks),
    3540  m_ItemAllocator(pAllocationCallbacks, 128),
    3541  m_pFront(VMA_NULL),
    3542  m_pBack(VMA_NULL),
    3543  m_Count(0)
    3544 {
    3545 }
    3546 
    3547 template<typename T>
    3548 VmaRawList<T>::~VmaRawList()
    3549 {
    3550  // Intentionally not calling Clear, because that would be unnecessary
    3551  // computations to return all items to m_ItemAllocator as free.
    3552 }
    3553 
    3554 template<typename T>
    3555 void VmaRawList<T>::Clear()
    3556 {
    3557  if(IsEmpty() == false)
    3558  {
    3559  ItemType* pItem = m_pBack;
    3560  while(pItem != VMA_NULL)
    3561  {
    3562  ItemType* const pPrevItem = pItem->pPrev;
    3563  m_ItemAllocator.Free(pItem);
    3564  pItem = pPrevItem;
    3565  }
    3566  m_pFront = VMA_NULL;
    3567  m_pBack = VMA_NULL;
    3568  m_Count = 0;
    3569  }
    3570 }
    3571 
    3572 template<typename T>
    3573 VmaListItem<T>* VmaRawList<T>::PushBack()
    3574 {
    3575  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3576  pNewItem->pNext = VMA_NULL;
    3577  if(IsEmpty())
    3578  {
    3579  pNewItem->pPrev = VMA_NULL;
    3580  m_pFront = pNewItem;
    3581  m_pBack = pNewItem;
    3582  m_Count = 1;
    3583  }
    3584  else
    3585  {
    3586  pNewItem->pPrev = m_pBack;
    3587  m_pBack->pNext = pNewItem;
    3588  m_pBack = pNewItem;
    3589  ++m_Count;
    3590  }
    3591  return pNewItem;
    3592 }
    3593 
    3594 template<typename T>
    3595 VmaListItem<T>* VmaRawList<T>::PushFront()
    3596 {
    3597  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3598  pNewItem->pPrev = VMA_NULL;
    3599  if(IsEmpty())
    3600  {
    3601  pNewItem->pNext = VMA_NULL;
    3602  m_pFront = pNewItem;
    3603  m_pBack = pNewItem;
    3604  m_Count = 1;
    3605  }
    3606  else
    3607  {
    3608  pNewItem->pNext = m_pFront;
    3609  m_pFront->pPrev = pNewItem;
    3610  m_pFront = pNewItem;
    3611  ++m_Count;
    3612  }
    3613  return pNewItem;
    3614 }
    3615 
    3616 template<typename T>
    3617 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
    3618 {
    3619  ItemType* const pNewItem = PushBack();
    3620  pNewItem->Value = value;
    3621  return pNewItem;
    3622 }
    3623 
    3624 template<typename T>
    3625 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
    3626 {
    3627  ItemType* const pNewItem = PushFront();
    3628  pNewItem->Value = value;
    3629  return pNewItem;
    3630 }
    3631 
    3632 template<typename T>
    3633 void VmaRawList<T>::PopBack()
    3634 {
    3635  VMA_HEAVY_ASSERT(m_Count > 0);
    3636  ItemType* const pBackItem = m_pBack;
    3637  ItemType* const pPrevItem = pBackItem->pPrev;
    3638  if(pPrevItem != VMA_NULL)
    3639  {
    3640  pPrevItem->pNext = VMA_NULL;
    3641  }
    3642  m_pBack = pPrevItem;
    3643  m_ItemAllocator.Free(pBackItem);
    3644  --m_Count;
    3645 }
    3646 
    3647 template<typename T>
    3648 void VmaRawList<T>::PopFront()
    3649 {
    3650  VMA_HEAVY_ASSERT(m_Count > 0);
    3651  ItemType* const pFrontItem = m_pFront;
    3652  ItemType* const pNextItem = pFrontItem->pNext;
    3653  if(pNextItem != VMA_NULL)
    3654  {
    3655  pNextItem->pPrev = VMA_NULL;
    3656  }
    3657  m_pFront = pNextItem;
    3658  m_ItemAllocator.Free(pFrontItem);
    3659  --m_Count;
    3660 }
    3661 
    3662 template<typename T>
    3663 void VmaRawList<T>::Remove(ItemType* pItem)
    3664 {
    3665  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
    3666  VMA_HEAVY_ASSERT(m_Count > 0);
    3667 
    3668  if(pItem->pPrev != VMA_NULL)
    3669  {
    3670  pItem->pPrev->pNext = pItem->pNext;
    3671  }
    3672  else
    3673  {
    3674  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3675  m_pFront = pItem->pNext;
    3676  }
    3677 
    3678  if(pItem->pNext != VMA_NULL)
    3679  {
    3680  pItem->pNext->pPrev = pItem->pPrev;
    3681  }
    3682  else
    3683  {
    3684  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3685  m_pBack = pItem->pPrev;
    3686  }
    3687 
    3688  m_ItemAllocator.Free(pItem);
    3689  --m_Count;
    3690 }
    3691 
    3692 template<typename T>
    3693 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
    3694 {
    3695  if(pItem != VMA_NULL)
    3696  {
    3697  ItemType* const prevItem = pItem->pPrev;
    3698  ItemType* const newItem = m_ItemAllocator.Alloc();
    3699  newItem->pPrev = prevItem;
    3700  newItem->pNext = pItem;
    3701  pItem->pPrev = newItem;
    3702  if(prevItem != VMA_NULL)
    3703  {
    3704  prevItem->pNext = newItem;
    3705  }
    3706  else
    3707  {
    3708  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3709  m_pFront = newItem;
    3710  }
    3711  ++m_Count;
    3712  return newItem;
    3713  }
    3714  else
    3715  return PushBack();
    3716 }
    3717 
    3718 template<typename T>
    3719 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
    3720 {
    3721  if(pItem != VMA_NULL)
    3722  {
    3723  ItemType* const nextItem = pItem->pNext;
    3724  ItemType* const newItem = m_ItemAllocator.Alloc();
    3725  newItem->pNext = nextItem;
    3726  newItem->pPrev = pItem;
    3727  pItem->pNext = newItem;
    3728  if(nextItem != VMA_NULL)
    3729  {
    3730  nextItem->pPrev = newItem;
    3731  }
    3732  else
    3733  {
    3734  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3735  m_pBack = newItem;
    3736  }
    3737  ++m_Count;
    3738  return newItem;
    3739  }
    3740  else
    3741  return PushFront();
    3742 }
    3743 
    3744 template<typename T>
    3745 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
    3746 {
    3747  ItemType* const newItem = InsertBefore(pItem);
    3748  newItem->Value = value;
    3749  return newItem;
    3750 }
    3751 
    3752 template<typename T>
    3753 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
    3754 {
    3755  ItemType* const newItem = InsertAfter(pItem);
    3756  newItem->Value = value;
    3757  return newItem;
    3758 }
    3759 
    3760 template<typename T, typename AllocatorT>
    3761 class VmaList
    3762 {
    3763  VMA_CLASS_NO_COPY(VmaList)
    3764 public:
    3765  class iterator
    3766  {
    3767  public:
    3768  iterator() :
    3769  m_pList(VMA_NULL),
    3770  m_pItem(VMA_NULL)
    3771  {
    3772  }
    3773 
    3774  T& operator*() const
    3775  {
    3776  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3777  return m_pItem->Value;
    3778  }
    3779  T* operator->() const
    3780  {
    3781  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3782  return &m_pItem->Value;
    3783  }
    3784 
    3785  iterator& operator++()
    3786  {
    3787  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3788  m_pItem = m_pItem->pNext;
    3789  return *this;
    3790  }
    3791  iterator& operator--()
    3792  {
    3793  if(m_pItem != VMA_NULL)
    3794  {
    3795  m_pItem = m_pItem->pPrev;
    3796  }
    3797  else
    3798  {
    3799  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3800  m_pItem = m_pList->Back();
    3801  }
    3802  return *this;
    3803  }
    3804 
    3805  iterator operator++(int)
    3806  {
    3807  iterator result = *this;
    3808  ++*this;
    3809  return result;
    3810  }
    3811  iterator operator--(int)
    3812  {
    3813  iterator result = *this;
    3814  --*this;
    3815  return result;
    3816  }
    3817 
    3818  bool operator==(const iterator& rhs) const
    3819  {
    3820  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3821  return m_pItem == rhs.m_pItem;
    3822  }
    3823  bool operator!=(const iterator& rhs) const
    3824  {
    3825  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3826  return m_pItem != rhs.m_pItem;
    3827  }
    3828 
    3829  private:
    3830  VmaRawList<T>* m_pList;
    3831  VmaListItem<T>* m_pItem;
    3832 
    3833  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
    3834  m_pList(pList),
    3835  m_pItem(pItem)
    3836  {
    3837  }
    3838 
    3839  friend class VmaList<T, AllocatorT>;
    3840  };
    3841 
    3842  class const_iterator
    3843  {
    3844  public:
    3845  const_iterator() :
    3846  m_pList(VMA_NULL),
    3847  m_pItem(VMA_NULL)
    3848  {
    3849  }
    3850 
    3851  const_iterator(const iterator& src) :
    3852  m_pList(src.m_pList),
    3853  m_pItem(src.m_pItem)
    3854  {
    3855  }
    3856 
    3857  const T& operator*() const
    3858  {
    3859  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3860  return m_pItem->Value;
    3861  }
    3862  const T* operator->() const
    3863  {
    3864  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3865  return &m_pItem->Value;
    3866  }
    3867 
    3868  const_iterator& operator++()
    3869  {
    3870  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3871  m_pItem = m_pItem->pNext;
    3872  return *this;
    3873  }
    3874  const_iterator& operator--()
    3875  {
    3876  if(m_pItem != VMA_NULL)
    3877  {
    3878  m_pItem = m_pItem->pPrev;
    3879  }
    3880  else
    3881  {
    3882  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3883  m_pItem = m_pList->Back();
    3884  }
    3885  return *this;
    3886  }
    3887 
    3888  const_iterator operator++(int)
    3889  {
    3890  const_iterator result = *this;
    3891  ++*this;
    3892  return result;
    3893  }
    3894  const_iterator operator--(int)
    3895  {
    3896  const_iterator result = *this;
    3897  --*this;
    3898  return result;
    3899  }
    3900 
    3901  bool operator==(const const_iterator& rhs) const
    3902  {
    3903  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3904  return m_pItem == rhs.m_pItem;
    3905  }
    3906  bool operator!=(const const_iterator& rhs) const
    3907  {
    3908  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3909  return m_pItem != rhs.m_pItem;
    3910  }
    3911 
    3912  private:
    3913  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
    3914  m_pList(pList),
    3915  m_pItem(pItem)
    3916  {
    3917  }
    3918 
    3919  const VmaRawList<T>* m_pList;
    3920  const VmaListItem<T>* m_pItem;
    3921 
    3922  friend class VmaList<T, AllocatorT>;
    3923  };
    3924 
    3925  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
    3926 
    3927  bool empty() const { return m_RawList.IsEmpty(); }
    3928  size_t size() const { return m_RawList.GetCount(); }
    3929 
    3930  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
    3931  iterator end() { return iterator(&m_RawList, VMA_NULL); }
    3932 
    3933  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
    3934  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
    3935 
    3936  void clear() { m_RawList.Clear(); }
    3937  void push_back(const T& value) { m_RawList.PushBack(value); }
    3938  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
    3939  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
    3940 
    3941 private:
    3942  VmaRawList<T> m_RawList;
    3943 };
    3944 
    3945 #endif // #if VMA_USE_STL_LIST
    3946 
    3948 // class VmaMap
    3949 
    3950 // Unused in this version.
    3951 #if 0
    3952 
    3953 #if VMA_USE_STL_UNORDERED_MAP
    3954 
    3955 #define VmaPair std::pair
    3956 
    3957 #define VMA_MAP_TYPE(KeyT, ValueT) \
    3958  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
    3959 
    3960 #else // #if VMA_USE_STL_UNORDERED_MAP
    3961 
    3962 template<typename T1, typename T2>
    3963 struct VmaPair
    3964 {
    3965  T1 first;
    3966  T2 second;
    3967 
    3968  VmaPair() : first(), second() { }
    3969  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
    3970 };
    3971 
    3972 /* Class compatible with subset of interface of std::unordered_map.
    3973 KeyT, ValueT must be POD because they will be stored in VmaVector.
    3974 */
    3975 template<typename KeyT, typename ValueT>
    3976 class VmaMap
    3977 {
    3978 public:
    3979  typedef VmaPair<KeyT, ValueT> PairType;
    3980  typedef PairType* iterator;
    3981 
    3982  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
    3983 
    3984  iterator begin() { return m_Vector.begin(); }
    3985  iterator end() { return m_Vector.end(); }
    3986 
    3987  void insert(const PairType& pair);
    3988  iterator find(const KeyT& key);
    3989  void erase(iterator it);
    3990 
    3991 private:
    3992  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
    3993 };
    3994 
    3995 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
    3996 
    3997 template<typename FirstT, typename SecondT>
    3998 struct VmaPairFirstLess
    3999 {
    4000  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
    4001  {
    4002  return lhs.first < rhs.first;
    4003  }
    4004  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
    4005  {
    4006  return lhs.first < rhsFirst;
    4007  }
    4008 };
    4009 
    4010 template<typename KeyT, typename ValueT>
    4011 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
    4012 {
    4013  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    4014  m_Vector.data(),
    4015  m_Vector.data() + m_Vector.size(),
    4016  pair,
    4017  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
    4018  VmaVectorInsert(m_Vector, indexToInsert, pair);
    4019 }
    4020 
    4021 template<typename KeyT, typename ValueT>
    4022 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
    4023 {
    4024  PairType* it = VmaBinaryFindFirstNotLess(
    4025  m_Vector.data(),
    4026  m_Vector.data() + m_Vector.size(),
    4027  key,
    4028  VmaPairFirstLess<KeyT, ValueT>());
    4029  if((it != m_Vector.end()) && (it->first == key))
    4030  {
    4031  return it;
    4032  }
    4033  else
    4034  {
    4035  return m_Vector.end();
    4036  }
    4037 }
    4038 
    4039 template<typename KeyT, typename ValueT>
    4040 void VmaMap<KeyT, ValueT>::erase(iterator it)
    4041 {
    4042  VmaVectorRemove(m_Vector, it - m_Vector.begin());
    4043 }
    4044 
    4045 #endif // #if VMA_USE_STL_UNORDERED_MAP
    4046 
    4047 #endif // #if 0
    4048 
    4050 
    4051 class VmaDeviceMemoryBlock;
    4052 
    4053 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
    4054 
    4055 struct VmaAllocation_T
    4056 {
    4057  VMA_CLASS_NO_COPY(VmaAllocation_T)
    4058 private:
    4059  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
    4060 
    4061  enum FLAGS
    4062  {
    4063  FLAG_USER_DATA_STRING = 0x01,
    4064  };
    4065 
    4066 public:
    4067  enum ALLOCATION_TYPE
    4068  {
    4069  ALLOCATION_TYPE_NONE,
    4070  ALLOCATION_TYPE_BLOCK,
    4071  ALLOCATION_TYPE_DEDICATED,
    4072  };
    4073 
    4074  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
    4075  m_Alignment(1),
    4076  m_Size(0),
    4077  m_pUserData(VMA_NULL),
    4078  m_LastUseFrameIndex(currentFrameIndex),
    4079  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
    4080  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
    4081  m_MapCount(0),
    4082  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
    4083  {
    4084 #if VMA_STATS_STRING_ENABLED
    4085  m_CreationFrameIndex = currentFrameIndex;
    4086  m_BufferImageUsage = 0;
    4087 #endif
    4088  }
    4089 
    4090  ~VmaAllocation_T()
    4091  {
    4092  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
    4093 
    4094  // Check if owned string was freed.
    4095  VMA_ASSERT(m_pUserData == VMA_NULL);
    4096  }
    4097 
    4098  void InitBlockAllocation(
    4099  VmaPool hPool,
    4100  VmaDeviceMemoryBlock* block,
    4101  VkDeviceSize offset,
    4102  VkDeviceSize alignment,
    4103  VkDeviceSize size,
    4104  VmaSuballocationType suballocationType,
    4105  bool mapped,
    4106  bool canBecomeLost)
    4107  {
    4108  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4109  VMA_ASSERT(block != VMA_NULL);
    4110  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4111  m_Alignment = alignment;
    4112  m_Size = size;
    4113  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4114  m_SuballocationType = (uint8_t)suballocationType;
    4115  m_BlockAllocation.m_hPool = hPool;
    4116  m_BlockAllocation.m_Block = block;
    4117  m_BlockAllocation.m_Offset = offset;
    4118  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
    4119  }
    4120 
    4121  void InitLost()
    4122  {
    4123  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4124  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
    4125  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4126  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
    4127  m_BlockAllocation.m_Block = VMA_NULL;
    4128  m_BlockAllocation.m_Offset = 0;
    4129  m_BlockAllocation.m_CanBecomeLost = true;
    4130  }
    4131 
    4132  void ChangeBlockAllocation(
    4133  VmaAllocator hAllocator,
    4134  VmaDeviceMemoryBlock* block,
    4135  VkDeviceSize offset);
    4136 
    4137  // pMappedData not null means allocation is created with MAPPED flag.
    4138  void InitDedicatedAllocation(
    4139  uint32_t memoryTypeIndex,
    4140  VkDeviceMemory hMemory,
    4141  VmaSuballocationType suballocationType,
    4142  void* pMappedData,
    4143  VkDeviceSize size)
    4144  {
    4145  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4146  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
    4147  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
    4148  m_Alignment = 0;
    4149  m_Size = size;
    4150  m_SuballocationType = (uint8_t)suballocationType;
    4151  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4152  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
    4153  m_DedicatedAllocation.m_hMemory = hMemory;
    4154  m_DedicatedAllocation.m_pMappedData = pMappedData;
    4155  }
    4156 
    4157  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
    4158  VkDeviceSize GetAlignment() const { return m_Alignment; }
    4159  VkDeviceSize GetSize() const { return m_Size; }
    4160  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
    4161  void* GetUserData() const { return m_pUserData; }
    4162  void SetUserData(VmaAllocator hAllocator, void* pUserData);
    4163  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
    4164 
    4165  VmaDeviceMemoryBlock* GetBlock() const
    4166  {
    4167  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    4168  return m_BlockAllocation.m_Block;
    4169  }
    4170  VkDeviceSize GetOffset() const;
    4171  VkDeviceMemory GetMemory() const;
    4172  uint32_t GetMemoryTypeIndex() const;
    4173  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
    4174  void* GetMappedData() const;
    4175  bool CanBecomeLost() const;
    4176  VmaPool GetPool() const;
    4177 
    4178  uint32_t GetLastUseFrameIndex() const
    4179  {
    4180  return m_LastUseFrameIndex.load();
    4181  }
    4182  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
    4183  {
    4184  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
    4185  }
    4186  /*
    4187  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
    4188  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
    4189  - Else, returns false.
    4190 
    4191  If hAllocation is already lost, assert - you should not call it then.
    4192  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
    4193  */
    4194  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4195 
    4196  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
    4197  {
    4198  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
    4199  outInfo.blockCount = 1;
    4200  outInfo.allocationCount = 1;
    4201  outInfo.unusedRangeCount = 0;
    4202  outInfo.usedBytes = m_Size;
    4203  outInfo.unusedBytes = 0;
    4204  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
    4205  outInfo.unusedRangeSizeMin = UINT64_MAX;
    4206  outInfo.unusedRangeSizeMax = 0;
    4207  }
    4208 
    4209  void BlockAllocMap();
    4210  void BlockAllocUnmap();
    4211  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
    4212  void DedicatedAllocUnmap(VmaAllocator hAllocator);
    4213 
    4214 #if VMA_STATS_STRING_ENABLED
    4215  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
    4216  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
    4217 
    4218  void InitBufferImageUsage(uint32_t bufferImageUsage)
    4219  {
    4220  VMA_ASSERT(m_BufferImageUsage == 0);
    4221  m_BufferImageUsage = bufferImageUsage;
    4222  }
    4223 
    4224  void PrintParameters(class VmaJsonWriter& json) const;
    4225 #endif
    4226 
    4227 private:
    4228  VkDeviceSize m_Alignment;
    4229  VkDeviceSize m_Size;
    4230  void* m_pUserData;
    4231  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
    4232  uint8_t m_Type; // ALLOCATION_TYPE
    4233  uint8_t m_SuballocationType; // VmaSuballocationType
    4234  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
    4235  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
    4236  uint8_t m_MapCount;
    4237  uint8_t m_Flags; // enum FLAGS
    4238 
    4239  // Allocation out of VmaDeviceMemoryBlock.
    4240  struct BlockAllocation
    4241  {
    4242  VmaPool m_hPool; // Null if belongs to general memory.
    4243  VmaDeviceMemoryBlock* m_Block;
    4244  VkDeviceSize m_Offset;
    4245  bool m_CanBecomeLost;
    4246  };
    4247 
    4248  // Allocation for an object that has its own private VkDeviceMemory.
    4249  struct DedicatedAllocation
    4250  {
    4251  uint32_t m_MemoryTypeIndex;
    4252  VkDeviceMemory m_hMemory;
    4253  void* m_pMappedData; // Not null means memory is mapped.
    4254  };
    4255 
    4256  union
    4257  {
    4258  // Allocation out of VmaDeviceMemoryBlock.
    4259  BlockAllocation m_BlockAllocation;
    4260  // Allocation for an object that has its own private VkDeviceMemory.
    4261  DedicatedAllocation m_DedicatedAllocation;
    4262  };
    4263 
    4264 #if VMA_STATS_STRING_ENABLED
    4265  uint32_t m_CreationFrameIndex;
    4266  uint32_t m_BufferImageUsage; // 0 if unknown.
    4267 #endif
    4268 
    4269  void FreeUserDataString(VmaAllocator hAllocator);
    4270 };
    4271 
    4272 /*
    4273 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
    4274 allocated memory block or free.
    4275 */
    4276 struct VmaSuballocation
    4277 {
    4278  VkDeviceSize offset;
    4279  VkDeviceSize size;
    4280  VmaAllocation hAllocation;
    4281  VmaSuballocationType type;
    4282 };
    4283 
    4284 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
    4285 
    4286 // Cost of one additional allocation lost, as equivalent in bytes.
    4287 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
    4288 
    4289 /*
    4290 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
    4291 
    4292 If canMakeOtherLost was false:
    4293 - item points to a FREE suballocation.
    4294 - itemsToMakeLostCount is 0.
    4295 
    4296 If canMakeOtherLost was true:
    4297 - item points to first of sequence of suballocations, which are either FREE,
    4298  or point to VmaAllocations that can become lost.
    4299 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
    4300  the requested allocation to succeed.
    4301 */
    4302 struct VmaAllocationRequest
    4303 {
    4304  VkDeviceSize offset;
    4305  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
    4306  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
    4307  VmaSuballocationList::iterator item;
    4308  size_t itemsToMakeLostCount;
    4309 
    4310  VkDeviceSize CalcCost() const
    4311  {
    4312  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
    4313  }
    4314 };
    4315 
    4316 /*
    4317 Data structure used for bookkeeping of allocations and unused ranges of memory
    4318 in a single VkDeviceMemory block.
    4319 */
    4320 class VmaBlockMetadata
    4321 {
    4322  VMA_CLASS_NO_COPY(VmaBlockMetadata)
    4323 public:
    4324  VmaBlockMetadata(VmaAllocator hAllocator);
    4325  ~VmaBlockMetadata();
    4326  void Init(VkDeviceSize size);
    4327 
    4328  // Validates all data structures inside this object. If not valid, returns false.
    4329  bool Validate() const;
    4330  VkDeviceSize GetSize() const { return m_Size; }
    4331  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
    4332  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
    4333  VkDeviceSize GetUnusedRangeSizeMax() const;
    4334  // Returns true if this block is empty - contains only single free suballocation.
    4335  bool IsEmpty() const;
    4336 
    4337  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
    4338  void AddPoolStats(VmaPoolStats& inoutStats) const;
    4339 
    4340 #if VMA_STATS_STRING_ENABLED
    4341  void PrintDetailedMap(class VmaJsonWriter& json) const;
    4342 #endif
    4343 
    4344  // Tries to find a place for suballocation with given parameters inside this block.
    4345  // If succeeded, fills pAllocationRequest and returns true.
    4346  // If failed, returns false.
    4347  bool CreateAllocationRequest(
    4348  uint32_t currentFrameIndex,
    4349  uint32_t frameInUseCount,
    4350  VkDeviceSize bufferImageGranularity,
    4351  VkDeviceSize allocSize,
    4352  VkDeviceSize allocAlignment,
    4353  VmaSuballocationType allocType,
    4354  bool canMakeOtherLost,
    4355  VmaAllocationRequest* pAllocationRequest);
    4356 
    4357  bool MakeRequestedAllocationsLost(
    4358  uint32_t currentFrameIndex,
    4359  uint32_t frameInUseCount,
    4360  VmaAllocationRequest* pAllocationRequest);
    4361 
    4362  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4363 
    4364  VkResult CheckCorruption(const void* pBlockData);
    4365 
    4366  // Makes actual allocation based on request. Request must already be checked and valid.
    4367  void Alloc(
    4368  const VmaAllocationRequest& request,
    4369  VmaSuballocationType type,
    4370  VkDeviceSize allocSize,
    4371  VmaAllocation hAllocation);
    4372 
    4373  // Frees suballocation assigned to given memory region.
    4374  void Free(const VmaAllocation allocation);
    4375  void FreeAtOffset(VkDeviceSize offset);
    4376 
    4377 private:
    4378  VkDeviceSize m_Size;
    4379  uint32_t m_FreeCount;
    4380  VkDeviceSize m_SumFreeSize;
    4381  VmaSuballocationList m_Suballocations;
    4382  // Suballocations that are free and have size greater than certain threshold.
    4383  // Sorted by size, ascending.
    4384  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
    4385 
    4386  bool ValidateFreeSuballocationList() const;
    4387 
    4388  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
    4389  // If yes, fills pOffset and returns true. If no, returns false.
    4390  bool CheckAllocation(
    4391  uint32_t currentFrameIndex,
    4392  uint32_t frameInUseCount,
    4393  VkDeviceSize bufferImageGranularity,
    4394  VkDeviceSize allocSize,
    4395  VkDeviceSize allocAlignment,
    4396  VmaSuballocationType allocType,
    4397  VmaSuballocationList::const_iterator suballocItem,
    4398  bool canMakeOtherLost,
    4399  VkDeviceSize* pOffset,
    4400  size_t* itemsToMakeLostCount,
    4401  VkDeviceSize* pSumFreeSize,
    4402  VkDeviceSize* pSumItemSize) const;
    4403  // Given free suballocation, it merges it with following one, which must also be free.
    4404  void MergeFreeWithNext(VmaSuballocationList::iterator item);
    4405  // Releases given suballocation, making it free.
    4406  // Merges it with adjacent free suballocations if applicable.
    4407  // Returns iterator to new free suballocation at this place.
    4408  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
    4409  // Given free suballocation, it inserts it into sorted list of
    4410  // m_FreeSuballocationsBySize if it's suitable.
    4411  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
    4412  // Given free suballocation, it removes it from sorted list of
    4413  // m_FreeSuballocationsBySize if it's suitable.
    4414  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
    4415 };
    4416 
    4417 /*
    4418 Represents a single block of device memory (`VkDeviceMemory`) with all the
    4419 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
    4420 
    4421 Thread-safety: This class must be externally synchronized.
    4422 */
    4423 class VmaDeviceMemoryBlock
    4424 {
    4425  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
    4426 public:
    4427  VmaBlockMetadata m_Metadata;
    4428 
    4429  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
    4430 
    4431  ~VmaDeviceMemoryBlock()
    4432  {
    4433  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
    4434  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    4435  }
    4436 
    4437  // Always call after construction.
    4438  void Init(
    4439  uint32_t newMemoryTypeIndex,
    4440  VkDeviceMemory newMemory,
    4441  VkDeviceSize newSize,
    4442  uint32_t id);
    4443  // Always call before destruction.
    4444  void Destroy(VmaAllocator allocator);
    4445 
    4446  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
    4447  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4448  uint32_t GetId() const { return m_Id; }
    4449  void* GetMappedData() const { return m_pMappedData; }
    4450 
    4451  // Validates all data structures inside this object. If not valid, returns false.
    4452  bool Validate() const;
    4453 
    4454  VkResult CheckCorruption(VmaAllocator hAllocator);
    4455 
    4456  // ppData can be null.
    4457  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
    4458  void Unmap(VmaAllocator hAllocator, uint32_t count);
    4459 
    4460  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4461  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4462 
    4463  VkResult BindBufferMemory(
    4464  const VmaAllocator hAllocator,
    4465  const VmaAllocation hAllocation,
    4466  VkBuffer hBuffer);
    4467  VkResult BindImageMemory(
    4468  const VmaAllocator hAllocator,
    4469  const VmaAllocation hAllocation,
    4470  VkImage hImage);
    4471 
    4472 private:
    4473  uint32_t m_MemoryTypeIndex;
    4474  uint32_t m_Id;
    4475  VkDeviceMemory m_hMemory;
    4476 
    4477  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
    4478  // Also protects m_MapCount, m_pMappedData.
    4479  VMA_MUTEX m_Mutex;
    4480  uint32_t m_MapCount;
    4481  void* m_pMappedData;
    4482 };
    4483 
    4484 struct VmaPointerLess
    4485 {
    4486  bool operator()(const void* lhs, const void* rhs) const
    4487  {
    4488  return lhs < rhs;
    4489  }
    4490 };
    4491 
    4492 class VmaDefragmentator;
    4493 
    4494 /*
    4495 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
    4496 Vulkan memory type.
    4497 
    4498 Synchronized internally with a mutex.
    4499 */
    4500 struct VmaBlockVector
    4501 {
    4502  VMA_CLASS_NO_COPY(VmaBlockVector)
    4503 public:
    4504  VmaBlockVector(
    4505  VmaAllocator hAllocator,
    4506  uint32_t memoryTypeIndex,
    4507  VkDeviceSize preferredBlockSize,
    4508  size_t minBlockCount,
    4509  size_t maxBlockCount,
    4510  VkDeviceSize bufferImageGranularity,
    4511  uint32_t frameInUseCount,
    4512  bool isCustomPool);
    4513  ~VmaBlockVector();
    4514 
    4515  VkResult CreateMinBlocks();
    4516 
    4517  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4518  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
    4519  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
    4520  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
    4521 
    4522  void GetPoolStats(VmaPoolStats* pStats);
    4523 
    4524  bool IsEmpty() const { return m_Blocks.empty(); }
    4525  bool IsCorruptionDetectionEnabled() const;
    4526 
    4527  VkResult Allocate(
    4528  VmaPool hCurrentPool,
    4529  uint32_t currentFrameIndex,
    4530  VkDeviceSize size,
    4531  VkDeviceSize alignment,
    4532  const VmaAllocationCreateInfo& createInfo,
    4533  VmaSuballocationType suballocType,
    4534  VmaAllocation* pAllocation);
    4535 
    4536  void Free(
    4537  VmaAllocation hAllocation);
    4538 
    4539  // Adds statistics of this BlockVector to pStats.
    4540  void AddStats(VmaStats* pStats);
    4541 
    4542 #if VMA_STATS_STRING_ENABLED
    4543  void PrintDetailedMap(class VmaJsonWriter& json);
    4544 #endif
    4545 
    4546  void MakePoolAllocationsLost(
    4547  uint32_t currentFrameIndex,
    4548  size_t* pLostAllocationCount);
    4549  VkResult CheckCorruption();
    4550 
    4551  VmaDefragmentator* EnsureDefragmentator(
    4552  VmaAllocator hAllocator,
    4553  uint32_t currentFrameIndex);
    4554 
    4555  VkResult Defragment(
    4556  VmaDefragmentationStats* pDefragmentationStats,
    4557  VkDeviceSize& maxBytesToMove,
    4558  uint32_t& maxAllocationsToMove);
    4559 
    4560  void DestroyDefragmentator();
    4561 
    4562 private:
    4563  friend class VmaDefragmentator;
    4564 
    4565  const VmaAllocator m_hAllocator;
    4566  const uint32_t m_MemoryTypeIndex;
    4567  const VkDeviceSize m_PreferredBlockSize;
    4568  const size_t m_MinBlockCount;
    4569  const size_t m_MaxBlockCount;
    4570  const VkDeviceSize m_BufferImageGranularity;
    4571  const uint32_t m_FrameInUseCount;
    4572  const bool m_IsCustomPool;
    4573  VMA_MUTEX m_Mutex;
    4574  // Incrementally sorted by sumFreeSize, ascending.
    4575  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
    4576  /* There can be at most one allocation that is completely empty - a
    4577  hysteresis to avoid pessimistic case of alternating creation and destruction
    4578  of a VkDeviceMemory. */
    4579  bool m_HasEmptyBlock;
    4580  VmaDefragmentator* m_pDefragmentator;
    4581  uint32_t m_NextBlockId;
    4582 
    4583  VkDeviceSize CalcMaxBlockSize() const;
    4584 
    4585  // Finds and removes given block from vector.
    4586  void Remove(VmaDeviceMemoryBlock* pBlock);
    4587 
    4588  // Performs single step in sorting m_Blocks. They may not be fully sorted
    4589  // after this call.
    4590  void IncrementallySortBlocks();
    4591 
    4592  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
    4593 };
    4594 
    4595 struct VmaPool_T
    4596 {
    4597  VMA_CLASS_NO_COPY(VmaPool_T)
    4598 public:
    4599  VmaBlockVector m_BlockVector;
    4600 
    4601  VmaPool_T(
    4602  VmaAllocator hAllocator,
    4603  const VmaPoolCreateInfo& createInfo);
    4604  ~VmaPool_T();
    4605 
    4606  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
    4607  uint32_t GetId() const { return m_Id; }
    4608  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
    4609 
    4610 #if VMA_STATS_STRING_ENABLED
    4611  //void PrintDetailedMap(class VmaStringBuilder& sb);
    4612 #endif
    4613 
    4614 private:
    4615  uint32_t m_Id;
    4616 };
    4617 
    4618 class VmaDefragmentator
    4619 {
    4620  VMA_CLASS_NO_COPY(VmaDefragmentator)
    4621 private:
    4622  const VmaAllocator m_hAllocator;
    4623  VmaBlockVector* const m_pBlockVector;
    4624  uint32_t m_CurrentFrameIndex;
    4625  VkDeviceSize m_BytesMoved;
    4626  uint32_t m_AllocationsMoved;
    4627 
    4628  struct AllocationInfo
    4629  {
    4630  VmaAllocation m_hAllocation;
    4631  VkBool32* m_pChanged;
    4632 
    4633  AllocationInfo() :
    4634  m_hAllocation(VK_NULL_HANDLE),
    4635  m_pChanged(VMA_NULL)
    4636  {
    4637  }
    4638  };
    4639 
    4640  struct AllocationInfoSizeGreater
    4641  {
    4642  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
    4643  {
    4644  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
    4645  }
    4646  };
    4647 
    4648  // Used between AddAllocation and Defragment.
    4649  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4650 
    4651  struct BlockInfo
    4652  {
    4653  VmaDeviceMemoryBlock* m_pBlock;
    4654  bool m_HasNonMovableAllocations;
    4655  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4656 
    4657  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
    4658  m_pBlock(VMA_NULL),
    4659  m_HasNonMovableAllocations(true),
    4660  m_Allocations(pAllocationCallbacks),
    4661  m_pMappedDataForDefragmentation(VMA_NULL)
    4662  {
    4663  }
    4664 
    4665  void CalcHasNonMovableAllocations()
    4666  {
    4667  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
    4668  const size_t defragmentAllocCount = m_Allocations.size();
    4669  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
    4670  }
    4671 
    4672  void SortAllocationsBySizeDescecnding()
    4673  {
    4674  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
    4675  }
    4676 
    4677  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
    4678  void Unmap(VmaAllocator hAllocator);
    4679 
    4680  private:
    4681  // Not null if mapped for defragmentation only, not originally mapped.
    4682  void* m_pMappedDataForDefragmentation;
    4683  };
    4684 
    4685  struct BlockPointerLess
    4686  {
    4687  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
    4688  {
    4689  return pLhsBlockInfo->m_pBlock < pRhsBlock;
    4690  }
    4691  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4692  {
    4693  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
    4694  }
    4695  };
    4696 
    4697  // 1. Blocks with some non-movable allocations go first.
    4698  // 2. Blocks with smaller sumFreeSize go first.
    4699  struct BlockInfoCompareMoveDestination
    4700  {
    4701  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4702  {
    4703  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
    4704  {
    4705  return true;
    4706  }
    4707  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
    4708  {
    4709  return false;
    4710  }
    4711  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
    4712  {
    4713  return true;
    4714  }
    4715  return false;
    4716  }
    4717  };
    4718 
    4719  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
    4720  BlockInfoVector m_Blocks;
    4721 
    4722  VkResult DefragmentRound(
    4723  VkDeviceSize maxBytesToMove,
    4724  uint32_t maxAllocationsToMove);
    4725 
    4726  static bool MoveMakesSense(
    4727  size_t dstBlockIndex, VkDeviceSize dstOffset,
    4728  size_t srcBlockIndex, VkDeviceSize srcOffset);
    4729 
    4730 public:
    4731  VmaDefragmentator(
    4732  VmaAllocator hAllocator,
    4733  VmaBlockVector* pBlockVector,
    4734  uint32_t currentFrameIndex);
    4735 
    4736  ~VmaDefragmentator();
    4737 
    4738  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
    4739  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
    4740 
    4741  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
    4742 
    4743  VkResult Defragment(
    4744  VkDeviceSize maxBytesToMove,
    4745  uint32_t maxAllocationsToMove);
    4746 };
    4747 
    4748 #if VMA_RECORDING_ENABLED
    4749 
    4750 class VmaRecorder
    4751 {
    4752 public:
    4753  VmaRecorder();
    4754  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
    4755  ~VmaRecorder();
    4756 
    4757  void RecordCreateAllocator(uint32_t frameIndex);
    4758  void RecordDestroyAllocator(uint32_t frameIndex);
    4759  void RecordCreatePool(uint32_t frameIndex,
    4760  const VmaPoolCreateInfo& createInfo,
    4761  VmaPool pool);
    4762  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
    4763  void RecordAllocateMemory(uint32_t frameIndex,
    4764  const VkMemoryRequirements& vkMemReq,
    4765  const VmaAllocationCreateInfo& createInfo,
    4766  VmaAllocation allocation);
    4767  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    4768  const VkMemoryRequirements& vkMemReq,
    4769  bool requiresDedicatedAllocation,
    4770  bool prefersDedicatedAllocation,
    4771  const VmaAllocationCreateInfo& createInfo,
    4772  VmaAllocation allocation);
    4773  void RecordAllocateMemoryForImage(uint32_t frameIndex,
    4774  const VkMemoryRequirements& vkMemReq,
    4775  bool requiresDedicatedAllocation,
    4776  bool prefersDedicatedAllocation,
    4777  const VmaAllocationCreateInfo& createInfo,
    4778  VmaAllocation allocation);
    4779  void RecordFreeMemory(uint32_t frameIndex,
    4780  VmaAllocation allocation);
    4781  void RecordSetAllocationUserData(uint32_t frameIndex,
    4782  VmaAllocation allocation,
    4783  const void* pUserData);
    4784  void RecordCreateLostAllocation(uint32_t frameIndex,
    4785  VmaAllocation allocation);
    4786  void RecordMapMemory(uint32_t frameIndex,
    4787  VmaAllocation allocation);
    4788  void RecordUnmapMemory(uint32_t frameIndex,
    4789  VmaAllocation allocation);
    4790  void RecordFlushAllocation(uint32_t frameIndex,
    4791  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4792  void RecordInvalidateAllocation(uint32_t frameIndex,
    4793  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4794  void RecordCreateBuffer(uint32_t frameIndex,
    4795  const VkBufferCreateInfo& bufCreateInfo,
    4796  const VmaAllocationCreateInfo& allocCreateInfo,
    4797  VmaAllocation allocation);
    4798  void RecordCreateImage(uint32_t frameIndex,
    4799  const VkImageCreateInfo& imageCreateInfo,
    4800  const VmaAllocationCreateInfo& allocCreateInfo,
    4801  VmaAllocation allocation);
    4802  void RecordDestroyBuffer(uint32_t frameIndex,
    4803  VmaAllocation allocation);
    4804  void RecordDestroyImage(uint32_t frameIndex,
    4805  VmaAllocation allocation);
    4806 
    4807 private:
    4808  struct CallParams
    4809  {
    4810  uint32_t threadId;
    4811  double time;
    4812  };
    4813 
    4814  class UserDataString
    4815  {
    4816  public:
    4817  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
    4818  const char* GetString() const { return m_Str; }
    4819 
    4820  private:
    4821  char m_PtrStr[17];
    4822  const char* m_Str;
    4823  };
    4824 
    4825  bool m_UseMutex;
    4826  VmaRecordFlags m_Flags;
    4827  FILE* m_File;
    4828  VMA_MUTEX m_FileMutex;
    4829  int64_t m_Freq;
    4830  int64_t m_StartCounter;
    4831 
    4832  void GetBasicParams(CallParams& outParams);
    4833  void Flush();
    4834 };
    4835 
    4836 #endif // #if VMA_RECORDING_ENABLED
    4837 
    4838 // Main allocator object.
    4839 struct VmaAllocator_T
    4840 {
    4841  VMA_CLASS_NO_COPY(VmaAllocator_T)
    4842 public:
    4843  bool m_UseMutex;
    4844  bool m_UseKhrDedicatedAllocation;
    4845  VkDevice m_hDevice;
    4846  bool m_AllocationCallbacksSpecified;
    4847  VkAllocationCallbacks m_AllocationCallbacks;
    4848  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
    4849 
    4850  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
    4851  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
    4852  VMA_MUTEX m_HeapSizeLimitMutex;
    4853 
    4854  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
    4855  VkPhysicalDeviceMemoryProperties m_MemProps;
    4856 
    4857  // Default pools.
    4858  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
    4859 
    4860  // Each vector is sorted by memory (handle value).
    4861  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
    4862  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
    4863  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
    4864 
    4865  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
    4866  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
    4867  ~VmaAllocator_T();
    4868 
    4869  const VkAllocationCallbacks* GetAllocationCallbacks() const
    4870  {
    4871  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
    4872  }
    4873  const VmaVulkanFunctions& GetVulkanFunctions() const
    4874  {
    4875  return m_VulkanFunctions;
    4876  }
    4877 
    4878  VkDeviceSize GetBufferImageGranularity() const
    4879  {
    4880  return VMA_MAX(
    4881  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
    4882  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
    4883  }
    4884 
    4885  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
    4886  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
    4887 
    4888  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
    4889  {
    4890  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
    4891  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
    4892  }
    4893  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
    4894  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
    4895  {
    4896  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
    4897  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    4898  }
    4899  // Minimum alignment for all allocations in specific memory type.
    4900  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
    4901  {
    4902  return IsMemoryTypeNonCoherent(memTypeIndex) ?
    4903  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
    4904  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
    4905  }
    4906 
    4907  bool IsIntegratedGpu() const
    4908  {
    4909  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
    4910  }
    4911 
    4912 #if VMA_RECORDING_ENABLED
    4913  VmaRecorder* GetRecorder() const { return m_pRecorder; }
    4914 #endif
    4915 
    4916  void GetBufferMemoryRequirements(
    4917  VkBuffer hBuffer,
    4918  VkMemoryRequirements& memReq,
    4919  bool& requiresDedicatedAllocation,
    4920  bool& prefersDedicatedAllocation) const;
    4921  void GetImageMemoryRequirements(
    4922  VkImage hImage,
    4923  VkMemoryRequirements& memReq,
    4924  bool& requiresDedicatedAllocation,
    4925  bool& prefersDedicatedAllocation) const;
    4926 
    4927  // Main allocation function.
    4928  VkResult AllocateMemory(
    4929  const VkMemoryRequirements& vkMemReq,
    4930  bool requiresDedicatedAllocation,
    4931  bool prefersDedicatedAllocation,
    4932  VkBuffer dedicatedBuffer,
    4933  VkImage dedicatedImage,
    4934  const VmaAllocationCreateInfo& createInfo,
    4935  VmaSuballocationType suballocType,
    4936  VmaAllocation* pAllocation);
    4937 
    4938  // Main deallocation function.
    4939  void FreeMemory(const VmaAllocation allocation);
    4940 
    4941  void CalculateStats(VmaStats* pStats);
    4942 
    4943 #if VMA_STATS_STRING_ENABLED
    4944  void PrintDetailedMap(class VmaJsonWriter& json);
    4945 #endif
    4946 
    4947  VkResult Defragment(
    4948  VmaAllocation* pAllocations,
    4949  size_t allocationCount,
    4950  VkBool32* pAllocationsChanged,
    4951  const VmaDefragmentationInfo* pDefragmentationInfo,
    4952  VmaDefragmentationStats* pDefragmentationStats);
    4953 
    4954  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
    4955  bool TouchAllocation(VmaAllocation hAllocation);
    4956 
    4957  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
    4958  void DestroyPool(VmaPool pool);
    4959  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
    4960 
    4961  void SetCurrentFrameIndex(uint32_t frameIndex);
    4962  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
    4963 
    4964  void MakePoolAllocationsLost(
    4965  VmaPool hPool,
    4966  size_t* pLostAllocationCount);
    4967  VkResult CheckPoolCorruption(VmaPool hPool);
    4968  VkResult CheckCorruption(uint32_t memoryTypeBits);
    4969 
    4970  void CreateLostAllocation(VmaAllocation* pAllocation);
    4971 
    4972  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
    4973  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
    4974 
    4975  VkResult Map(VmaAllocation hAllocation, void** ppData);
    4976  void Unmap(VmaAllocation hAllocation);
    4977 
    4978  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
    4979  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
    4980 
    4981  void FlushOrInvalidateAllocation(
    4982  VmaAllocation hAllocation,
    4983  VkDeviceSize offset, VkDeviceSize size,
    4984  VMA_CACHE_OPERATION op);
    4985 
    4986  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
    4987 
    4988 private:
    4989  VkDeviceSize m_PreferredLargeHeapBlockSize;
    4990 
    4991  VkPhysicalDevice m_PhysicalDevice;
    4992  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
    4993 
    4994  VMA_MUTEX m_PoolsMutex;
    4995  // Protected by m_PoolsMutex. Sorted by pointer value.
    4996  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
    4997  uint32_t m_NextPoolId;
    4998 
    4999  VmaVulkanFunctions m_VulkanFunctions;
    5000 
    5001 #if VMA_RECORDING_ENABLED
    5002  VmaRecorder* m_pRecorder;
    5003 #endif
    5004 
    5005  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
    5006 
    5007  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
    5008 
    5009  VkResult AllocateMemoryOfType(
    5010  VkDeviceSize size,
    5011  VkDeviceSize alignment,
    5012  bool dedicatedAllocation,
    5013  VkBuffer dedicatedBuffer,
    5014  VkImage dedicatedImage,
    5015  const VmaAllocationCreateInfo& createInfo,
    5016  uint32_t memTypeIndex,
    5017  VmaSuballocationType suballocType,
    5018  VmaAllocation* pAllocation);
    5019 
    5020  // Allocates and registers new VkDeviceMemory specifically for single allocation.
    5021  VkResult AllocateDedicatedMemory(
    5022  VkDeviceSize size,
    5023  VmaSuballocationType suballocType,
    5024  uint32_t memTypeIndex,
    5025  bool map,
    5026  bool isUserDataString,
    5027  void* pUserData,
    5028  VkBuffer dedicatedBuffer,
    5029  VkImage dedicatedImage,
    5030  VmaAllocation* pAllocation);
    5031 
    5032  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
    5033  void FreeDedicatedMemory(VmaAllocation allocation);
    5034 };
    5035 
    5037 // Memory allocation #2 after VmaAllocator_T definition
    5038 
    5039 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
    5040 {
    5041  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
    5042 }
    5043 
    5044 static void VmaFree(VmaAllocator hAllocator, void* ptr)
    5045 {
    5046  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
    5047 }
    5048 
    5049 template<typename T>
    5050 static T* VmaAllocate(VmaAllocator hAllocator)
    5051 {
    5052  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
    5053 }
    5054 
    5055 template<typename T>
    5056 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
    5057 {
    5058  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
    5059 }
    5060 
    5061 template<typename T>
    5062 static void vma_delete(VmaAllocator hAllocator, T* ptr)
    5063 {
    5064  if(ptr != VMA_NULL)
    5065  {
    5066  ptr->~T();
    5067  VmaFree(hAllocator, ptr);
    5068  }
    5069 }
    5070 
    5071 template<typename T>
    5072 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
    5073 {
    5074  if(ptr != VMA_NULL)
    5075  {
    5076  for(size_t i = count; i--; )
    5077  ptr[i].~T();
    5078  VmaFree(hAllocator, ptr);
    5079  }
    5080 }
    5081 
    5083 // VmaStringBuilder
    5084 
    5085 #if VMA_STATS_STRING_ENABLED
    5086 
    5087 class VmaStringBuilder
    5088 {
    5089 public:
    5090  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
    5091  size_t GetLength() const { return m_Data.size(); }
    5092  const char* GetData() const { return m_Data.data(); }
    5093 
    5094  void Add(char ch) { m_Data.push_back(ch); }
    5095  void Add(const char* pStr);
    5096  void AddNewLine() { Add('\n'); }
    5097  void AddNumber(uint32_t num);
    5098  void AddNumber(uint64_t num);
    5099  void AddPointer(const void* ptr);
    5100 
    5101 private:
    5102  VmaVector< char, VmaStlAllocator<char> > m_Data;
    5103 };
    5104 
    5105 void VmaStringBuilder::Add(const char* pStr)
    5106 {
    5107  const size_t strLen = strlen(pStr);
    5108  if(strLen > 0)
    5109  {
    5110  const size_t oldCount = m_Data.size();
    5111  m_Data.resize(oldCount + strLen);
    5112  memcpy(m_Data.data() + oldCount, pStr, strLen);
    5113  }
    5114 }
    5115 
    5116 void VmaStringBuilder::AddNumber(uint32_t num)
    5117 {
    5118  char buf[11];
    5119  VmaUint32ToStr(buf, sizeof(buf), num);
    5120  Add(buf);
    5121 }
    5122 
    5123 void VmaStringBuilder::AddNumber(uint64_t num)
    5124 {
    5125  char buf[21];
    5126  VmaUint64ToStr(buf, sizeof(buf), num);
    5127  Add(buf);
    5128 }
    5129 
    5130 void VmaStringBuilder::AddPointer(const void* ptr)
    5131 {
    5132  char buf[21];
    5133  VmaPtrToStr(buf, sizeof(buf), ptr);
    5134  Add(buf);
    5135 }
    5136 
    5137 #endif // #if VMA_STATS_STRING_ENABLED
    5138 
    5140 // VmaJsonWriter
    5141 
    5142 #if VMA_STATS_STRING_ENABLED
    5143 
    5144 class VmaJsonWriter
    5145 {
    5146  VMA_CLASS_NO_COPY(VmaJsonWriter)
    5147 public:
    5148  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
    5149  ~VmaJsonWriter();
    5150 
    5151  void BeginObject(bool singleLine = false);
    5152  void EndObject();
    5153 
    5154  void BeginArray(bool singleLine = false);
    5155  void EndArray();
    5156 
    5157  void WriteString(const char* pStr);
    5158  void BeginString(const char* pStr = VMA_NULL);
    5159  void ContinueString(const char* pStr);
    5160  void ContinueString(uint32_t n);
    5161  void ContinueString(uint64_t n);
    5162  void ContinueString_Pointer(const void* ptr);
    5163  void EndString(const char* pStr = VMA_NULL);
    5164 
    5165  void WriteNumber(uint32_t n);
    5166  void WriteNumber(uint64_t n);
    5167  void WriteBool(bool b);
    5168  void WriteNull();
    5169 
    5170 private:
    5171  static const char* const INDENT;
    5172 
    5173  enum COLLECTION_TYPE
    5174  {
    5175  COLLECTION_TYPE_OBJECT,
    5176  COLLECTION_TYPE_ARRAY,
    5177  };
    5178  struct StackItem
    5179  {
    5180  COLLECTION_TYPE type;
    5181  uint32_t valueCount;
    5182  bool singleLineMode;
    5183  };
    5184 
    5185  VmaStringBuilder& m_SB;
    5186  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
    5187  bool m_InsideString;
    5188 
    5189  void BeginValue(bool isString);
    5190  void WriteIndent(bool oneLess = false);
    5191 };
    5192 
    5193 const char* const VmaJsonWriter::INDENT = " ";
    5194 
    5195 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
    5196  m_SB(sb),
    5197  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
    5198  m_InsideString(false)
    5199 {
    5200 }
    5201 
    5202 VmaJsonWriter::~VmaJsonWriter()
    5203 {
    5204  VMA_ASSERT(!m_InsideString);
    5205  VMA_ASSERT(m_Stack.empty());
    5206 }
    5207 
    5208 void VmaJsonWriter::BeginObject(bool singleLine)
    5209 {
    5210  VMA_ASSERT(!m_InsideString);
    5211 
    5212  BeginValue(false);
    5213  m_SB.Add('{');
    5214 
    5215  StackItem item;
    5216  item.type = COLLECTION_TYPE_OBJECT;
    5217  item.valueCount = 0;
    5218  item.singleLineMode = singleLine;
    5219  m_Stack.push_back(item);
    5220 }
    5221 
    5222 void VmaJsonWriter::EndObject()
    5223 {
    5224  VMA_ASSERT(!m_InsideString);
    5225 
    5226  WriteIndent(true);
    5227  m_SB.Add('}');
    5228 
    5229  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
    5230  m_Stack.pop_back();
    5231 }
    5232 
    5233 void VmaJsonWriter::BeginArray(bool singleLine)
    5234 {
    5235  VMA_ASSERT(!m_InsideString);
    5236 
    5237  BeginValue(false);
    5238  m_SB.Add('[');
    5239 
    5240  StackItem item;
    5241  item.type = COLLECTION_TYPE_ARRAY;
    5242  item.valueCount = 0;
    5243  item.singleLineMode = singleLine;
    5244  m_Stack.push_back(item);
    5245 }
    5246 
    5247 void VmaJsonWriter::EndArray()
    5248 {
    5249  VMA_ASSERT(!m_InsideString);
    5250 
    5251  WriteIndent(true);
    5252  m_SB.Add(']');
    5253 
    5254  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
    5255  m_Stack.pop_back();
    5256 }
    5257 
    5258 void VmaJsonWriter::WriteString(const char* pStr)
    5259 {
    5260  BeginString(pStr);
    5261  EndString();
    5262 }
    5263 
    5264 void VmaJsonWriter::BeginString(const char* pStr)
    5265 {
    5266  VMA_ASSERT(!m_InsideString);
    5267 
    5268  BeginValue(true);
    5269  m_SB.Add('"');
    5270  m_InsideString = true;
    5271  if(pStr != VMA_NULL && pStr[0] != '\0')
    5272  {
    5273  ContinueString(pStr);
    5274  }
    5275 }
    5276 
    5277 void VmaJsonWriter::ContinueString(const char* pStr)
    5278 {
    5279  VMA_ASSERT(m_InsideString);
    5280 
    5281  const size_t strLen = strlen(pStr);
    5282  for(size_t i = 0; i < strLen; ++i)
    5283  {
    5284  char ch = pStr[i];
    5285  if(ch == '\\')
    5286  {
    5287  m_SB.Add("\\\\");
    5288  }
    5289  else if(ch == '"')
    5290  {
    5291  m_SB.Add("\\\"");
    5292  }
    5293  else if(ch >= 32)
    5294  {
    5295  m_SB.Add(ch);
    5296  }
    5297  else switch(ch)
    5298  {
    5299  case '\b':
    5300  m_SB.Add("\\b");
    5301  break;
    5302  case '\f':
    5303  m_SB.Add("\\f");
    5304  break;
    5305  case '\n':
    5306  m_SB.Add("\\n");
    5307  break;
    5308  case '\r':
    5309  m_SB.Add("\\r");
    5310  break;
    5311  case '\t':
    5312  m_SB.Add("\\t");
    5313  break;
    5314  default:
    5315  VMA_ASSERT(0 && "Character not currently supported.");
    5316  break;
    5317  }
    5318  }
    5319 }
    5320 
    5321 void VmaJsonWriter::ContinueString(uint32_t n)
    5322 {
    5323  VMA_ASSERT(m_InsideString);
    5324  m_SB.AddNumber(n);
    5325 }
    5326 
    5327 void VmaJsonWriter::ContinueString(uint64_t n)
    5328 {
    5329  VMA_ASSERT(m_InsideString);
    5330  m_SB.AddNumber(n);
    5331 }
    5332 
    5333 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
    5334 {
    5335  VMA_ASSERT(m_InsideString);
    5336  m_SB.AddPointer(ptr);
    5337 }
    5338 
    5339 void VmaJsonWriter::EndString(const char* pStr)
    5340 {
    5341  VMA_ASSERT(m_InsideString);
    5342  if(pStr != VMA_NULL && pStr[0] != '\0')
    5343  {
    5344  ContinueString(pStr);
    5345  }
    5346  m_SB.Add('"');
    5347  m_InsideString = false;
    5348 }
    5349 
    5350 void VmaJsonWriter::WriteNumber(uint32_t n)
    5351 {
    5352  VMA_ASSERT(!m_InsideString);
    5353  BeginValue(false);
    5354  m_SB.AddNumber(n);
    5355 }
    5356 
    5357 void VmaJsonWriter::WriteNumber(uint64_t n)
    5358 {
    5359  VMA_ASSERT(!m_InsideString);
    5360  BeginValue(false);
    5361  m_SB.AddNumber(n);
    5362 }
    5363 
    5364 void VmaJsonWriter::WriteBool(bool b)
    5365 {
    5366  VMA_ASSERT(!m_InsideString);
    5367  BeginValue(false);
    5368  m_SB.Add(b ? "true" : "false");
    5369 }
    5370 
    5371 void VmaJsonWriter::WriteNull()
    5372 {
    5373  VMA_ASSERT(!m_InsideString);
    5374  BeginValue(false);
    5375  m_SB.Add("null");
    5376 }
    5377 
    5378 void VmaJsonWriter::BeginValue(bool isString)
    5379 {
    5380  if(!m_Stack.empty())
    5381  {
    5382  StackItem& currItem = m_Stack.back();
    5383  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5384  currItem.valueCount % 2 == 0)
    5385  {
    5386  VMA_ASSERT(isString);
    5387  }
    5388 
    5389  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5390  currItem.valueCount % 2 != 0)
    5391  {
    5392  m_SB.Add(": ");
    5393  }
    5394  else if(currItem.valueCount > 0)
    5395  {
    5396  m_SB.Add(", ");
    5397  WriteIndent();
    5398  }
    5399  else
    5400  {
    5401  WriteIndent();
    5402  }
    5403  ++currItem.valueCount;
    5404  }
    5405 }
    5406 
    5407 void VmaJsonWriter::WriteIndent(bool oneLess)
    5408 {
    5409  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
    5410  {
    5411  m_SB.AddNewLine();
    5412 
    5413  size_t count = m_Stack.size();
    5414  if(count > 0 && oneLess)
    5415  {
    5416  --count;
    5417  }
    5418  for(size_t i = 0; i < count; ++i)
    5419  {
    5420  m_SB.Add(INDENT);
    5421  }
    5422  }
    5423 }
    5424 
    5425 #endif // #if VMA_STATS_STRING_ENABLED
    5426 
    5428 
    5429 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
    5430 {
    5431  if(IsUserDataString())
    5432  {
    5433  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
    5434 
    5435  FreeUserDataString(hAllocator);
    5436 
    5437  if(pUserData != VMA_NULL)
    5438  {
    5439  const char* const newStrSrc = (char*)pUserData;
    5440  const size_t newStrLen = strlen(newStrSrc);
    5441  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
    5442  memcpy(newStrDst, newStrSrc, newStrLen + 1);
    5443  m_pUserData = newStrDst;
    5444  }
    5445  }
    5446  else
    5447  {
    5448  m_pUserData = pUserData;
    5449  }
    5450 }
    5451 
    5452 void VmaAllocation_T::ChangeBlockAllocation(
    5453  VmaAllocator hAllocator,
    5454  VmaDeviceMemoryBlock* block,
    5455  VkDeviceSize offset)
    5456 {
    5457  VMA_ASSERT(block != VMA_NULL);
    5458  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5459 
    5460  // Move mapping reference counter from old block to new block.
    5461  if(block != m_BlockAllocation.m_Block)
    5462  {
    5463  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
    5464  if(IsPersistentMap())
    5465  ++mapRefCount;
    5466  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
    5467  block->Map(hAllocator, mapRefCount, VMA_NULL);
    5468  }
    5469 
    5470  m_BlockAllocation.m_Block = block;
    5471  m_BlockAllocation.m_Offset = offset;
    5472 }
    5473 
    5474 VkDeviceSize VmaAllocation_T::GetOffset() const
    5475 {
    5476  switch(m_Type)
    5477  {
    5478  case ALLOCATION_TYPE_BLOCK:
    5479  return m_BlockAllocation.m_Offset;
    5480  case ALLOCATION_TYPE_DEDICATED:
    5481  return 0;
    5482  default:
    5483  VMA_ASSERT(0);
    5484  return 0;
    5485  }
    5486 }
    5487 
    5488 VkDeviceMemory VmaAllocation_T::GetMemory() const
    5489 {
    5490  switch(m_Type)
    5491  {
    5492  case ALLOCATION_TYPE_BLOCK:
    5493  return m_BlockAllocation.m_Block->GetDeviceMemory();
    5494  case ALLOCATION_TYPE_DEDICATED:
    5495  return m_DedicatedAllocation.m_hMemory;
    5496  default:
    5497  VMA_ASSERT(0);
    5498  return VK_NULL_HANDLE;
    5499  }
    5500 }
    5501 
    5502 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
    5503 {
    5504  switch(m_Type)
    5505  {
    5506  case ALLOCATION_TYPE_BLOCK:
    5507  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
    5508  case ALLOCATION_TYPE_DEDICATED:
    5509  return m_DedicatedAllocation.m_MemoryTypeIndex;
    5510  default:
    5511  VMA_ASSERT(0);
    5512  return UINT32_MAX;
    5513  }
    5514 }
    5515 
    5516 void* VmaAllocation_T::GetMappedData() const
    5517 {
    5518  switch(m_Type)
    5519  {
    5520  case ALLOCATION_TYPE_BLOCK:
    5521  if(m_MapCount != 0)
    5522  {
    5523  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
    5524  VMA_ASSERT(pBlockData != VMA_NULL);
    5525  return (char*)pBlockData + m_BlockAllocation.m_Offset;
    5526  }
    5527  else
    5528  {
    5529  return VMA_NULL;
    5530  }
    5531  break;
    5532  case ALLOCATION_TYPE_DEDICATED:
    5533  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
    5534  return m_DedicatedAllocation.m_pMappedData;
    5535  default:
    5536  VMA_ASSERT(0);
    5537  return VMA_NULL;
    5538  }
    5539 }
    5540 
    5541 bool VmaAllocation_T::CanBecomeLost() const
    5542 {
    5543  switch(m_Type)
    5544  {
    5545  case ALLOCATION_TYPE_BLOCK:
    5546  return m_BlockAllocation.m_CanBecomeLost;
    5547  case ALLOCATION_TYPE_DEDICATED:
    5548  return false;
    5549  default:
    5550  VMA_ASSERT(0);
    5551  return false;
    5552  }
    5553 }
    5554 
    5555 VmaPool VmaAllocation_T::GetPool() const
    5556 {
    5557  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5558  return m_BlockAllocation.m_hPool;
    5559 }
    5560 
    5561 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    5562 {
    5563  VMA_ASSERT(CanBecomeLost());
    5564 
    5565  /*
    5566  Warning: This is a carefully designed algorithm.
    5567  Do not modify unless you really know what you're doing :)
    5568  */
    5569  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
    5570  for(;;)
    5571  {
    5572  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    5573  {
    5574  VMA_ASSERT(0);
    5575  return false;
    5576  }
    5577  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
    5578  {
    5579  return false;
    5580  }
    5581  else // Last use time earlier than current time.
    5582  {
    5583  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
    5584  {
    5585  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
    5586  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
    5587  return true;
    5588  }
    5589  }
    5590  }
    5591 }
    5592 
    5593 #if VMA_STATS_STRING_ENABLED
    5594 
    5595 // Correspond to values of enum VmaSuballocationType.
    5596 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
    5597  "FREE",
    5598  "UNKNOWN",
    5599  "BUFFER",
    5600  "IMAGE_UNKNOWN",
    5601  "IMAGE_LINEAR",
    5602  "IMAGE_OPTIMAL",
    5603 };
    5604 
    5605 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
    5606 {
    5607  json.WriteString("Type");
    5608  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
    5609 
    5610  json.WriteString("Size");
    5611  json.WriteNumber(m_Size);
    5612 
    5613  if(m_pUserData != VMA_NULL)
    5614  {
    5615  json.WriteString("UserData");
    5616  if(IsUserDataString())
    5617  {
    5618  json.WriteString((const char*)m_pUserData);
    5619  }
    5620  else
    5621  {
    5622  json.BeginString();
    5623  json.ContinueString_Pointer(m_pUserData);
    5624  json.EndString();
    5625  }
    5626  }
    5627 
    5628  json.WriteString("CreationFrameIndex");
    5629  json.WriteNumber(m_CreationFrameIndex);
    5630 
    5631  json.WriteString("LastUseFrameIndex");
    5632  json.WriteNumber(GetLastUseFrameIndex());
    5633 
    5634  if(m_BufferImageUsage != 0)
    5635  {
    5636  json.WriteString("Usage");
    5637  json.WriteNumber(m_BufferImageUsage);
    5638  }
    5639 }
    5640 
    5641 #endif
    5642 
    5643 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
    5644 {
    5645  VMA_ASSERT(IsUserDataString());
    5646  if(m_pUserData != VMA_NULL)
    5647  {
    5648  char* const oldStr = (char*)m_pUserData;
    5649  const size_t oldStrLen = strlen(oldStr);
    5650  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
    5651  m_pUserData = VMA_NULL;
    5652  }
    5653 }
    5654 
    5655 void VmaAllocation_T::BlockAllocMap()
    5656 {
    5657  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5658 
    5659  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5660  {
    5661  ++m_MapCount;
    5662  }
    5663  else
    5664  {
    5665  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
    5666  }
    5667 }
    5668 
    5669 void VmaAllocation_T::BlockAllocUnmap()
    5670 {
    5671  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5672 
    5673  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5674  {
    5675  --m_MapCount;
    5676  }
    5677  else
    5678  {
    5679  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
    5680  }
    5681 }
    5682 
    5683 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
    5684 {
    5685  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5686 
    5687  if(m_MapCount != 0)
    5688  {
    5689  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5690  {
    5691  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
    5692  *ppData = m_DedicatedAllocation.m_pMappedData;
    5693  ++m_MapCount;
    5694  return VK_SUCCESS;
    5695  }
    5696  else
    5697  {
    5698  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
    5699  return VK_ERROR_MEMORY_MAP_FAILED;
    5700  }
    5701  }
    5702  else
    5703  {
    5704  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    5705  hAllocator->m_hDevice,
    5706  m_DedicatedAllocation.m_hMemory,
    5707  0, // offset
    5708  VK_WHOLE_SIZE,
    5709  0, // flags
    5710  ppData);
    5711  if(result == VK_SUCCESS)
    5712  {
    5713  m_DedicatedAllocation.m_pMappedData = *ppData;
    5714  m_MapCount = 1;
    5715  }
    5716  return result;
    5717  }
    5718 }
    5719 
    5720 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
    5721 {
    5722  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5723 
    5724  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5725  {
    5726  --m_MapCount;
    5727  if(m_MapCount == 0)
    5728  {
    5729  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
    5730  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
    5731  hAllocator->m_hDevice,
    5732  m_DedicatedAllocation.m_hMemory);
    5733  }
    5734  }
    5735  else
    5736  {
    5737  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
    5738  }
    5739 }
    5740 
    5741 #if VMA_STATS_STRING_ENABLED
    5742 
    5743 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
    5744 {
    5745  json.BeginObject();
    5746 
    5747  json.WriteString("Blocks");
    5748  json.WriteNumber(stat.blockCount);
    5749 
    5750  json.WriteString("Allocations");
    5751  json.WriteNumber(stat.allocationCount);
    5752 
    5753  json.WriteString("UnusedRanges");
    5754  json.WriteNumber(stat.unusedRangeCount);
    5755 
    5756  json.WriteString("UsedBytes");
    5757  json.WriteNumber(stat.usedBytes);
    5758 
    5759  json.WriteString("UnusedBytes");
    5760  json.WriteNumber(stat.unusedBytes);
    5761 
    5762  if(stat.allocationCount > 1)
    5763  {
    5764  json.WriteString("AllocationSize");
    5765  json.BeginObject(true);
    5766  json.WriteString("Min");
    5767  json.WriteNumber(stat.allocationSizeMin);
    5768  json.WriteString("Avg");
    5769  json.WriteNumber(stat.allocationSizeAvg);
    5770  json.WriteString("Max");
    5771  json.WriteNumber(stat.allocationSizeMax);
    5772  json.EndObject();
    5773  }
    5774 
    5775  if(stat.unusedRangeCount > 1)
    5776  {
    5777  json.WriteString("UnusedRangeSize");
    5778  json.BeginObject(true);
    5779  json.WriteString("Min");
    5780  json.WriteNumber(stat.unusedRangeSizeMin);
    5781  json.WriteString("Avg");
    5782  json.WriteNumber(stat.unusedRangeSizeAvg);
    5783  json.WriteString("Max");
    5784  json.WriteNumber(stat.unusedRangeSizeMax);
    5785  json.EndObject();
    5786  }
    5787 
    5788  json.EndObject();
    5789 }
    5790 
    5791 #endif // #if VMA_STATS_STRING_ENABLED
    5792 
    5793 struct VmaSuballocationItemSizeLess
    5794 {
    5795  bool operator()(
    5796  const VmaSuballocationList::iterator lhs,
    5797  const VmaSuballocationList::iterator rhs) const
    5798  {
    5799  return lhs->size < rhs->size;
    5800  }
    5801  bool operator()(
    5802  const VmaSuballocationList::iterator lhs,
    5803  VkDeviceSize rhsSize) const
    5804  {
    5805  return lhs->size < rhsSize;
    5806  }
    5807 };
    5808 
    5810 // class VmaBlockMetadata
    5811 
    5812 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
    5813  m_Size(0),
    5814  m_FreeCount(0),
    5815  m_SumFreeSize(0),
    5816  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
    5817  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
    5818 {
    5819 }
    5820 
    5821 VmaBlockMetadata::~VmaBlockMetadata()
    5822 {
    5823 }
    5824 
    5825 void VmaBlockMetadata::Init(VkDeviceSize size)
    5826 {
    5827  m_Size = size;
    5828  m_FreeCount = 1;
    5829  m_SumFreeSize = size;
    5830 
    5831  VmaSuballocation suballoc = {};
    5832  suballoc.offset = 0;
    5833  suballoc.size = size;
    5834  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    5835  suballoc.hAllocation = VK_NULL_HANDLE;
    5836 
    5837  m_Suballocations.push_back(suballoc);
    5838  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
    5839  --suballocItem;
    5840  m_FreeSuballocationsBySize.push_back(suballocItem);
    5841 }
    5842 
    5843 bool VmaBlockMetadata::Validate() const
    5844 {
    5845  if(m_Suballocations.empty())
    5846  {
    5847  return false;
    5848  }
    5849 
    5850  // Expected offset of new suballocation as calculates from previous ones.
    5851  VkDeviceSize calculatedOffset = 0;
    5852  // Expected number of free suballocations as calculated from traversing their list.
    5853  uint32_t calculatedFreeCount = 0;
    5854  // Expected sum size of free suballocations as calculated from traversing their list.
    5855  VkDeviceSize calculatedSumFreeSize = 0;
    5856  // Expected number of free suballocations that should be registered in
    5857  // m_FreeSuballocationsBySize calculated from traversing their list.
    5858  size_t freeSuballocationsToRegister = 0;
    5859  // True if previous visited suballocation was free.
    5860  bool prevFree = false;
    5861 
    5862  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5863  suballocItem != m_Suballocations.cend();
    5864  ++suballocItem)
    5865  {
    5866  const VmaSuballocation& subAlloc = *suballocItem;
    5867 
    5868  // Actual offset of this suballocation doesn't match expected one.
    5869  if(subAlloc.offset != calculatedOffset)
    5870  {
    5871  return false;
    5872  }
    5873 
    5874  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
    5875  // Two adjacent free suballocations are invalid. They should be merged.
    5876  if(prevFree && currFree)
    5877  {
    5878  return false;
    5879  }
    5880 
    5881  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
    5882  {
    5883  return false;
    5884  }
    5885 
    5886  if(currFree)
    5887  {
    5888  calculatedSumFreeSize += subAlloc.size;
    5889  ++calculatedFreeCount;
    5890  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    5891  {
    5892  ++freeSuballocationsToRegister;
    5893  }
    5894 
    5895  // Margin required between allocations - every free space must be at least that large.
    5896  if(subAlloc.size < VMA_DEBUG_MARGIN)
    5897  {
    5898  return false;
    5899  }
    5900  }
    5901  else
    5902  {
    5903  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
    5904  {
    5905  return false;
    5906  }
    5907  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
    5908  {
    5909  return false;
    5910  }
    5911 
    5912  // Margin required between allocations - previous allocation must be free.
    5913  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
    5914  {
    5915  return false;
    5916  }
    5917  }
    5918 
    5919  calculatedOffset += subAlloc.size;
    5920  prevFree = currFree;
    5921  }
    5922 
    5923  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
    5924  // match expected one.
    5925  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
    5926  {
    5927  return false;
    5928  }
    5929 
    5930  VkDeviceSize lastSize = 0;
    5931  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
    5932  {
    5933  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
    5934 
    5935  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
    5936  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
    5937  {
    5938  return false;
    5939  }
    5940  // They must be sorted by size ascending.
    5941  if(suballocItem->size < lastSize)
    5942  {
    5943  return false;
    5944  }
    5945 
    5946  lastSize = suballocItem->size;
    5947  }
    5948 
    5949  // Check if totals match calculacted values.
    5950  if(!ValidateFreeSuballocationList() ||
    5951  (calculatedOffset != m_Size) ||
    5952  (calculatedSumFreeSize != m_SumFreeSize) ||
    5953  (calculatedFreeCount != m_FreeCount))
    5954  {
    5955  return false;
    5956  }
    5957 
    5958  return true;
    5959 }
    5960 
    5961 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
    5962 {
    5963  if(!m_FreeSuballocationsBySize.empty())
    5964  {
    5965  return m_FreeSuballocationsBySize.back()->size;
    5966  }
    5967  else
    5968  {
    5969  return 0;
    5970  }
    5971 }
    5972 
    5973 bool VmaBlockMetadata::IsEmpty() const
    5974 {
    5975  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
    5976 }
    5977 
    5978 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
    5979 {
    5980  outInfo.blockCount = 1;
    5981 
    5982  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    5983  outInfo.allocationCount = rangeCount - m_FreeCount;
    5984  outInfo.unusedRangeCount = m_FreeCount;
    5985 
    5986  outInfo.unusedBytes = m_SumFreeSize;
    5987  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
    5988 
    5989  outInfo.allocationSizeMin = UINT64_MAX;
    5990  outInfo.allocationSizeMax = 0;
    5991  outInfo.unusedRangeSizeMin = UINT64_MAX;
    5992  outInfo.unusedRangeSizeMax = 0;
    5993 
    5994  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5995  suballocItem != m_Suballocations.cend();
    5996  ++suballocItem)
    5997  {
    5998  const VmaSuballocation& suballoc = *suballocItem;
    5999  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
    6000  {
    6001  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
    6002  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
    6003  }
    6004  else
    6005  {
    6006  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
    6007  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
    6008  }
    6009  }
    6010 }
    6011 
    6012 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
    6013 {
    6014  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    6015 
    6016  inoutStats.size += m_Size;
    6017  inoutStats.unusedSize += m_SumFreeSize;
    6018  inoutStats.allocationCount += rangeCount - m_FreeCount;
    6019  inoutStats.unusedRangeCount += m_FreeCount;
    6020  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
    6021 }
    6022 
    6023 #if VMA_STATS_STRING_ENABLED
    6024 
    6025 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
    6026 {
    6027  json.BeginObject();
    6028 
    6029  json.WriteString("TotalBytes");
    6030  json.WriteNumber(m_Size);
    6031 
    6032  json.WriteString("UnusedBytes");
    6033  json.WriteNumber(m_SumFreeSize);
    6034 
    6035  json.WriteString("Allocations");
    6036  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
    6037 
    6038  json.WriteString("UnusedRanges");
    6039  json.WriteNumber(m_FreeCount);
    6040 
    6041  json.WriteString("Suballocations");
    6042  json.BeginArray();
    6043  size_t i = 0;
    6044  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    6045  suballocItem != m_Suballocations.cend();
    6046  ++suballocItem, ++i)
    6047  {
    6048  json.BeginObject(true);
    6049 
    6050  json.WriteString("Offset");
    6051  json.WriteNumber(suballocItem->offset);
    6052 
    6053  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6054  {
    6055  json.WriteString("Type");
    6056  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
    6057 
    6058  json.WriteString("Size");
    6059  json.WriteNumber(suballocItem->size);
    6060  }
    6061  else
    6062  {
    6063  suballocItem->hAllocation->PrintParameters(json);
    6064  }
    6065 
    6066  json.EndObject();
    6067  }
    6068  json.EndArray();
    6069 
    6070  json.EndObject();
    6071 }
    6072 
    6073 #endif // #if VMA_STATS_STRING_ENABLED
    6074 
    6075 /*
    6076 How many suitable free suballocations to analyze before choosing best one.
    6077 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
    6078  be chosen.
    6079 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
    6080  suballocations will be analized and best one will be chosen.
    6081 - Any other value is also acceptable.
    6082 */
    6083 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
    6084 
    6085 bool VmaBlockMetadata::CreateAllocationRequest(
    6086  uint32_t currentFrameIndex,
    6087  uint32_t frameInUseCount,
    6088  VkDeviceSize bufferImageGranularity,
    6089  VkDeviceSize allocSize,
    6090  VkDeviceSize allocAlignment,
    6091  VmaSuballocationType allocType,
    6092  bool canMakeOtherLost,
    6093  VmaAllocationRequest* pAllocationRequest)
    6094 {
    6095  VMA_ASSERT(allocSize > 0);
    6096  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6097  VMA_ASSERT(pAllocationRequest != VMA_NULL);
    6098  VMA_HEAVY_ASSERT(Validate());
    6099 
    6100  // There is not enough total free space in this block to fullfill the request: Early return.
    6101  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
    6102  {
    6103  return false;
    6104  }
    6105 
    6106  // New algorithm, efficiently searching freeSuballocationsBySize.
    6107  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
    6108  if(freeSuballocCount > 0)
    6109  {
    6110  if(VMA_BEST_FIT)
    6111  {
    6112  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
    6113  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6114  m_FreeSuballocationsBySize.data(),
    6115  m_FreeSuballocationsBySize.data() + freeSuballocCount,
    6116  allocSize + 2 * VMA_DEBUG_MARGIN,
    6117  VmaSuballocationItemSizeLess());
    6118  size_t index = it - m_FreeSuballocationsBySize.data();
    6119  for(; index < freeSuballocCount; ++index)
    6120  {
    6121  if(CheckAllocation(
    6122  currentFrameIndex,
    6123  frameInUseCount,
    6124  bufferImageGranularity,
    6125  allocSize,
    6126  allocAlignment,
    6127  allocType,
    6128  m_FreeSuballocationsBySize[index],
    6129  false, // canMakeOtherLost
    6130  &pAllocationRequest->offset,
    6131  &pAllocationRequest->itemsToMakeLostCount,
    6132  &pAllocationRequest->sumFreeSize,
    6133  &pAllocationRequest->sumItemSize))
    6134  {
    6135  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6136  return true;
    6137  }
    6138  }
    6139  }
    6140  else
    6141  {
    6142  // Search staring from biggest suballocations.
    6143  for(size_t index = freeSuballocCount; index--; )
    6144  {
    6145  if(CheckAllocation(
    6146  currentFrameIndex,
    6147  frameInUseCount,
    6148  bufferImageGranularity,
    6149  allocSize,
    6150  allocAlignment,
    6151  allocType,
    6152  m_FreeSuballocationsBySize[index],
    6153  false, // canMakeOtherLost
    6154  &pAllocationRequest->offset,
    6155  &pAllocationRequest->itemsToMakeLostCount,
    6156  &pAllocationRequest->sumFreeSize,
    6157  &pAllocationRequest->sumItemSize))
    6158  {
    6159  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6160  return true;
    6161  }
    6162  }
    6163  }
    6164  }
    6165 
    6166  if(canMakeOtherLost)
    6167  {
    6168  // Brute-force algorithm. TODO: Come up with something better.
    6169 
    6170  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
    6171  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
    6172 
    6173  VmaAllocationRequest tmpAllocRequest = {};
    6174  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
    6175  suballocIt != m_Suballocations.end();
    6176  ++suballocIt)
    6177  {
    6178  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
    6179  suballocIt->hAllocation->CanBecomeLost())
    6180  {
    6181  if(CheckAllocation(
    6182  currentFrameIndex,
    6183  frameInUseCount,
    6184  bufferImageGranularity,
    6185  allocSize,
    6186  allocAlignment,
    6187  allocType,
    6188  suballocIt,
    6189  canMakeOtherLost,
    6190  &tmpAllocRequest.offset,
    6191  &tmpAllocRequest.itemsToMakeLostCount,
    6192  &tmpAllocRequest.sumFreeSize,
    6193  &tmpAllocRequest.sumItemSize))
    6194  {
    6195  tmpAllocRequest.item = suballocIt;
    6196 
    6197  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
    6198  {
    6199  *pAllocationRequest = tmpAllocRequest;
    6200  }
    6201  }
    6202  }
    6203  }
    6204 
    6205  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
    6206  {
    6207  return true;
    6208  }
    6209  }
    6210 
    6211  return false;
    6212 }
    6213 
    6214 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
    6215  uint32_t currentFrameIndex,
    6216  uint32_t frameInUseCount,
    6217  VmaAllocationRequest* pAllocationRequest)
    6218 {
    6219  while(pAllocationRequest->itemsToMakeLostCount > 0)
    6220  {
    6221  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
    6222  {
    6223  ++pAllocationRequest->item;
    6224  }
    6225  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6226  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
    6227  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
    6228  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6229  {
    6230  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
    6231  --pAllocationRequest->itemsToMakeLostCount;
    6232  }
    6233  else
    6234  {
    6235  return false;
    6236  }
    6237  }
    6238 
    6239  VMA_HEAVY_ASSERT(Validate());
    6240  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6241  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6242 
    6243  return true;
    6244 }
    6245 
    6246 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    6247 {
    6248  uint32_t lostAllocationCount = 0;
    6249  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6250  it != m_Suballocations.end();
    6251  ++it)
    6252  {
    6253  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
    6254  it->hAllocation->CanBecomeLost() &&
    6255  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6256  {
    6257  it = FreeSuballocation(it);
    6258  ++lostAllocationCount;
    6259  }
    6260  }
    6261  return lostAllocationCount;
    6262 }
    6263 
    6264 VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData)
    6265 {
    6266  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6267  it != m_Suballocations.end();
    6268  ++it)
    6269  {
    6270  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6271  {
    6272  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
    6273  {
    6274  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
    6275  return VK_ERROR_VALIDATION_FAILED_EXT;
    6276  }
    6277  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
    6278  {
    6279  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
    6280  return VK_ERROR_VALIDATION_FAILED_EXT;
    6281  }
    6282  }
    6283  }
    6284 
    6285  return VK_SUCCESS;
    6286 }
    6287 
    6288 void VmaBlockMetadata::Alloc(
    6289  const VmaAllocationRequest& request,
    6290  VmaSuballocationType type,
    6291  VkDeviceSize allocSize,
    6292  VmaAllocation hAllocation)
    6293 {
    6294  VMA_ASSERT(request.item != m_Suballocations.end());
    6295  VmaSuballocation& suballoc = *request.item;
    6296  // Given suballocation is a free block.
    6297  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6298  // Given offset is inside this suballocation.
    6299  VMA_ASSERT(request.offset >= suballoc.offset);
    6300  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
    6301  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
    6302  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
    6303 
    6304  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
    6305  // it to become used.
    6306  UnregisterFreeSuballocation(request.item);
    6307 
    6308  suballoc.offset = request.offset;
    6309  suballoc.size = allocSize;
    6310  suballoc.type = type;
    6311  suballoc.hAllocation = hAllocation;
    6312 
    6313  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
    6314  if(paddingEnd)
    6315  {
    6316  VmaSuballocation paddingSuballoc = {};
    6317  paddingSuballoc.offset = request.offset + allocSize;
    6318  paddingSuballoc.size = paddingEnd;
    6319  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6320  VmaSuballocationList::iterator next = request.item;
    6321  ++next;
    6322  const VmaSuballocationList::iterator paddingEndItem =
    6323  m_Suballocations.insert(next, paddingSuballoc);
    6324  RegisterFreeSuballocation(paddingEndItem);
    6325  }
    6326 
    6327  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
    6328  if(paddingBegin)
    6329  {
    6330  VmaSuballocation paddingSuballoc = {};
    6331  paddingSuballoc.offset = request.offset - paddingBegin;
    6332  paddingSuballoc.size = paddingBegin;
    6333  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6334  const VmaSuballocationList::iterator paddingBeginItem =
    6335  m_Suballocations.insert(request.item, paddingSuballoc);
    6336  RegisterFreeSuballocation(paddingBeginItem);
    6337  }
    6338 
    6339  // Update totals.
    6340  m_FreeCount = m_FreeCount - 1;
    6341  if(paddingBegin > 0)
    6342  {
    6343  ++m_FreeCount;
    6344  }
    6345  if(paddingEnd > 0)
    6346  {
    6347  ++m_FreeCount;
    6348  }
    6349  m_SumFreeSize -= allocSize;
    6350 }
    6351 
    6352 void VmaBlockMetadata::Free(const VmaAllocation allocation)
    6353 {
    6354  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6355  suballocItem != m_Suballocations.end();
    6356  ++suballocItem)
    6357  {
    6358  VmaSuballocation& suballoc = *suballocItem;
    6359  if(suballoc.hAllocation == allocation)
    6360  {
    6361  FreeSuballocation(suballocItem);
    6362  VMA_HEAVY_ASSERT(Validate());
    6363  return;
    6364  }
    6365  }
    6366  VMA_ASSERT(0 && "Not found!");
    6367 }
    6368 
    6369 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
    6370 {
    6371  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6372  suballocItem != m_Suballocations.end();
    6373  ++suballocItem)
    6374  {
    6375  VmaSuballocation& suballoc = *suballocItem;
    6376  if(suballoc.offset == offset)
    6377  {
    6378  FreeSuballocation(suballocItem);
    6379  return;
    6380  }
    6381  }
    6382  VMA_ASSERT(0 && "Not found!");
    6383 }
    6384 
    6385 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
    6386 {
    6387  VkDeviceSize lastSize = 0;
    6388  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
    6389  {
    6390  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
    6391 
    6392  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6393  {
    6394  VMA_ASSERT(0);
    6395  return false;
    6396  }
    6397  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6398  {
    6399  VMA_ASSERT(0);
    6400  return false;
    6401  }
    6402  if(it->size < lastSize)
    6403  {
    6404  VMA_ASSERT(0);
    6405  return false;
    6406  }
    6407 
    6408  lastSize = it->size;
    6409  }
    6410  return true;
    6411 }
    6412 
    6413 bool VmaBlockMetadata::CheckAllocation(
    6414  uint32_t currentFrameIndex,
    6415  uint32_t frameInUseCount,
    6416  VkDeviceSize bufferImageGranularity,
    6417  VkDeviceSize allocSize,
    6418  VkDeviceSize allocAlignment,
    6419  VmaSuballocationType allocType,
    6420  VmaSuballocationList::const_iterator suballocItem,
    6421  bool canMakeOtherLost,
    6422  VkDeviceSize* pOffset,
    6423  size_t* itemsToMakeLostCount,
    6424  VkDeviceSize* pSumFreeSize,
    6425  VkDeviceSize* pSumItemSize) const
    6426 {
    6427  VMA_ASSERT(allocSize > 0);
    6428  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6429  VMA_ASSERT(suballocItem != m_Suballocations.cend());
    6430  VMA_ASSERT(pOffset != VMA_NULL);
    6431 
    6432  *itemsToMakeLostCount = 0;
    6433  *pSumFreeSize = 0;
    6434  *pSumItemSize = 0;
    6435 
    6436  if(canMakeOtherLost)
    6437  {
    6438  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6439  {
    6440  *pSumFreeSize = suballocItem->size;
    6441  }
    6442  else
    6443  {
    6444  if(suballocItem->hAllocation->CanBecomeLost() &&
    6445  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6446  {
    6447  ++*itemsToMakeLostCount;
    6448  *pSumItemSize = suballocItem->size;
    6449  }
    6450  else
    6451  {
    6452  return false;
    6453  }
    6454  }
    6455 
    6456  // Remaining size is too small for this request: Early return.
    6457  if(m_Size - suballocItem->offset < allocSize)
    6458  {
    6459  return false;
    6460  }
    6461 
    6462  // Start from offset equal to beginning of this suballocation.
    6463  *pOffset = suballocItem->offset;
    6464 
    6465  // Apply VMA_DEBUG_MARGIN at the beginning.
    6466  if(VMA_DEBUG_MARGIN > 0)
    6467  {
    6468  *pOffset += VMA_DEBUG_MARGIN;
    6469  }
    6470 
    6471  // Apply alignment.
    6472  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6473 
    6474  // Check previous suballocations for BufferImageGranularity conflicts.
    6475  // Make bigger alignment if necessary.
    6476  if(bufferImageGranularity > 1)
    6477  {
    6478  bool bufferImageGranularityConflict = false;
    6479  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6480  while(prevSuballocItem != m_Suballocations.cbegin())
    6481  {
    6482  --prevSuballocItem;
    6483  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6484  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6485  {
    6486  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6487  {
    6488  bufferImageGranularityConflict = true;
    6489  break;
    6490  }
    6491  }
    6492  else
    6493  // Already on previous page.
    6494  break;
    6495  }
    6496  if(bufferImageGranularityConflict)
    6497  {
    6498  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6499  }
    6500  }
    6501 
    6502  // Now that we have final *pOffset, check if we are past suballocItem.
    6503  // If yes, return false - this function should be called for another suballocItem as starting point.
    6504  if(*pOffset >= suballocItem->offset + suballocItem->size)
    6505  {
    6506  return false;
    6507  }
    6508 
    6509  // Calculate padding at the beginning based on current offset.
    6510  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
    6511 
    6512  // Calculate required margin at the end.
    6513  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6514 
    6515  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
    6516  // Another early return check.
    6517  if(suballocItem->offset + totalSize > m_Size)
    6518  {
    6519  return false;
    6520  }
    6521 
    6522  // Advance lastSuballocItem until desired size is reached.
    6523  // Update itemsToMakeLostCount.
    6524  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
    6525  if(totalSize > suballocItem->size)
    6526  {
    6527  VkDeviceSize remainingSize = totalSize - suballocItem->size;
    6528  while(remainingSize > 0)
    6529  {
    6530  ++lastSuballocItem;
    6531  if(lastSuballocItem == m_Suballocations.cend())
    6532  {
    6533  return false;
    6534  }
    6535  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6536  {
    6537  *pSumFreeSize += lastSuballocItem->size;
    6538  }
    6539  else
    6540  {
    6541  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
    6542  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
    6543  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6544  {
    6545  ++*itemsToMakeLostCount;
    6546  *pSumItemSize += lastSuballocItem->size;
    6547  }
    6548  else
    6549  {
    6550  return false;
    6551  }
    6552  }
    6553  remainingSize = (lastSuballocItem->size < remainingSize) ?
    6554  remainingSize - lastSuballocItem->size : 0;
    6555  }
    6556  }
    6557 
    6558  // Check next suballocations for BufferImageGranularity conflicts.
    6559  // If conflict exists, we must mark more allocations lost or fail.
    6560  if(bufferImageGranularity > 1)
    6561  {
    6562  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
    6563  ++nextSuballocItem;
    6564  while(nextSuballocItem != m_Suballocations.cend())
    6565  {
    6566  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6567  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6568  {
    6569  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6570  {
    6571  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
    6572  if(nextSuballoc.hAllocation->CanBecomeLost() &&
    6573  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6574  {
    6575  ++*itemsToMakeLostCount;
    6576  }
    6577  else
    6578  {
    6579  return false;
    6580  }
    6581  }
    6582  }
    6583  else
    6584  {
    6585  // Already on next page.
    6586  break;
    6587  }
    6588  ++nextSuballocItem;
    6589  }
    6590  }
    6591  }
    6592  else
    6593  {
    6594  const VmaSuballocation& suballoc = *suballocItem;
    6595  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6596 
    6597  *pSumFreeSize = suballoc.size;
    6598 
    6599  // Size of this suballocation is too small for this request: Early return.
    6600  if(suballoc.size < allocSize)
    6601  {
    6602  return false;
    6603  }
    6604 
    6605  // Start from offset equal to beginning of this suballocation.
    6606  *pOffset = suballoc.offset;
    6607 
    6608  // Apply VMA_DEBUG_MARGIN at the beginning.
    6609  if(VMA_DEBUG_MARGIN > 0)
    6610  {
    6611  *pOffset += VMA_DEBUG_MARGIN;
    6612  }
    6613 
    6614  // Apply alignment.
    6615  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6616 
    6617  // Check previous suballocations for BufferImageGranularity conflicts.
    6618  // Make bigger alignment if necessary.
    6619  if(bufferImageGranularity > 1)
    6620  {
    6621  bool bufferImageGranularityConflict = false;
    6622  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6623  while(prevSuballocItem != m_Suballocations.cbegin())
    6624  {
    6625  --prevSuballocItem;
    6626  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6627  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6628  {
    6629  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6630  {
    6631  bufferImageGranularityConflict = true;
    6632  break;
    6633  }
    6634  }
    6635  else
    6636  // Already on previous page.
    6637  break;
    6638  }
    6639  if(bufferImageGranularityConflict)
    6640  {
    6641  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6642  }
    6643  }
    6644 
    6645  // Calculate padding at the beginning based on current offset.
    6646  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
    6647 
    6648  // Calculate required margin at the end.
    6649  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6650 
    6651  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
    6652  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
    6653  {
    6654  return false;
    6655  }
    6656 
    6657  // Check next suballocations for BufferImageGranularity conflicts.
    6658  // If conflict exists, allocation cannot be made here.
    6659  if(bufferImageGranularity > 1)
    6660  {
    6661  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
    6662  ++nextSuballocItem;
    6663  while(nextSuballocItem != m_Suballocations.cend())
    6664  {
    6665  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6666  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6667  {
    6668  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6669  {
    6670  return false;
    6671  }
    6672  }
    6673  else
    6674  {
    6675  // Already on next page.
    6676  break;
    6677  }
    6678  ++nextSuballocItem;
    6679  }
    6680  }
    6681  }
    6682 
    6683  // All tests passed: Success. pOffset is already filled.
    6684  return true;
    6685 }
    6686 
    6687 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
    6688 {
    6689  VMA_ASSERT(item != m_Suballocations.end());
    6690  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6691 
    6692  VmaSuballocationList::iterator nextItem = item;
    6693  ++nextItem;
    6694  VMA_ASSERT(nextItem != m_Suballocations.end());
    6695  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
    6696 
    6697  item->size += nextItem->size;
    6698  --m_FreeCount;
    6699  m_Suballocations.erase(nextItem);
    6700 }
    6701 
    6702 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
    6703 {
    6704  // Change this suballocation to be marked as free.
    6705  VmaSuballocation& suballoc = *suballocItem;
    6706  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6707  suballoc.hAllocation = VK_NULL_HANDLE;
    6708 
    6709  // Update totals.
    6710  ++m_FreeCount;
    6711  m_SumFreeSize += suballoc.size;
    6712 
    6713  // Merge with previous and/or next suballocation if it's also free.
    6714  bool mergeWithNext = false;
    6715  bool mergeWithPrev = false;
    6716 
    6717  VmaSuballocationList::iterator nextItem = suballocItem;
    6718  ++nextItem;
    6719  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
    6720  {
    6721  mergeWithNext = true;
    6722  }
    6723 
    6724  VmaSuballocationList::iterator prevItem = suballocItem;
    6725  if(suballocItem != m_Suballocations.begin())
    6726  {
    6727  --prevItem;
    6728  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6729  {
    6730  mergeWithPrev = true;
    6731  }
    6732  }
    6733 
    6734  if(mergeWithNext)
    6735  {
    6736  UnregisterFreeSuballocation(nextItem);
    6737  MergeFreeWithNext(suballocItem);
    6738  }
    6739 
    6740  if(mergeWithPrev)
    6741  {
    6742  UnregisterFreeSuballocation(prevItem);
    6743  MergeFreeWithNext(prevItem);
    6744  RegisterFreeSuballocation(prevItem);
    6745  return prevItem;
    6746  }
    6747  else
    6748  {
    6749  RegisterFreeSuballocation(suballocItem);
    6750  return suballocItem;
    6751  }
    6752 }
    6753 
    6754 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
    6755 {
    6756  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6757  VMA_ASSERT(item->size > 0);
    6758 
    6759  // You may want to enable this validation at the beginning or at the end of
    6760  // this function, depending on what do you want to check.
    6761  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6762 
    6763  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6764  {
    6765  if(m_FreeSuballocationsBySize.empty())
    6766  {
    6767  m_FreeSuballocationsBySize.push_back(item);
    6768  }
    6769  else
    6770  {
    6771  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
    6772  }
    6773  }
    6774 
    6775  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6776 }
    6777 
    6778 
    6779 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
    6780 {
    6781  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6782  VMA_ASSERT(item->size > 0);
    6783 
    6784  // You may want to enable this validation at the beginning or at the end of
    6785  // this function, depending on what do you want to check.
    6786  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6787 
    6788  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6789  {
    6790  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6791  m_FreeSuballocationsBySize.data(),
    6792  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
    6793  item,
    6794  VmaSuballocationItemSizeLess());
    6795  for(size_t index = it - m_FreeSuballocationsBySize.data();
    6796  index < m_FreeSuballocationsBySize.size();
    6797  ++index)
    6798  {
    6799  if(m_FreeSuballocationsBySize[index] == item)
    6800  {
    6801  VmaVectorRemove(m_FreeSuballocationsBySize, index);
    6802  return;
    6803  }
    6804  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
    6805  }
    6806  VMA_ASSERT(0 && "Not found.");
    6807  }
    6808 
    6809  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6810 }
    6811 
    6813 // class VmaDeviceMemoryBlock
    6814 
    6815 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
    6816  m_Metadata(hAllocator),
    6817  m_MemoryTypeIndex(UINT32_MAX),
    6818  m_Id(0),
    6819  m_hMemory(VK_NULL_HANDLE),
    6820  m_MapCount(0),
    6821  m_pMappedData(VMA_NULL)
    6822 {
    6823 }
    6824 
    6825 void VmaDeviceMemoryBlock::Init(
    6826  uint32_t newMemoryTypeIndex,
    6827  VkDeviceMemory newMemory,
    6828  VkDeviceSize newSize,
    6829  uint32_t id)
    6830 {
    6831  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    6832 
    6833  m_MemoryTypeIndex = newMemoryTypeIndex;
    6834  m_Id = id;
    6835  m_hMemory = newMemory;
    6836 
    6837  m_Metadata.Init(newSize);
    6838 }
    6839 
    6840 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
    6841 {
    6842  // This is the most important assert in the entire library.
    6843  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
    6844  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
    6845 
    6846  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
    6847  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
    6848  m_hMemory = VK_NULL_HANDLE;
    6849 }
    6850 
    6851 bool VmaDeviceMemoryBlock::Validate() const
    6852 {
    6853  if((m_hMemory == VK_NULL_HANDLE) ||
    6854  (m_Metadata.GetSize() == 0))
    6855  {
    6856  return false;
    6857  }
    6858 
    6859  return m_Metadata.Validate();
    6860 }
    6861 
    6862 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
    6863 {
    6864  void* pData = nullptr;
    6865  VkResult res = Map(hAllocator, 1, &pData);
    6866  if(res != VK_SUCCESS)
    6867  {
    6868  return res;
    6869  }
    6870 
    6871  res = m_Metadata.CheckCorruption(pData);
    6872 
    6873  Unmap(hAllocator, 1);
    6874 
    6875  return res;
    6876 }
    6877 
    6878 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
    6879 {
    6880  if(count == 0)
    6881  {
    6882  return VK_SUCCESS;
    6883  }
    6884 
    6885  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6886  if(m_MapCount != 0)
    6887  {
    6888  m_MapCount += count;
    6889  VMA_ASSERT(m_pMappedData != VMA_NULL);
    6890  if(ppData != VMA_NULL)
    6891  {
    6892  *ppData = m_pMappedData;
    6893  }
    6894  return VK_SUCCESS;
    6895  }
    6896  else
    6897  {
    6898  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    6899  hAllocator->m_hDevice,
    6900  m_hMemory,
    6901  0, // offset
    6902  VK_WHOLE_SIZE,
    6903  0, // flags
    6904  &m_pMappedData);
    6905  if(result == VK_SUCCESS)
    6906  {
    6907  if(ppData != VMA_NULL)
    6908  {
    6909  *ppData = m_pMappedData;
    6910  }
    6911  m_MapCount = count;
    6912  }
    6913  return result;
    6914  }
    6915 }
    6916 
    6917 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
    6918 {
    6919  if(count == 0)
    6920  {
    6921  return;
    6922  }
    6923 
    6924  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6925  if(m_MapCount >= count)
    6926  {
    6927  m_MapCount -= count;
    6928  if(m_MapCount == 0)
    6929  {
    6930  m_pMappedData = VMA_NULL;
    6931  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
    6932  }
    6933  }
    6934  else
    6935  {
    6936  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
    6937  }
    6938 }
    6939 
    6940 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6941 {
    6942  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    6943  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    6944 
    6945  void* pData;
    6946  VkResult res = Map(hAllocator, 1, &pData);
    6947  if(res != VK_SUCCESS)
    6948  {
    6949  return res;
    6950  }
    6951 
    6952  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
    6953  VmaWriteMagicValue(pData, allocOffset + allocSize);
    6954 
    6955  Unmap(hAllocator, 1);
    6956 
    6957  return VK_SUCCESS;
    6958 }
    6959 
    6960 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6961 {
    6962  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    6963  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    6964 
    6965  void* pData;
    6966  VkResult res = Map(hAllocator, 1, &pData);
    6967  if(res != VK_SUCCESS)
    6968  {
    6969  return res;
    6970  }
    6971 
    6972  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
    6973  {
    6974  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
    6975  }
    6976  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
    6977  {
    6978  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
    6979  }
    6980 
    6981  Unmap(hAllocator, 1);
    6982 
    6983  return VK_SUCCESS;
    6984 }
    6985 
    6986 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
    6987  const VmaAllocator hAllocator,
    6988  const VmaAllocation hAllocation,
    6989  VkBuffer hBuffer)
    6990 {
    6991  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    6992  hAllocation->GetBlock() == this);
    6993  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    6994  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6995  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
    6996  hAllocator->m_hDevice,
    6997  hBuffer,
    6998  m_hMemory,
    6999  hAllocation->GetOffset());
    7000 }
    7001 
    7002 VkResult VmaDeviceMemoryBlock::BindImageMemory(
    7003  const VmaAllocator hAllocator,
    7004  const VmaAllocation hAllocation,
    7005  VkImage hImage)
    7006 {
    7007  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    7008  hAllocation->GetBlock() == this);
    7009  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    7010  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    7011  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
    7012  hAllocator->m_hDevice,
    7013  hImage,
    7014  m_hMemory,
    7015  hAllocation->GetOffset());
    7016 }
    7017 
    7018 static void InitStatInfo(VmaStatInfo& outInfo)
    7019 {
    7020  memset(&outInfo, 0, sizeof(outInfo));
    7021  outInfo.allocationSizeMin = UINT64_MAX;
    7022  outInfo.unusedRangeSizeMin = UINT64_MAX;
    7023 }
    7024 
    7025 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
    7026 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
    7027 {
    7028  inoutInfo.blockCount += srcInfo.blockCount;
    7029  inoutInfo.allocationCount += srcInfo.allocationCount;
    7030  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
    7031  inoutInfo.usedBytes += srcInfo.usedBytes;
    7032  inoutInfo.unusedBytes += srcInfo.unusedBytes;
    7033  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
    7034  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
    7035  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
    7036  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
    7037 }
    7038 
    7039 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
    7040 {
    7041  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
    7042  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
    7043  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
    7044  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
    7045 }
    7046 
    7047 VmaPool_T::VmaPool_T(
    7048  VmaAllocator hAllocator,
    7049  const VmaPoolCreateInfo& createInfo) :
    7050  m_BlockVector(
    7051  hAllocator,
    7052  createInfo.memoryTypeIndex,
    7053  createInfo.blockSize,
    7054  createInfo.minBlockCount,
    7055  createInfo.maxBlockCount,
    7056  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
    7057  createInfo.frameInUseCount,
    7058  true), // isCustomPool
    7059  m_Id(0)
    7060 {
    7061 }
    7062 
    7063 VmaPool_T::~VmaPool_T()
    7064 {
    7065 }
    7066 
    7067 #if VMA_STATS_STRING_ENABLED
    7068 
    7069 #endif // #if VMA_STATS_STRING_ENABLED
    7070 
    7071 VmaBlockVector::VmaBlockVector(
    7072  VmaAllocator hAllocator,
    7073  uint32_t memoryTypeIndex,
    7074  VkDeviceSize preferredBlockSize,
    7075  size_t minBlockCount,
    7076  size_t maxBlockCount,
    7077  VkDeviceSize bufferImageGranularity,
    7078  uint32_t frameInUseCount,
    7079  bool isCustomPool) :
    7080  m_hAllocator(hAllocator),
    7081  m_MemoryTypeIndex(memoryTypeIndex),
    7082  m_PreferredBlockSize(preferredBlockSize),
    7083  m_MinBlockCount(minBlockCount),
    7084  m_MaxBlockCount(maxBlockCount),
    7085  m_BufferImageGranularity(bufferImageGranularity),
    7086  m_FrameInUseCount(frameInUseCount),
    7087  m_IsCustomPool(isCustomPool),
    7088  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
    7089  m_HasEmptyBlock(false),
    7090  m_pDefragmentator(VMA_NULL),
    7091  m_NextBlockId(0)
    7092 {
    7093 }
    7094 
    7095 VmaBlockVector::~VmaBlockVector()
    7096 {
    7097  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
    7098 
    7099  for(size_t i = m_Blocks.size(); i--; )
    7100  {
    7101  m_Blocks[i]->Destroy(m_hAllocator);
    7102  vma_delete(m_hAllocator, m_Blocks[i]);
    7103  }
    7104 }
    7105 
    7106 VkResult VmaBlockVector::CreateMinBlocks()
    7107 {
    7108  for(size_t i = 0; i < m_MinBlockCount; ++i)
    7109  {
    7110  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
    7111  if(res != VK_SUCCESS)
    7112  {
    7113  return res;
    7114  }
    7115  }
    7116  return VK_SUCCESS;
    7117 }
    7118 
    7119 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
    7120 {
    7121  pStats->size = 0;
    7122  pStats->unusedSize = 0;
    7123  pStats->allocationCount = 0;
    7124  pStats->unusedRangeCount = 0;
    7125  pStats->unusedRangeSizeMax = 0;
    7126 
    7127  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7128 
    7129  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7130  {
    7131  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7132  VMA_ASSERT(pBlock);
    7133  VMA_HEAVY_ASSERT(pBlock->Validate());
    7134  pBlock->m_Metadata.AddPoolStats(*pStats);
    7135  }
    7136 }
    7137 
    7138 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
    7139 {
    7140  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    7141  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
    7142  (VMA_DEBUG_MARGIN > 0) &&
    7143  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
    7144 }
    7145 
    7146 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
    7147 
    7148 VkResult VmaBlockVector::Allocate(
    7149  VmaPool hCurrentPool,
    7150  uint32_t currentFrameIndex,
    7151  VkDeviceSize size,
    7152  VkDeviceSize alignment,
    7153  const VmaAllocationCreateInfo& createInfo,
    7154  VmaSuballocationType suballocType,
    7155  VmaAllocation* pAllocation)
    7156 {
    7157  // Early reject: requested allocation size is larger that maximum block size for this block vector.
    7158  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
    7159  {
    7160  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7161  }
    7162 
    7163  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    7164  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
    7165 
    7166  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7167 
    7168  // 1. Search existing allocations. Try to allocate without making other allocations lost.
    7169  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7170  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7171  {
    7172  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7173  VMA_ASSERT(pCurrBlock);
    7174  VmaAllocationRequest currRequest = {};
    7175  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7176  currentFrameIndex,
    7177  m_FrameInUseCount,
    7178  m_BufferImageGranularity,
    7179  size,
    7180  alignment,
    7181  suballocType,
    7182  false, // canMakeOtherLost
    7183  &currRequest))
    7184  {
    7185  // Allocate from pCurrBlock.
    7186  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
    7187 
    7188  if(mapped)
    7189  {
    7190  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
    7191  if(res != VK_SUCCESS)
    7192  {
    7193  return res;
    7194  }
    7195  }
    7196 
    7197  // We no longer have an empty Allocation.
    7198  if(pCurrBlock->m_Metadata.IsEmpty())
    7199  {
    7200  m_HasEmptyBlock = false;
    7201  }
    7202 
    7203  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7204  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation);
    7205  (*pAllocation)->InitBlockAllocation(
    7206  hCurrentPool,
    7207  pCurrBlock,
    7208  currRequest.offset,
    7209  alignment,
    7210  size,
    7211  suballocType,
    7212  mapped,
    7213  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7214  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
    7215  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7216  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7217  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7218  {
    7219  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7220  }
    7221  if(IsCorruptionDetectionEnabled())
    7222  {
    7223  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
    7224  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7225  }
    7226  return VK_SUCCESS;
    7227  }
    7228  }
    7229 
    7230  const bool canCreateNewBlock =
    7231  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
    7232  (m_Blocks.size() < m_MaxBlockCount);
    7233 
    7234  // 2. Try to create new block.
    7235  if(canCreateNewBlock)
    7236  {
    7237  // Calculate optimal size for new block.
    7238  VkDeviceSize newBlockSize = m_PreferredBlockSize;
    7239  uint32_t newBlockSizeShift = 0;
    7240  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
    7241 
    7242  // Allocating blocks of other sizes is allowed only in default pools.
    7243  // In custom pools block size is fixed.
    7244  if(m_IsCustomPool == false)
    7245  {
    7246  // Allocate 1/8, 1/4, 1/2 as first blocks.
    7247  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
    7248  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
    7249  {
    7250  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7251  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
    7252  {
    7253  newBlockSize = smallerNewBlockSize;
    7254  ++newBlockSizeShift;
    7255  }
    7256  else
    7257  {
    7258  break;
    7259  }
    7260  }
    7261  }
    7262 
    7263  size_t newBlockIndex = 0;
    7264  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
    7265  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
    7266  if(m_IsCustomPool == false)
    7267  {
    7268  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
    7269  {
    7270  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7271  if(smallerNewBlockSize >= size)
    7272  {
    7273  newBlockSize = smallerNewBlockSize;
    7274  ++newBlockSizeShift;
    7275  res = CreateBlock(newBlockSize, &newBlockIndex);
    7276  }
    7277  else
    7278  {
    7279  break;
    7280  }
    7281  }
    7282  }
    7283 
    7284  if(res == VK_SUCCESS)
    7285  {
    7286  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
    7287  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size);
    7288 
    7289  if(mapped)
    7290  {
    7291  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
    7292  if(res != VK_SUCCESS)
    7293  {
    7294  return res;
    7295  }
    7296  }
    7297 
    7298  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
    7299  VmaAllocationRequest allocRequest;
    7300  if(pBlock->m_Metadata.CreateAllocationRequest(
    7301  currentFrameIndex,
    7302  m_FrameInUseCount,
    7303  m_BufferImageGranularity,
    7304  size,
    7305  alignment,
    7306  suballocType,
    7307  false, // canMakeOtherLost
    7308  &allocRequest))
    7309  {
    7310  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7311  pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation);
    7312  (*pAllocation)->InitBlockAllocation(
    7313  hCurrentPool,
    7314  pBlock,
    7315  allocRequest.offset,
    7316  alignment,
    7317  size,
    7318  suballocType,
    7319  mapped,
    7320  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7321  VMA_HEAVY_ASSERT(pBlock->Validate());
    7322  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
    7323  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7324  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7325  {
    7326  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7327  }
    7328  if(IsCorruptionDetectionEnabled())
    7329  {
    7330  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
    7331  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7332  }
    7333  return VK_SUCCESS;
    7334  }
    7335  else
    7336  {
    7337  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
    7338  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7339  }
    7340  }
    7341  }
    7342 
    7343  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
    7344 
    7345  // 3. Try to allocate from existing blocks with making other allocations lost.
    7346  if(canMakeOtherLost)
    7347  {
    7348  uint32_t tryIndex = 0;
    7349  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
    7350  {
    7351  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
    7352  VmaAllocationRequest bestRequest = {};
    7353  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
    7354 
    7355  // 1. Search existing allocations.
    7356  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7357  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7358  {
    7359  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7360  VMA_ASSERT(pCurrBlock);
    7361  VmaAllocationRequest currRequest = {};
    7362  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7363  currentFrameIndex,
    7364  m_FrameInUseCount,
    7365  m_BufferImageGranularity,
    7366  size,
    7367  alignment,
    7368  suballocType,
    7369  canMakeOtherLost,
    7370  &currRequest))
    7371  {
    7372  const VkDeviceSize currRequestCost = currRequest.CalcCost();
    7373  if(pBestRequestBlock == VMA_NULL ||
    7374  currRequestCost < bestRequestCost)
    7375  {
    7376  pBestRequestBlock = pCurrBlock;
    7377  bestRequest = currRequest;
    7378  bestRequestCost = currRequestCost;
    7379 
    7380  if(bestRequestCost == 0)
    7381  {
    7382  break;
    7383  }
    7384  }
    7385  }
    7386  }
    7387 
    7388  if(pBestRequestBlock != VMA_NULL)
    7389  {
    7390  if(mapped)
    7391  {
    7392  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
    7393  if(res != VK_SUCCESS)
    7394  {
    7395  return res;
    7396  }
    7397  }
    7398 
    7399  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
    7400  currentFrameIndex,
    7401  m_FrameInUseCount,
    7402  &bestRequest))
    7403  {
    7404  // We no longer have an empty Allocation.
    7405  if(pBestRequestBlock->m_Metadata.IsEmpty())
    7406  {
    7407  m_HasEmptyBlock = false;
    7408  }
    7409  // Allocate from this pBlock.
    7410  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7411  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation);
    7412  (*pAllocation)->InitBlockAllocation(
    7413  hCurrentPool,
    7414  pBestRequestBlock,
    7415  bestRequest.offset,
    7416  alignment,
    7417  size,
    7418  suballocType,
    7419  mapped,
    7420  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7421  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
    7422  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7423  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7424  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7425  {
    7426  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7427  }
    7428  if(IsCorruptionDetectionEnabled())
    7429  {
    7430  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
    7431  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7432  }
    7433  return VK_SUCCESS;
    7434  }
    7435  // else: Some allocations must have been touched while we are here. Next try.
    7436  }
    7437  else
    7438  {
    7439  // Could not find place in any of the blocks - break outer loop.
    7440  break;
    7441  }
    7442  }
    7443  /* Maximum number of tries exceeded - a very unlike event when many other
    7444  threads are simultaneously touching allocations making it impossible to make
    7445  lost at the same time as we try to allocate. */
    7446  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
    7447  {
    7448  return VK_ERROR_TOO_MANY_OBJECTS;
    7449  }
    7450  }
    7451 
    7452  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7453 }
    7454 
    7455 void VmaBlockVector::Free(
    7456  VmaAllocation hAllocation)
    7457 {
    7458  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
    7459 
    7460  // Scope for lock.
    7461  {
    7462  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7463 
    7464  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    7465 
    7466  if(IsCorruptionDetectionEnabled())
    7467  {
    7468  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
    7469  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
    7470  }
    7471 
    7472  if(hAllocation->IsPersistentMap())
    7473  {
    7474  pBlock->Unmap(m_hAllocator, 1);
    7475  }
    7476 
    7477  pBlock->m_Metadata.Free(hAllocation);
    7478  VMA_HEAVY_ASSERT(pBlock->Validate());
    7479 
    7480  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
    7481 
    7482  // pBlock became empty after this deallocation.
    7483  if(pBlock->m_Metadata.IsEmpty())
    7484  {
    7485  // Already has empty Allocation. We don't want to have two, so delete this one.
    7486  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
    7487  {
    7488  pBlockToDelete = pBlock;
    7489  Remove(pBlock);
    7490  }
    7491  // We now have first empty block.
    7492  else
    7493  {
    7494  m_HasEmptyBlock = true;
    7495  }
    7496  }
    7497  // pBlock didn't become empty, but we have another empty block - find and free that one.
    7498  // (This is optional, heuristics.)
    7499  else if(m_HasEmptyBlock)
    7500  {
    7501  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
    7502  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
    7503  {
    7504  pBlockToDelete = pLastBlock;
    7505  m_Blocks.pop_back();
    7506  m_HasEmptyBlock = false;
    7507  }
    7508  }
    7509 
    7510  IncrementallySortBlocks();
    7511  }
    7512 
    7513  // Destruction of a free Allocation. Deferred until this point, outside of mutex
    7514  // lock, for performance reason.
    7515  if(pBlockToDelete != VMA_NULL)
    7516  {
    7517  VMA_DEBUG_LOG(" Deleted empty allocation");
    7518  pBlockToDelete->Destroy(m_hAllocator);
    7519  vma_delete(m_hAllocator, pBlockToDelete);
    7520  }
    7521 }
    7522 
    7523 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
    7524 {
    7525  VkDeviceSize result = 0;
    7526  for(size_t i = m_Blocks.size(); i--; )
    7527  {
    7528  result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize());
    7529  if(result >= m_PreferredBlockSize)
    7530  {
    7531  break;
    7532  }
    7533  }
    7534  return result;
    7535 }
    7536 
    7537 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
    7538 {
    7539  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7540  {
    7541  if(m_Blocks[blockIndex] == pBlock)
    7542  {
    7543  VmaVectorRemove(m_Blocks, blockIndex);
    7544  return;
    7545  }
    7546  }
    7547  VMA_ASSERT(0);
    7548 }
    7549 
    7550 void VmaBlockVector::IncrementallySortBlocks()
    7551 {
    7552  // Bubble sort only until first swap.
    7553  for(size_t i = 1; i < m_Blocks.size(); ++i)
    7554  {
    7555  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
    7556  {
    7557  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
    7558  return;
    7559  }
    7560  }
    7561 }
    7562 
    7563 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
    7564 {
    7565  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    7566  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
    7567  allocInfo.allocationSize = blockSize;
    7568  VkDeviceMemory mem = VK_NULL_HANDLE;
    7569  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
    7570  if(res < 0)
    7571  {
    7572  return res;
    7573  }
    7574 
    7575  // New VkDeviceMemory successfully created.
    7576 
    7577  // Create new Allocation for it.
    7578  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
    7579  pBlock->Init(
    7580  m_MemoryTypeIndex,
    7581  mem,
    7582  allocInfo.allocationSize,
    7583  m_NextBlockId++);
    7584 
    7585  m_Blocks.push_back(pBlock);
    7586  if(pNewBlockIndex != VMA_NULL)
    7587  {
    7588  *pNewBlockIndex = m_Blocks.size() - 1;
    7589  }
    7590 
    7591  return VK_SUCCESS;
    7592 }
    7593 
    7594 #if VMA_STATS_STRING_ENABLED
    7595 
    7596 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
    7597 {
    7598  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7599 
    7600  json.BeginObject();
    7601 
    7602  if(m_IsCustomPool)
    7603  {
    7604  json.WriteString("MemoryTypeIndex");
    7605  json.WriteNumber(m_MemoryTypeIndex);
    7606 
    7607  json.WriteString("BlockSize");
    7608  json.WriteNumber(m_PreferredBlockSize);
    7609 
    7610  json.WriteString("BlockCount");
    7611  json.BeginObject(true);
    7612  if(m_MinBlockCount > 0)
    7613  {
    7614  json.WriteString("Min");
    7615  json.WriteNumber((uint64_t)m_MinBlockCount);
    7616  }
    7617  if(m_MaxBlockCount < SIZE_MAX)
    7618  {
    7619  json.WriteString("Max");
    7620  json.WriteNumber((uint64_t)m_MaxBlockCount);
    7621  }
    7622  json.WriteString("Cur");
    7623  json.WriteNumber((uint64_t)m_Blocks.size());
    7624  json.EndObject();
    7625 
    7626  if(m_FrameInUseCount > 0)
    7627  {
    7628  json.WriteString("FrameInUseCount");
    7629  json.WriteNumber(m_FrameInUseCount);
    7630  }
    7631  }
    7632  else
    7633  {
    7634  json.WriteString("PreferredBlockSize");
    7635  json.WriteNumber(m_PreferredBlockSize);
    7636  }
    7637 
    7638  json.WriteString("Blocks");
    7639  json.BeginObject();
    7640  for(size_t i = 0; i < m_Blocks.size(); ++i)
    7641  {
    7642  json.BeginString();
    7643  json.ContinueString(m_Blocks[i]->GetId());
    7644  json.EndString();
    7645 
    7646  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
    7647  }
    7648  json.EndObject();
    7649 
    7650  json.EndObject();
    7651 }
    7652 
    7653 #endif // #if VMA_STATS_STRING_ENABLED
    7654 
    7655 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
    7656  VmaAllocator hAllocator,
    7657  uint32_t currentFrameIndex)
    7658 {
    7659  if(m_pDefragmentator == VMA_NULL)
    7660  {
    7661  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
    7662  hAllocator,
    7663  this,
    7664  currentFrameIndex);
    7665  }
    7666 
    7667  return m_pDefragmentator;
    7668 }
    7669 
    7670 VkResult VmaBlockVector::Defragment(
    7671  VmaDefragmentationStats* pDefragmentationStats,
    7672  VkDeviceSize& maxBytesToMove,
    7673  uint32_t& maxAllocationsToMove)
    7674 {
    7675  if(m_pDefragmentator == VMA_NULL)
    7676  {
    7677  return VK_SUCCESS;
    7678  }
    7679 
    7680  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7681 
    7682  // Defragment.
    7683  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
    7684 
    7685  // Accumulate statistics.
    7686  if(pDefragmentationStats != VMA_NULL)
    7687  {
    7688  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
    7689  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
    7690  pDefragmentationStats->bytesMoved += bytesMoved;
    7691  pDefragmentationStats->allocationsMoved += allocationsMoved;
    7692  VMA_ASSERT(bytesMoved <= maxBytesToMove);
    7693  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
    7694  maxBytesToMove -= bytesMoved;
    7695  maxAllocationsToMove -= allocationsMoved;
    7696  }
    7697 
    7698  // Free empty blocks.
    7699  m_HasEmptyBlock = false;
    7700  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
    7701  {
    7702  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
    7703  if(pBlock->m_Metadata.IsEmpty())
    7704  {
    7705  if(m_Blocks.size() > m_MinBlockCount)
    7706  {
    7707  if(pDefragmentationStats != VMA_NULL)
    7708  {
    7709  ++pDefragmentationStats->deviceMemoryBlocksFreed;
    7710  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
    7711  }
    7712 
    7713  VmaVectorRemove(m_Blocks, blockIndex);
    7714  pBlock->Destroy(m_hAllocator);
    7715  vma_delete(m_hAllocator, pBlock);
    7716  }
    7717  else
    7718  {
    7719  m_HasEmptyBlock = true;
    7720  }
    7721  }
    7722  }
    7723 
    7724  return result;
    7725 }
    7726 
    7727 void VmaBlockVector::DestroyDefragmentator()
    7728 {
    7729  if(m_pDefragmentator != VMA_NULL)
    7730  {
    7731  vma_delete(m_hAllocator, m_pDefragmentator);
    7732  m_pDefragmentator = VMA_NULL;
    7733  }
    7734 }
    7735 
    7736 void VmaBlockVector::MakePoolAllocationsLost(
    7737  uint32_t currentFrameIndex,
    7738  size_t* pLostAllocationCount)
    7739 {
    7740  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7741  size_t lostAllocationCount = 0;
    7742  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7743  {
    7744  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7745  VMA_ASSERT(pBlock);
    7746  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
    7747  }
    7748  if(pLostAllocationCount != VMA_NULL)
    7749  {
    7750  *pLostAllocationCount = lostAllocationCount;
    7751  }
    7752 }
    7753 
    7754 VkResult VmaBlockVector::CheckCorruption()
    7755 {
    7756  if(!IsCorruptionDetectionEnabled())
    7757  {
    7758  return VK_ERROR_FEATURE_NOT_PRESENT;
    7759  }
    7760 
    7761  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7762  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7763  {
    7764  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7765  VMA_ASSERT(pBlock);
    7766  VkResult res = pBlock->CheckCorruption(m_hAllocator);
    7767  if(res != VK_SUCCESS)
    7768  {
    7769  return res;
    7770  }
    7771  }
    7772  return VK_SUCCESS;
    7773 }
    7774 
    7775 void VmaBlockVector::AddStats(VmaStats* pStats)
    7776 {
    7777  const uint32_t memTypeIndex = m_MemoryTypeIndex;
    7778  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
    7779 
    7780  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7781 
    7782  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7783  {
    7784  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7785  VMA_ASSERT(pBlock);
    7786  VMA_HEAVY_ASSERT(pBlock->Validate());
    7787  VmaStatInfo allocationStatInfo;
    7788  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
    7789  VmaAddStatInfo(pStats->total, allocationStatInfo);
    7790  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    7791  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    7792  }
    7793 }
    7794 
    7796 // VmaDefragmentator members definition
    7797 
    7798 VmaDefragmentator::VmaDefragmentator(
    7799  VmaAllocator hAllocator,
    7800  VmaBlockVector* pBlockVector,
    7801  uint32_t currentFrameIndex) :
    7802  m_hAllocator(hAllocator),
    7803  m_pBlockVector(pBlockVector),
    7804  m_CurrentFrameIndex(currentFrameIndex),
    7805  m_BytesMoved(0),
    7806  m_AllocationsMoved(0),
    7807  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
    7808  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
    7809 {
    7810 }
    7811 
    7812 VmaDefragmentator::~VmaDefragmentator()
    7813 {
    7814  for(size_t i = m_Blocks.size(); i--; )
    7815  {
    7816  vma_delete(m_hAllocator, m_Blocks[i]);
    7817  }
    7818 }
    7819 
    7820 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
    7821 {
    7822  AllocationInfo allocInfo;
    7823  allocInfo.m_hAllocation = hAlloc;
    7824  allocInfo.m_pChanged = pChanged;
    7825  m_Allocations.push_back(allocInfo);
    7826 }
    7827 
    7828 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
    7829 {
    7830  // It has already been mapped for defragmentation.
    7831  if(m_pMappedDataForDefragmentation)
    7832  {
    7833  *ppMappedData = m_pMappedDataForDefragmentation;
    7834  return VK_SUCCESS;
    7835  }
    7836 
    7837  // It is originally mapped.
    7838  if(m_pBlock->GetMappedData())
    7839  {
    7840  *ppMappedData = m_pBlock->GetMappedData();
    7841  return VK_SUCCESS;
    7842  }
    7843 
    7844  // Map on first usage.
    7845  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
    7846  *ppMappedData = m_pMappedDataForDefragmentation;
    7847  return res;
    7848 }
    7849 
    7850 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
    7851 {
    7852  if(m_pMappedDataForDefragmentation != VMA_NULL)
    7853  {
    7854  m_pBlock->Unmap(hAllocator, 1);
    7855  }
    7856 }
    7857 
    7858 VkResult VmaDefragmentator::DefragmentRound(
    7859  VkDeviceSize maxBytesToMove,
    7860  uint32_t maxAllocationsToMove)
    7861 {
    7862  if(m_Blocks.empty())
    7863  {
    7864  return VK_SUCCESS;
    7865  }
    7866 
    7867  size_t srcBlockIndex = m_Blocks.size() - 1;
    7868  size_t srcAllocIndex = SIZE_MAX;
    7869  for(;;)
    7870  {
    7871  // 1. Find next allocation to move.
    7872  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
    7873  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
    7874  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
    7875  {
    7876  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
    7877  {
    7878  // Finished: no more allocations to process.
    7879  if(srcBlockIndex == 0)
    7880  {
    7881  return VK_SUCCESS;
    7882  }
    7883  else
    7884  {
    7885  --srcBlockIndex;
    7886  srcAllocIndex = SIZE_MAX;
    7887  }
    7888  }
    7889  else
    7890  {
    7891  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
    7892  }
    7893  }
    7894 
    7895  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
    7896  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
    7897 
    7898  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
    7899  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
    7900  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
    7901  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
    7902 
    7903  // 2. Try to find new place for this allocation in preceding or current block.
    7904  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
    7905  {
    7906  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
    7907  VmaAllocationRequest dstAllocRequest;
    7908  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
    7909  m_CurrentFrameIndex,
    7910  m_pBlockVector->GetFrameInUseCount(),
    7911  m_pBlockVector->GetBufferImageGranularity(),
    7912  size,
    7913  alignment,
    7914  suballocType,
    7915  false, // canMakeOtherLost
    7916  &dstAllocRequest) &&
    7917  MoveMakesSense(
    7918  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
    7919  {
    7920  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
    7921 
    7922  // Reached limit on number of allocations or bytes to move.
    7923  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
    7924  (m_BytesMoved + size > maxBytesToMove))
    7925  {
    7926  return VK_INCOMPLETE;
    7927  }
    7928 
    7929  void* pDstMappedData = VMA_NULL;
    7930  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
    7931  if(res != VK_SUCCESS)
    7932  {
    7933  return res;
    7934  }
    7935 
    7936  void* pSrcMappedData = VMA_NULL;
    7937  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
    7938  if(res != VK_SUCCESS)
    7939  {
    7940  return res;
    7941  }
    7942 
    7943  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
    7944  memcpy(
    7945  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
    7946  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
    7947  static_cast<size_t>(size));
    7948 
    7949  if(VMA_DEBUG_MARGIN > 0)
    7950  {
    7951  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
    7952  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
    7953  }
    7954 
    7955  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
    7956  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
    7957 
    7958  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
    7959 
    7960  if(allocInfo.m_pChanged != VMA_NULL)
    7961  {
    7962  *allocInfo.m_pChanged = VK_TRUE;
    7963  }
    7964 
    7965  ++m_AllocationsMoved;
    7966  m_BytesMoved += size;
    7967 
    7968  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
    7969 
    7970  break;
    7971  }
    7972  }
    7973 
    7974  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
    7975 
    7976  if(srcAllocIndex > 0)
    7977  {
    7978  --srcAllocIndex;
    7979  }
    7980  else
    7981  {
    7982  if(srcBlockIndex > 0)
    7983  {
    7984  --srcBlockIndex;
    7985  srcAllocIndex = SIZE_MAX;
    7986  }
    7987  else
    7988  {
    7989  return VK_SUCCESS;
    7990  }
    7991  }
    7992  }
    7993 }
    7994 
    7995 VkResult VmaDefragmentator::Defragment(
    7996  VkDeviceSize maxBytesToMove,
    7997  uint32_t maxAllocationsToMove)
    7998 {
    7999  if(m_Allocations.empty())
    8000  {
    8001  return VK_SUCCESS;
    8002  }
    8003 
    8004  // Create block info for each block.
    8005  const size_t blockCount = m_pBlockVector->m_Blocks.size();
    8006  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8007  {
    8008  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
    8009  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
    8010  m_Blocks.push_back(pBlockInfo);
    8011  }
    8012 
    8013  // Sort them by m_pBlock pointer value.
    8014  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
    8015 
    8016  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
    8017  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
    8018  {
    8019  AllocationInfo& allocInfo = m_Allocations[blockIndex];
    8020  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
    8021  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    8022  {
    8023  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
    8024  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
    8025  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
    8026  {
    8027  (*it)->m_Allocations.push_back(allocInfo);
    8028  }
    8029  else
    8030  {
    8031  VMA_ASSERT(0);
    8032  }
    8033  }
    8034  }
    8035  m_Allocations.clear();
    8036 
    8037  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8038  {
    8039  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
    8040  pBlockInfo->CalcHasNonMovableAllocations();
    8041  pBlockInfo->SortAllocationsBySizeDescecnding();
    8042  }
    8043 
    8044  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
    8045  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
    8046 
    8047  // Execute defragmentation rounds (the main part).
    8048  VkResult result = VK_SUCCESS;
    8049  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
    8050  {
    8051  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
    8052  }
    8053 
    8054  // Unmap blocks that were mapped for defragmentation.
    8055  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8056  {
    8057  m_Blocks[blockIndex]->Unmap(m_hAllocator);
    8058  }
    8059 
    8060  return result;
    8061 }
    8062 
    8063 bool VmaDefragmentator::MoveMakesSense(
    8064  size_t dstBlockIndex, VkDeviceSize dstOffset,
    8065  size_t srcBlockIndex, VkDeviceSize srcOffset)
    8066 {
    8067  if(dstBlockIndex < srcBlockIndex)
    8068  {
    8069  return true;
    8070  }
    8071  if(dstBlockIndex > srcBlockIndex)
    8072  {
    8073  return false;
    8074  }
    8075  if(dstOffset < srcOffset)
    8076  {
    8077  return true;
    8078  }
    8079  return false;
    8080 }
    8081 
    8083 // VmaRecorder
    8084 
    8085 #if VMA_RECORDING_ENABLED
    8086 
    8087 VmaRecorder::VmaRecorder() :
    8088  m_UseMutex(true),
    8089  m_Flags(0),
    8090  m_File(VMA_NULL),
    8091  m_Freq(INT64_MAX),
    8092  m_StartCounter(INT64_MAX)
    8093 {
    8094 }
    8095 
    8096 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
    8097 {
    8098  m_UseMutex = useMutex;
    8099  m_Flags = settings.flags;
    8100 
    8101  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
    8102  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
    8103 
    8104  // Open file for writing.
    8105  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
    8106  if(err != 0)
    8107  {
    8108  return VK_ERROR_INITIALIZATION_FAILED;
    8109  }
    8110 
    8111  // Write header.
    8112  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
    8113  fprintf(m_File, "%s\n", "1,2");
    8114 
    8115  return VK_SUCCESS;
    8116 }
    8117 
    8118 VmaRecorder::~VmaRecorder()
    8119 {
    8120  if(m_File != VMA_NULL)
    8121  {
    8122  fclose(m_File);
    8123  }
    8124 }
    8125 
    8126 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
    8127 {
    8128  CallParams callParams;
    8129  GetBasicParams(callParams);
    8130 
    8131  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8132  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8133  Flush();
    8134 }
    8135 
    8136 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
    8137 {
    8138  CallParams callParams;
    8139  GetBasicParams(callParams);
    8140 
    8141  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8142  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8143  Flush();
    8144 }
    8145 
    8146 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
    8147 {
    8148  CallParams callParams;
    8149  GetBasicParams(callParams);
    8150 
    8151  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8152  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
    8153  createInfo.memoryTypeIndex,
    8154  createInfo.flags,
    8155  createInfo.blockSize,
    8156  createInfo.minBlockCount,
    8157  createInfo.maxBlockCount,
    8158  createInfo.frameInUseCount,
    8159  pool);
    8160  Flush();
    8161 }
    8162 
    8163 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
    8164 {
    8165  CallParams callParams;
    8166  GetBasicParams(callParams);
    8167 
    8168  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8169  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
    8170  pool);
    8171  Flush();
    8172 }
    8173 
    8174 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
    8175  const VkMemoryRequirements& vkMemReq,
    8176  const VmaAllocationCreateInfo& createInfo,
    8177  VmaAllocation allocation)
    8178 {
    8179  CallParams callParams;
    8180  GetBasicParams(callParams);
    8181 
    8182  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8183  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8184  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8185  vkMemReq.size,
    8186  vkMemReq.alignment,
    8187  vkMemReq.memoryTypeBits,
    8188  createInfo.flags,
    8189  createInfo.usage,
    8190  createInfo.requiredFlags,
    8191  createInfo.preferredFlags,
    8192  createInfo.memoryTypeBits,
    8193  createInfo.pool,
    8194  allocation,
    8195  userDataStr.GetString());
    8196  Flush();
    8197 }
    8198 
    8199 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    8200  const VkMemoryRequirements& vkMemReq,
    8201  bool requiresDedicatedAllocation,
    8202  bool prefersDedicatedAllocation,
    8203  const VmaAllocationCreateInfo& createInfo,
    8204  VmaAllocation allocation)
    8205 {
    8206  CallParams callParams;
    8207  GetBasicParams(callParams);
    8208 
    8209  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8210  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8211  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8212  vkMemReq.size,
    8213  vkMemReq.alignment,
    8214  vkMemReq.memoryTypeBits,
    8215  requiresDedicatedAllocation ? 1 : 0,
    8216  prefersDedicatedAllocation ? 1 : 0,
    8217  createInfo.flags,
    8218  createInfo.usage,
    8219  createInfo.requiredFlags,
    8220  createInfo.preferredFlags,
    8221  createInfo.memoryTypeBits,
    8222  createInfo.pool,
    8223  allocation,
    8224  userDataStr.GetString());
    8225  Flush();
    8226 }
    8227 
    8228 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
    8229  const VkMemoryRequirements& vkMemReq,
    8230  bool requiresDedicatedAllocation,
    8231  bool prefersDedicatedAllocation,
    8232  const VmaAllocationCreateInfo& createInfo,
    8233  VmaAllocation allocation)
    8234 {
    8235  CallParams callParams;
    8236  GetBasicParams(callParams);
    8237 
    8238  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8239  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8240  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8241  vkMemReq.size,
    8242  vkMemReq.alignment,
    8243  vkMemReq.memoryTypeBits,
    8244  requiresDedicatedAllocation ? 1 : 0,
    8245  prefersDedicatedAllocation ? 1 : 0,
    8246  createInfo.flags,
    8247  createInfo.usage,
    8248  createInfo.requiredFlags,
    8249  createInfo.preferredFlags,
    8250  createInfo.memoryTypeBits,
    8251  createInfo.pool,
    8252  allocation,
    8253  userDataStr.GetString());
    8254  Flush();
    8255 }
    8256 
    8257 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
    8258  VmaAllocation allocation)
    8259 {
    8260  CallParams callParams;
    8261  GetBasicParams(callParams);
    8262 
    8263  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8264  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8265  allocation);
    8266  Flush();
    8267 }
    8268 
    8269 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
    8270  VmaAllocation allocation,
    8271  const void* pUserData)
    8272 {
    8273  CallParams callParams;
    8274  GetBasicParams(callParams);
    8275 
    8276  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8277  UserDataString userDataStr(
    8278  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
    8279  pUserData);
    8280  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8281  allocation,
    8282  userDataStr.GetString());
    8283  Flush();
    8284 }
    8285 
    8286 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
    8287  VmaAllocation allocation)
    8288 {
    8289  CallParams callParams;
    8290  GetBasicParams(callParams);
    8291 
    8292  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8293  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
    8294  allocation);
    8295  Flush();
    8296 }
    8297 
    8298 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
    8299  VmaAllocation allocation)
    8300 {
    8301  CallParams callParams;
    8302  GetBasicParams(callParams);
    8303 
    8304  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8305  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8306  allocation);
    8307  Flush();
    8308 }
    8309 
    8310 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
    8311  VmaAllocation allocation)
    8312 {
    8313  CallParams callParams;
    8314  GetBasicParams(callParams);
    8315 
    8316  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8317  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8318  allocation);
    8319  Flush();
    8320 }
    8321 
    8322 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
    8323  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8324 {
    8325  CallParams callParams;
    8326  GetBasicParams(callParams);
    8327 
    8328  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8329  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8330  allocation,
    8331  offset,
    8332  size);
    8333  Flush();
    8334 }
    8335 
    8336 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
    8337  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8338 {
    8339  CallParams callParams;
    8340  GetBasicParams(callParams);
    8341 
    8342  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8343  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8344  allocation,
    8345  offset,
    8346  size);
    8347  Flush();
    8348 }
    8349 
    8350 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
    8351  const VkBufferCreateInfo& bufCreateInfo,
    8352  const VmaAllocationCreateInfo& allocCreateInfo,
    8353  VmaAllocation allocation)
    8354 {
    8355  CallParams callParams;
    8356  GetBasicParams(callParams);
    8357 
    8358  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8359  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8360  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8361  bufCreateInfo.flags,
    8362  bufCreateInfo.size,
    8363  bufCreateInfo.usage,
    8364  bufCreateInfo.sharingMode,
    8365  allocCreateInfo.flags,
    8366  allocCreateInfo.usage,
    8367  allocCreateInfo.requiredFlags,
    8368  allocCreateInfo.preferredFlags,
    8369  allocCreateInfo.memoryTypeBits,
    8370  allocCreateInfo.pool,
    8371  allocation,
    8372  userDataStr.GetString());
    8373  Flush();
    8374 }
    8375 
    8376 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
    8377  const VkImageCreateInfo& imageCreateInfo,
    8378  const VmaAllocationCreateInfo& allocCreateInfo,
    8379  VmaAllocation allocation)
    8380 {
    8381  CallParams callParams;
    8382  GetBasicParams(callParams);
    8383 
    8384  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8385  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8386  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8387  imageCreateInfo.flags,
    8388  imageCreateInfo.imageType,
    8389  imageCreateInfo.format,
    8390  imageCreateInfo.extent.width,
    8391  imageCreateInfo.extent.height,
    8392  imageCreateInfo.extent.depth,
    8393  imageCreateInfo.mipLevels,
    8394  imageCreateInfo.arrayLayers,
    8395  imageCreateInfo.samples,
    8396  imageCreateInfo.tiling,
    8397  imageCreateInfo.usage,
    8398  imageCreateInfo.sharingMode,
    8399  imageCreateInfo.initialLayout,
    8400  allocCreateInfo.flags,
    8401  allocCreateInfo.usage,
    8402  allocCreateInfo.requiredFlags,
    8403  allocCreateInfo.preferredFlags,
    8404  allocCreateInfo.memoryTypeBits,
    8405  allocCreateInfo.pool,
    8406  allocation,
    8407  userDataStr.GetString());
    8408  Flush();
    8409 }
    8410 
    8411 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
    8412  VmaAllocation allocation)
    8413 {
    8414  CallParams callParams;
    8415  GetBasicParams(callParams);
    8416 
    8417  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8418  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
    8419  allocation);
    8420  Flush();
    8421 }
    8422 
    8423 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
    8424  VmaAllocation allocation)
    8425 {
    8426  CallParams callParams;
    8427  GetBasicParams(callParams);
    8428 
    8429  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8430  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
    8431  allocation);
    8432  Flush();
    8433 }
    8434 
    8435 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
    8436 {
    8437  if(pUserData != VMA_NULL)
    8438  {
    8439  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
    8440  {
    8441  m_Str = (const char*)pUserData;
    8442  }
    8443  else
    8444  {
    8445  sprintf_s(m_PtrStr, "%p", pUserData);
    8446  m_Str = m_PtrStr;
    8447  }
    8448  }
    8449  else
    8450  {
    8451  m_Str = "";
    8452  }
    8453 }
    8454 
    8455 void VmaRecorder::GetBasicParams(CallParams& outParams)
    8456 {
    8457  outParams.threadId = GetCurrentThreadId();
    8458 
    8459  LARGE_INTEGER counter;
    8460  QueryPerformanceCounter(&counter);
    8461  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
    8462 }
    8463 
    8464 void VmaRecorder::Flush()
    8465 {
    8466  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
    8467  {
    8468  fflush(m_File);
    8469  }
    8470 }
    8471 
    8472 #endif // #if VMA_RECORDING_ENABLED
    8473 
    8475 // VmaAllocator_T
    8476 
    8477 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
    8478  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
    8479  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
    8480  m_hDevice(pCreateInfo->device),
    8481  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
    8482  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
    8483  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
    8484  m_PreferredLargeHeapBlockSize(0),
    8485  m_PhysicalDevice(pCreateInfo->physicalDevice),
    8486  m_CurrentFrameIndex(0),
    8487  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
    8488  m_NextPoolId(0)
    8489 #if VMA_RECORDING_ENABLED
    8490  ,m_pRecorder(VMA_NULL)
    8491 #endif
    8492 {
    8493  if(VMA_DEBUG_DETECT_CORRUPTION)
    8494  {
    8495  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
    8496  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
    8497  }
    8498 
    8499  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
    8500 
    8501 #if !(VMA_DEDICATED_ALLOCATION)
    8502  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
    8503  {
    8504  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
    8505  }
    8506 #endif
    8507 
    8508  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
    8509  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
    8510  memset(&m_MemProps, 0, sizeof(m_MemProps));
    8511 
    8512  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
    8513  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
    8514 
    8515  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    8516  {
    8517  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
    8518  }
    8519 
    8520  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
    8521  {
    8522  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
    8523  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
    8524  }
    8525 
    8526  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
    8527 
    8528  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
    8529  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
    8530 
    8531  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
    8532  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
    8533 
    8534  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
    8535  {
    8536  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
    8537  {
    8538  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
    8539  if(limit != VK_WHOLE_SIZE)
    8540  {
    8541  m_HeapSizeLimit[heapIndex] = limit;
    8542  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
    8543  {
    8544  m_MemProps.memoryHeaps[heapIndex].size = limit;
    8545  }
    8546  }
    8547  }
    8548  }
    8549 
    8550  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    8551  {
    8552  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
    8553 
    8554  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
    8555  this,
    8556  memTypeIndex,
    8557  preferredBlockSize,
    8558  0,
    8559  SIZE_MAX,
    8560  GetBufferImageGranularity(),
    8561  pCreateInfo->frameInUseCount,
    8562  false); // isCustomPool
    8563  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
    8564  // becase minBlockCount is 0.
    8565  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
    8566 
    8567  }
    8568 }
    8569 
    8570 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
    8571 {
    8572  VkResult res = VK_SUCCESS;
    8573 
    8574  if(pCreateInfo->pRecordSettings != VMA_NULL &&
    8575  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
    8576  {
    8577 #if VMA_RECORDING_ENABLED
    8578  m_pRecorder = vma_new(this, VmaRecorder)();
    8579  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
    8580  if(res != VK_SUCCESS)
    8581  {
    8582  return res;
    8583  }
    8584  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
    8585 #else
    8586  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
    8587  return VK_ERROR_FEATURE_NOT_PRESENT;
    8588 #endif
    8589  }
    8590 
    8591  return res;
    8592 }
    8593 
    8594 VmaAllocator_T::~VmaAllocator_T()
    8595 {
    8596 #if VMA_RECORDING_ENABLED
    8597  if(m_pRecorder != VMA_NULL)
    8598  {
    8599  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
    8600  vma_delete(this, m_pRecorder);
    8601  }
    8602 #endif
    8603 
    8604  VMA_ASSERT(m_Pools.empty());
    8605 
    8606  for(size_t i = GetMemoryTypeCount(); i--; )
    8607  {
    8608  vma_delete(this, m_pDedicatedAllocations[i]);
    8609  vma_delete(this, m_pBlockVectors[i]);
    8610  }
    8611 }
    8612 
    8613 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
    8614 {
    8615 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8616  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
    8617  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
    8618  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    8619  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
    8620  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
    8621  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
    8622  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
    8623  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
    8624  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
    8625  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
    8626  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
    8627  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
    8628  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
    8629  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
    8630  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
    8631  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
    8632 #if VMA_DEDICATED_ALLOCATION
    8633  if(m_UseKhrDedicatedAllocation)
    8634  {
    8635  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
    8636  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
    8637  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
    8638  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
    8639  }
    8640 #endif // #if VMA_DEDICATED_ALLOCATION
    8641 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8642 
    8643 #define VMA_COPY_IF_NOT_NULL(funcName) \
    8644  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
    8645 
    8646  if(pVulkanFunctions != VMA_NULL)
    8647  {
    8648  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
    8649  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
    8650  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
    8651  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
    8652  VMA_COPY_IF_NOT_NULL(vkMapMemory);
    8653  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
    8654  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
    8655  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
    8656  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
    8657  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
    8658  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
    8659  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
    8660  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
    8661  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
    8662  VMA_COPY_IF_NOT_NULL(vkCreateImage);
    8663  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
    8664 #if VMA_DEDICATED_ALLOCATION
    8665  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
    8666  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
    8667 #endif
    8668  }
    8669 
    8670 #undef VMA_COPY_IF_NOT_NULL
    8671 
    8672  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
    8673  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
    8674  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
    8675  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
    8676  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
    8677  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
    8678  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
    8679  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
    8680  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
    8681  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
    8682  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
    8683  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
    8684  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
    8685  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
    8686  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
    8687  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
    8688  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
    8689  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
    8690 #if VMA_DEDICATED_ALLOCATION
    8691  if(m_UseKhrDedicatedAllocation)
    8692  {
    8693  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
    8694  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
    8695  }
    8696 #endif
    8697 }
    8698 
    8699 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
    8700 {
    8701  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    8702  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
    8703  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
    8704  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
    8705 }
    8706 
    8707 VkResult VmaAllocator_T::AllocateMemoryOfType(
    8708  VkDeviceSize size,
    8709  VkDeviceSize alignment,
    8710  bool dedicatedAllocation,
    8711  VkBuffer dedicatedBuffer,
    8712  VkImage dedicatedImage,
    8713  const VmaAllocationCreateInfo& createInfo,
    8714  uint32_t memTypeIndex,
    8715  VmaSuballocationType suballocType,
    8716  VmaAllocation* pAllocation)
    8717 {
    8718  VMA_ASSERT(pAllocation != VMA_NULL);
    8719  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
    8720 
    8721  VmaAllocationCreateInfo finalCreateInfo = createInfo;
    8722 
    8723  // If memory type is not HOST_VISIBLE, disable MAPPED.
    8724  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8725  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    8726  {
    8727  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
    8728  }
    8729 
    8730  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
    8731  VMA_ASSERT(blockVector);
    8732 
    8733  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
    8734  bool preferDedicatedMemory =
    8735  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
    8736  dedicatedAllocation ||
    8737  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
    8738  size > preferredBlockSize / 2;
    8739 
    8740  if(preferDedicatedMemory &&
    8741  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
    8742  finalCreateInfo.pool == VK_NULL_HANDLE)
    8743  {
    8744  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
    8745  }
    8746 
    8747  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
    8748  {
    8749  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8750  {
    8751  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8752  }
    8753  else
    8754  {
    8755  return AllocateDedicatedMemory(
    8756  size,
    8757  suballocType,
    8758  memTypeIndex,
    8759  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8760  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8761  finalCreateInfo.pUserData,
    8762  dedicatedBuffer,
    8763  dedicatedImage,
    8764  pAllocation);
    8765  }
    8766  }
    8767  else
    8768  {
    8769  VkResult res = blockVector->Allocate(
    8770  VK_NULL_HANDLE, // hCurrentPool
    8771  m_CurrentFrameIndex.load(),
    8772  size,
    8773  alignment,
    8774  finalCreateInfo,
    8775  suballocType,
    8776  pAllocation);
    8777  if(res == VK_SUCCESS)
    8778  {
    8779  return res;
    8780  }
    8781 
    8782  // 5. Try dedicated memory.
    8783  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8784  {
    8785  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8786  }
    8787  else
    8788  {
    8789  res = AllocateDedicatedMemory(
    8790  size,
    8791  suballocType,
    8792  memTypeIndex,
    8793  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8794  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8795  finalCreateInfo.pUserData,
    8796  dedicatedBuffer,
    8797  dedicatedImage,
    8798  pAllocation);
    8799  if(res == VK_SUCCESS)
    8800  {
    8801  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
    8802  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
    8803  return VK_SUCCESS;
    8804  }
    8805  else
    8806  {
    8807  // Everything failed: Return error code.
    8808  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8809  return res;
    8810  }
    8811  }
    8812  }
    8813 }
    8814 
    8815 VkResult VmaAllocator_T::AllocateDedicatedMemory(
    8816  VkDeviceSize size,
    8817  VmaSuballocationType suballocType,
    8818  uint32_t memTypeIndex,
    8819  bool map,
    8820  bool isUserDataString,
    8821  void* pUserData,
    8822  VkBuffer dedicatedBuffer,
    8823  VkImage dedicatedImage,
    8824  VmaAllocation* pAllocation)
    8825 {
    8826  VMA_ASSERT(pAllocation);
    8827 
    8828  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    8829  allocInfo.memoryTypeIndex = memTypeIndex;
    8830  allocInfo.allocationSize = size;
    8831 
    8832 #if VMA_DEDICATED_ALLOCATION
    8833  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
    8834  if(m_UseKhrDedicatedAllocation)
    8835  {
    8836  if(dedicatedBuffer != VK_NULL_HANDLE)
    8837  {
    8838  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
    8839  dedicatedAllocInfo.buffer = dedicatedBuffer;
    8840  allocInfo.pNext = &dedicatedAllocInfo;
    8841  }
    8842  else if(dedicatedImage != VK_NULL_HANDLE)
    8843  {
    8844  dedicatedAllocInfo.image = dedicatedImage;
    8845  allocInfo.pNext = &dedicatedAllocInfo;
    8846  }
    8847  }
    8848 #endif // #if VMA_DEDICATED_ALLOCATION
    8849 
    8850  // Allocate VkDeviceMemory.
    8851  VkDeviceMemory hMemory = VK_NULL_HANDLE;
    8852  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
    8853  if(res < 0)
    8854  {
    8855  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8856  return res;
    8857  }
    8858 
    8859  void* pMappedData = VMA_NULL;
    8860  if(map)
    8861  {
    8862  res = (*m_VulkanFunctions.vkMapMemory)(
    8863  m_hDevice,
    8864  hMemory,
    8865  0,
    8866  VK_WHOLE_SIZE,
    8867  0,
    8868  &pMappedData);
    8869  if(res < 0)
    8870  {
    8871  VMA_DEBUG_LOG(" vkMapMemory FAILED");
    8872  FreeVulkanMemory(memTypeIndex, size, hMemory);
    8873  return res;
    8874  }
    8875  }
    8876 
    8877  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
    8878  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
    8879  (*pAllocation)->SetUserData(this, pUserData);
    8880  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    8881  {
    8882  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    8883  }
    8884 
    8885  // Register it in m_pDedicatedAllocations.
    8886  {
    8887  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    8888  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    8889  VMA_ASSERT(pDedicatedAllocations);
    8890  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
    8891  }
    8892 
    8893  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
    8894 
    8895  return VK_SUCCESS;
    8896 }
    8897 
    8898 void VmaAllocator_T::GetBufferMemoryRequirements(
    8899  VkBuffer hBuffer,
    8900  VkMemoryRequirements& memReq,
    8901  bool& requiresDedicatedAllocation,
    8902  bool& prefersDedicatedAllocation) const
    8903 {
    8904 #if VMA_DEDICATED_ALLOCATION
    8905  if(m_UseKhrDedicatedAllocation)
    8906  {
    8907  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
    8908  memReqInfo.buffer = hBuffer;
    8909 
    8910  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    8911 
    8912  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    8913  memReq2.pNext = &memDedicatedReq;
    8914 
    8915  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    8916 
    8917  memReq = memReq2.memoryRequirements;
    8918  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    8919  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    8920  }
    8921  else
    8922 #endif // #if VMA_DEDICATED_ALLOCATION
    8923  {
    8924  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
    8925  requiresDedicatedAllocation = false;
    8926  prefersDedicatedAllocation = false;
    8927  }
    8928 }
    8929 
    8930 void VmaAllocator_T::GetImageMemoryRequirements(
    8931  VkImage hImage,
    8932  VkMemoryRequirements& memReq,
    8933  bool& requiresDedicatedAllocation,
    8934  bool& prefersDedicatedAllocation) const
    8935 {
    8936 #if VMA_DEDICATED_ALLOCATION
    8937  if(m_UseKhrDedicatedAllocation)
    8938  {
    8939  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
    8940  memReqInfo.image = hImage;
    8941 
    8942  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    8943 
    8944  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    8945  memReq2.pNext = &memDedicatedReq;
    8946 
    8947  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    8948 
    8949  memReq = memReq2.memoryRequirements;
    8950  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    8951  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    8952  }
    8953  else
    8954 #endif // #if VMA_DEDICATED_ALLOCATION
    8955  {
    8956  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
    8957  requiresDedicatedAllocation = false;
    8958  prefersDedicatedAllocation = false;
    8959  }
    8960 }
    8961 
    8962 VkResult VmaAllocator_T::AllocateMemory(
    8963  const VkMemoryRequirements& vkMemReq,
    8964  bool requiresDedicatedAllocation,
    8965  bool prefersDedicatedAllocation,
    8966  VkBuffer dedicatedBuffer,
    8967  VkImage dedicatedImage,
    8968  const VmaAllocationCreateInfo& createInfo,
    8969  VmaSuballocationType suballocType,
    8970  VmaAllocation* pAllocation)
    8971 {
    8972  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
    8973  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8974  {
    8975  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
    8976  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8977  }
    8978  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8979  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
    8980  {
    8981  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
    8982  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8983  }
    8984  if(requiresDedicatedAllocation)
    8985  {
    8986  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8987  {
    8988  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
    8989  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8990  }
    8991  if(createInfo.pool != VK_NULL_HANDLE)
    8992  {
    8993  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
    8994  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8995  }
    8996  }
    8997  if((createInfo.pool != VK_NULL_HANDLE) &&
    8998  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
    8999  {
    9000  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
    9001  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9002  }
    9003 
    9004  if(createInfo.pool != VK_NULL_HANDLE)
    9005  {
    9006  const VkDeviceSize alignmentForPool = VMA_MAX(
    9007  vkMemReq.alignment,
    9008  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
    9009  return createInfo.pool->m_BlockVector.Allocate(
    9010  createInfo.pool,
    9011  m_CurrentFrameIndex.load(),
    9012  vkMemReq.size,
    9013  alignmentForPool,
    9014  createInfo,
    9015  suballocType,
    9016  pAllocation);
    9017  }
    9018  else
    9019  {
    9020  // Bit mask of memory Vulkan types acceptable for this allocation.
    9021  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
    9022  uint32_t memTypeIndex = UINT32_MAX;
    9023  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9024  if(res == VK_SUCCESS)
    9025  {
    9026  VkDeviceSize alignmentForMemType = VMA_MAX(
    9027  vkMemReq.alignment,
    9028  GetMemoryTypeMinAlignment(memTypeIndex));
    9029 
    9030  res = AllocateMemoryOfType(
    9031  vkMemReq.size,
    9032  alignmentForMemType,
    9033  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9034  dedicatedBuffer,
    9035  dedicatedImage,
    9036  createInfo,
    9037  memTypeIndex,
    9038  suballocType,
    9039  pAllocation);
    9040  // Succeeded on first try.
    9041  if(res == VK_SUCCESS)
    9042  {
    9043  return res;
    9044  }
    9045  // Allocation from this memory type failed. Try other compatible memory types.
    9046  else
    9047  {
    9048  for(;;)
    9049  {
    9050  // Remove old memTypeIndex from list of possibilities.
    9051  memoryTypeBits &= ~(1u << memTypeIndex);
    9052  // Find alternative memTypeIndex.
    9053  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9054  if(res == VK_SUCCESS)
    9055  {
    9056  alignmentForMemType = VMA_MAX(
    9057  vkMemReq.alignment,
    9058  GetMemoryTypeMinAlignment(memTypeIndex));
    9059 
    9060  res = AllocateMemoryOfType(
    9061  vkMemReq.size,
    9062  alignmentForMemType,
    9063  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9064  dedicatedBuffer,
    9065  dedicatedImage,
    9066  createInfo,
    9067  memTypeIndex,
    9068  suballocType,
    9069  pAllocation);
    9070  // Allocation from this alternative memory type succeeded.
    9071  if(res == VK_SUCCESS)
    9072  {
    9073  return res;
    9074  }
    9075  // else: Allocation from this memory type failed. Try next one - next loop iteration.
    9076  }
    9077  // No other matching memory type index could be found.
    9078  else
    9079  {
    9080  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
    9081  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9082  }
    9083  }
    9084  }
    9085  }
    9086  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
    9087  else
    9088  return res;
    9089  }
    9090 }
    9091 
    9092 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
    9093 {
    9094  VMA_ASSERT(allocation);
    9095 
    9096  if(allocation->CanBecomeLost() == false ||
    9097  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    9098  {
    9099  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    9100  {
    9101  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
    9102  }
    9103 
    9104  switch(allocation->GetType())
    9105  {
    9106  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9107  {
    9108  VmaBlockVector* pBlockVector = VMA_NULL;
    9109  VmaPool hPool = allocation->GetPool();
    9110  if(hPool != VK_NULL_HANDLE)
    9111  {
    9112  pBlockVector = &hPool->m_BlockVector;
    9113  }
    9114  else
    9115  {
    9116  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9117  pBlockVector = m_pBlockVectors[memTypeIndex];
    9118  }
    9119  pBlockVector->Free(allocation);
    9120  }
    9121  break;
    9122  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9123  FreeDedicatedMemory(allocation);
    9124  break;
    9125  default:
    9126  VMA_ASSERT(0);
    9127  }
    9128  }
    9129 
    9130  allocation->SetUserData(this, VMA_NULL);
    9131  vma_delete(this, allocation);
    9132 }
    9133 
    9134 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
    9135 {
    9136  // Initialize.
    9137  InitStatInfo(pStats->total);
    9138  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
    9139  InitStatInfo(pStats->memoryType[i]);
    9140  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    9141  InitStatInfo(pStats->memoryHeap[i]);
    9142 
    9143  // Process default pools.
    9144  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9145  {
    9146  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9147  VMA_ASSERT(pBlockVector);
    9148  pBlockVector->AddStats(pStats);
    9149  }
    9150 
    9151  // Process custom pools.
    9152  {
    9153  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9154  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9155  {
    9156  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
    9157  }
    9158  }
    9159 
    9160  // Process dedicated allocations.
    9161  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9162  {
    9163  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    9164  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9165  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9166  VMA_ASSERT(pDedicatedAllocVector);
    9167  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
    9168  {
    9169  VmaStatInfo allocationStatInfo;
    9170  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
    9171  VmaAddStatInfo(pStats->total, allocationStatInfo);
    9172  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    9173  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    9174  }
    9175  }
    9176 
    9177  // Postprocess.
    9178  VmaPostprocessCalcStatInfo(pStats->total);
    9179  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
    9180  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
    9181  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
    9182  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
    9183 }
    9184 
    9185 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
    9186 
    9187 VkResult VmaAllocator_T::Defragment(
    9188  VmaAllocation* pAllocations,
    9189  size_t allocationCount,
    9190  VkBool32* pAllocationsChanged,
    9191  const VmaDefragmentationInfo* pDefragmentationInfo,
    9192  VmaDefragmentationStats* pDefragmentationStats)
    9193 {
    9194  if(pAllocationsChanged != VMA_NULL)
    9195  {
    9196  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
    9197  }
    9198  if(pDefragmentationStats != VMA_NULL)
    9199  {
    9200  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
    9201  }
    9202 
    9203  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
    9204 
    9205  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
    9206 
    9207  const size_t poolCount = m_Pools.size();
    9208 
    9209  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
    9210  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
    9211  {
    9212  VmaAllocation hAlloc = pAllocations[allocIndex];
    9213  VMA_ASSERT(hAlloc);
    9214  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
    9215  // DedicatedAlloc cannot be defragmented.
    9216  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
    9217  // Only HOST_VISIBLE memory types can be defragmented.
    9218  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
    9219  // Lost allocation cannot be defragmented.
    9220  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
    9221  {
    9222  VmaBlockVector* pAllocBlockVector = VMA_NULL;
    9223 
    9224  const VmaPool hAllocPool = hAlloc->GetPool();
    9225  // This allocation belongs to custom pool.
    9226  if(hAllocPool != VK_NULL_HANDLE)
    9227  {
    9228  pAllocBlockVector = &hAllocPool->GetBlockVector();
    9229  }
    9230  // This allocation belongs to general pool.
    9231  else
    9232  {
    9233  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
    9234  }
    9235 
    9236  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
    9237 
    9238  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
    9239  &pAllocationsChanged[allocIndex] : VMA_NULL;
    9240  pDefragmentator->AddAllocation(hAlloc, pChanged);
    9241  }
    9242  }
    9243 
    9244  VkResult result = VK_SUCCESS;
    9245 
    9246  // ======== Main processing.
    9247 
    9248  VkDeviceSize maxBytesToMove = SIZE_MAX;
    9249  uint32_t maxAllocationsToMove = UINT32_MAX;
    9250  if(pDefragmentationInfo != VMA_NULL)
    9251  {
    9252  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
    9253  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
    9254  }
    9255 
    9256  // Process standard memory.
    9257  for(uint32_t memTypeIndex = 0;
    9258  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
    9259  ++memTypeIndex)
    9260  {
    9261  // Only HOST_VISIBLE memory types can be defragmented.
    9262  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9263  {
    9264  result = m_pBlockVectors[memTypeIndex]->Defragment(
    9265  pDefragmentationStats,
    9266  maxBytesToMove,
    9267  maxAllocationsToMove);
    9268  }
    9269  }
    9270 
    9271  // Process custom pools.
    9272  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
    9273  {
    9274  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
    9275  pDefragmentationStats,
    9276  maxBytesToMove,
    9277  maxAllocationsToMove);
    9278  }
    9279 
    9280  // ======== Destroy defragmentators.
    9281 
    9282  // Process custom pools.
    9283  for(size_t poolIndex = poolCount; poolIndex--; )
    9284  {
    9285  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
    9286  }
    9287 
    9288  // Process standard memory.
    9289  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
    9290  {
    9291  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9292  {
    9293  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
    9294  }
    9295  }
    9296 
    9297  return result;
    9298 }
    9299 
    9300 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
    9301 {
    9302  if(hAllocation->CanBecomeLost())
    9303  {
    9304  /*
    9305  Warning: This is a carefully designed algorithm.
    9306  Do not modify unless you really know what you're doing :)
    9307  */
    9308  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9309  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9310  for(;;)
    9311  {
    9312  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9313  {
    9314  pAllocationInfo->memoryType = UINT32_MAX;
    9315  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
    9316  pAllocationInfo->offset = 0;
    9317  pAllocationInfo->size = hAllocation->GetSize();
    9318  pAllocationInfo->pMappedData = VMA_NULL;
    9319  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9320  return;
    9321  }
    9322  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9323  {
    9324  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9325  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9326  pAllocationInfo->offset = hAllocation->GetOffset();
    9327  pAllocationInfo->size = hAllocation->GetSize();
    9328  pAllocationInfo->pMappedData = VMA_NULL;
    9329  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9330  return;
    9331  }
    9332  else // Last use time earlier than current time.
    9333  {
    9334  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9335  {
    9336  localLastUseFrameIndex = localCurrFrameIndex;
    9337  }
    9338  }
    9339  }
    9340  }
    9341  else
    9342  {
    9343 #if VMA_STATS_STRING_ENABLED
    9344  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9345  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9346  for(;;)
    9347  {
    9348  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9349  if(localLastUseFrameIndex == localCurrFrameIndex)
    9350  {
    9351  break;
    9352  }
    9353  else // Last use time earlier than current time.
    9354  {
    9355  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9356  {
    9357  localLastUseFrameIndex = localCurrFrameIndex;
    9358  }
    9359  }
    9360  }
    9361 #endif
    9362 
    9363  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9364  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9365  pAllocationInfo->offset = hAllocation->GetOffset();
    9366  pAllocationInfo->size = hAllocation->GetSize();
    9367  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
    9368  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9369  }
    9370 }
    9371 
    9372 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
    9373 {
    9374  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
    9375  if(hAllocation->CanBecomeLost())
    9376  {
    9377  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9378  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9379  for(;;)
    9380  {
    9381  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9382  {
    9383  return false;
    9384  }
    9385  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9386  {
    9387  return true;
    9388  }
    9389  else // Last use time earlier than current time.
    9390  {
    9391  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9392  {
    9393  localLastUseFrameIndex = localCurrFrameIndex;
    9394  }
    9395  }
    9396  }
    9397  }
    9398  else
    9399  {
    9400 #if VMA_STATS_STRING_ENABLED
    9401  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9402  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9403  for(;;)
    9404  {
    9405  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9406  if(localLastUseFrameIndex == localCurrFrameIndex)
    9407  {
    9408  break;
    9409  }
    9410  else // Last use time earlier than current time.
    9411  {
    9412  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9413  {
    9414  localLastUseFrameIndex = localCurrFrameIndex;
    9415  }
    9416  }
    9417  }
    9418 #endif
    9419 
    9420  return true;
    9421  }
    9422 }
    9423 
    9424 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
    9425 {
    9426  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
    9427 
    9428  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
    9429 
    9430  if(newCreateInfo.maxBlockCount == 0)
    9431  {
    9432  newCreateInfo.maxBlockCount = SIZE_MAX;
    9433  }
    9434  if(newCreateInfo.blockSize == 0)
    9435  {
    9436  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
    9437  }
    9438 
    9439  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
    9440 
    9441  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
    9442  if(res != VK_SUCCESS)
    9443  {
    9444  vma_delete(this, *pPool);
    9445  *pPool = VMA_NULL;
    9446  return res;
    9447  }
    9448 
    9449  // Add to m_Pools.
    9450  {
    9451  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9452  (*pPool)->SetId(m_NextPoolId++);
    9453  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
    9454  }
    9455 
    9456  return VK_SUCCESS;
    9457 }
    9458 
    9459 void VmaAllocator_T::DestroyPool(VmaPool pool)
    9460 {
    9461  // Remove from m_Pools.
    9462  {
    9463  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9464  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
    9465  VMA_ASSERT(success && "Pool not found in Allocator.");
    9466  }
    9467 
    9468  vma_delete(this, pool);
    9469 }
    9470 
    9471 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
    9472 {
    9473  pool->m_BlockVector.GetPoolStats(pPoolStats);
    9474 }
    9475 
    9476 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
    9477 {
    9478  m_CurrentFrameIndex.store(frameIndex);
    9479 }
    9480 
    9481 void VmaAllocator_T::MakePoolAllocationsLost(
    9482  VmaPool hPool,
    9483  size_t* pLostAllocationCount)
    9484 {
    9485  hPool->m_BlockVector.MakePoolAllocationsLost(
    9486  m_CurrentFrameIndex.load(),
    9487  pLostAllocationCount);
    9488 }
    9489 
    9490 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
    9491 {
    9492  return hPool->m_BlockVector.CheckCorruption();
    9493 }
    9494 
    9495 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
    9496 {
    9497  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
    9498 
    9499  // Process default pools.
    9500  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9501  {
    9502  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
    9503  {
    9504  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9505  VMA_ASSERT(pBlockVector);
    9506  VkResult localRes = pBlockVector->CheckCorruption();
    9507  switch(localRes)
    9508  {
    9509  case VK_ERROR_FEATURE_NOT_PRESENT:
    9510  break;
    9511  case VK_SUCCESS:
    9512  finalRes = VK_SUCCESS;
    9513  break;
    9514  default:
    9515  return localRes;
    9516  }
    9517  }
    9518  }
    9519 
    9520  // Process custom pools.
    9521  {
    9522  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9523  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9524  {
    9525  if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0)
    9526  {
    9527  VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption();
    9528  switch(localRes)
    9529  {
    9530  case VK_ERROR_FEATURE_NOT_PRESENT:
    9531  break;
    9532  case VK_SUCCESS:
    9533  finalRes = VK_SUCCESS;
    9534  break;
    9535  default:
    9536  return localRes;
    9537  }
    9538  }
    9539  }
    9540  }
    9541 
    9542  return finalRes;
    9543 }
    9544 
    9545 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
    9546 {
    9547  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
    9548  (*pAllocation)->InitLost();
    9549 }
    9550 
    9551 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
    9552 {
    9553  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
    9554 
    9555  VkResult res;
    9556  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9557  {
    9558  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9559  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
    9560  {
    9561  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9562  if(res == VK_SUCCESS)
    9563  {
    9564  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
    9565  }
    9566  }
    9567  else
    9568  {
    9569  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9570  }
    9571  }
    9572  else
    9573  {
    9574  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9575  }
    9576 
    9577  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
    9578  {
    9579  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
    9580  }
    9581 
    9582  return res;
    9583 }
    9584 
    9585 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
    9586 {
    9587  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
    9588  {
    9589  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
    9590  }
    9591 
    9592  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
    9593 
    9594  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
    9595  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9596  {
    9597  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9598  m_HeapSizeLimit[heapIndex] += size;
    9599  }
    9600 }
    9601 
    9602 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
    9603 {
    9604  if(hAllocation->CanBecomeLost())
    9605  {
    9606  return VK_ERROR_MEMORY_MAP_FAILED;
    9607  }
    9608 
    9609  switch(hAllocation->GetType())
    9610  {
    9611  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9612  {
    9613  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9614  char *pBytes = VMA_NULL;
    9615  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
    9616  if(res == VK_SUCCESS)
    9617  {
    9618  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
    9619  hAllocation->BlockAllocMap();
    9620  }
    9621  return res;
    9622  }
    9623  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9624  return hAllocation->DedicatedAllocMap(this, ppData);
    9625  default:
    9626  VMA_ASSERT(0);
    9627  return VK_ERROR_MEMORY_MAP_FAILED;
    9628  }
    9629 }
    9630 
    9631 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
    9632 {
    9633  switch(hAllocation->GetType())
    9634  {
    9635  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9636  {
    9637  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9638  hAllocation->BlockAllocUnmap();
    9639  pBlock->Unmap(this, 1);
    9640  }
    9641  break;
    9642  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9643  hAllocation->DedicatedAllocUnmap(this);
    9644  break;
    9645  default:
    9646  VMA_ASSERT(0);
    9647  }
    9648 }
    9649 
    9650 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
    9651 {
    9652  VkResult res = VK_SUCCESS;
    9653  switch(hAllocation->GetType())
    9654  {
    9655  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9656  res = GetVulkanFunctions().vkBindBufferMemory(
    9657  m_hDevice,
    9658  hBuffer,
    9659  hAllocation->GetMemory(),
    9660  0); //memoryOffset
    9661  break;
    9662  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9663  {
    9664  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9665  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
    9666  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
    9667  break;
    9668  }
    9669  default:
    9670  VMA_ASSERT(0);
    9671  }
    9672  return res;
    9673 }
    9674 
    9675 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
    9676 {
    9677  VkResult res = VK_SUCCESS;
    9678  switch(hAllocation->GetType())
    9679  {
    9680  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9681  res = GetVulkanFunctions().vkBindImageMemory(
    9682  m_hDevice,
    9683  hImage,
    9684  hAllocation->GetMemory(),
    9685  0); //memoryOffset
    9686  break;
    9687  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9688  {
    9689  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9690  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
    9691  res = pBlock->BindImageMemory(this, hAllocation, hImage);
    9692  break;
    9693  }
    9694  default:
    9695  VMA_ASSERT(0);
    9696  }
    9697  return res;
    9698 }
    9699 
    9700 void VmaAllocator_T::FlushOrInvalidateAllocation(
    9701  VmaAllocation hAllocation,
    9702  VkDeviceSize offset, VkDeviceSize size,
    9703  VMA_CACHE_OPERATION op)
    9704 {
    9705  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
    9706  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
    9707  {
    9708  const VkDeviceSize allocationSize = hAllocation->GetSize();
    9709  VMA_ASSERT(offset <= allocationSize);
    9710 
    9711  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
    9712 
    9713  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
    9714  memRange.memory = hAllocation->GetMemory();
    9715 
    9716  switch(hAllocation->GetType())
    9717  {
    9718  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9719  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9720  if(size == VK_WHOLE_SIZE)
    9721  {
    9722  memRange.size = allocationSize - memRange.offset;
    9723  }
    9724  else
    9725  {
    9726  VMA_ASSERT(offset + size <= allocationSize);
    9727  memRange.size = VMA_MIN(
    9728  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
    9729  allocationSize - memRange.offset);
    9730  }
    9731  break;
    9732 
    9733  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9734  {
    9735  // 1. Still within this allocation.
    9736  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9737  if(size == VK_WHOLE_SIZE)
    9738  {
    9739  size = allocationSize - offset;
    9740  }
    9741  else
    9742  {
    9743  VMA_ASSERT(offset + size <= allocationSize);
    9744  }
    9745  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
    9746 
    9747  // 2. Adjust to whole block.
    9748  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
    9749  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
    9750  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize();
    9751  memRange.offset += allocationOffset;
    9752  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
    9753 
    9754  break;
    9755  }
    9756 
    9757  default:
    9758  VMA_ASSERT(0);
    9759  }
    9760 
    9761  switch(op)
    9762  {
    9763  case VMA_CACHE_FLUSH:
    9764  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9765  break;
    9766  case VMA_CACHE_INVALIDATE:
    9767  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9768  break;
    9769  default:
    9770  VMA_ASSERT(0);
    9771  }
    9772  }
    9773  // else: Just ignore this call.
    9774 }
    9775 
    9776 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
    9777 {
    9778  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
    9779 
    9780  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9781  {
    9782  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9783  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    9784  VMA_ASSERT(pDedicatedAllocations);
    9785  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
    9786  VMA_ASSERT(success);
    9787  }
    9788 
    9789  VkDeviceMemory hMemory = allocation->GetMemory();
    9790 
    9791  if(allocation->GetMappedData() != VMA_NULL)
    9792  {
    9793  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
    9794  }
    9795 
    9796  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
    9797 
    9798  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
    9799 }
    9800 
    9801 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
    9802 {
    9803  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
    9804  !hAllocation->CanBecomeLost() &&
    9805  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9806  {
    9807  void* pData = VMA_NULL;
    9808  VkResult res = Map(hAllocation, &pData);
    9809  if(res == VK_SUCCESS)
    9810  {
    9811  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
    9812  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
    9813  Unmap(hAllocation);
    9814  }
    9815  else
    9816  {
    9817  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
    9818  }
    9819  }
    9820 }
    9821 
    9822 #if VMA_STATS_STRING_ENABLED
    9823 
    9824 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
    9825 {
    9826  bool dedicatedAllocationsStarted = false;
    9827  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9828  {
    9829  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9830  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9831  VMA_ASSERT(pDedicatedAllocVector);
    9832  if(pDedicatedAllocVector->empty() == false)
    9833  {
    9834  if(dedicatedAllocationsStarted == false)
    9835  {
    9836  dedicatedAllocationsStarted = true;
    9837  json.WriteString("DedicatedAllocations");
    9838  json.BeginObject();
    9839  }
    9840 
    9841  json.BeginString("Type ");
    9842  json.ContinueString(memTypeIndex);
    9843  json.EndString();
    9844 
    9845  json.BeginArray();
    9846 
    9847  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
    9848  {
    9849  json.BeginObject(true);
    9850  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
    9851  hAlloc->PrintParameters(json);
    9852  json.EndObject();
    9853  }
    9854 
    9855  json.EndArray();
    9856  }
    9857  }
    9858  if(dedicatedAllocationsStarted)
    9859  {
    9860  json.EndObject();
    9861  }
    9862 
    9863  {
    9864  bool allocationsStarted = false;
    9865  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9866  {
    9867  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
    9868  {
    9869  if(allocationsStarted == false)
    9870  {
    9871  allocationsStarted = true;
    9872  json.WriteString("DefaultPools");
    9873  json.BeginObject();
    9874  }
    9875 
    9876  json.BeginString("Type ");
    9877  json.ContinueString(memTypeIndex);
    9878  json.EndString();
    9879 
    9880  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
    9881  }
    9882  }
    9883  if(allocationsStarted)
    9884  {
    9885  json.EndObject();
    9886  }
    9887  }
    9888 
    9889  {
    9890  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9891  const size_t poolCount = m_Pools.size();
    9892  if(poolCount > 0)
    9893  {
    9894  json.WriteString("Pools");
    9895  json.BeginObject();
    9896  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
    9897  {
    9898  json.BeginString();
    9899  json.ContinueString(m_Pools[poolIndex]->GetId());
    9900  json.EndString();
    9901 
    9902  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
    9903  }
    9904  json.EndObject();
    9905  }
    9906  }
    9907 }
    9908 
    9909 #endif // #if VMA_STATS_STRING_ENABLED
    9910 
    9912 // Public interface
    9913 
    9914 VkResult vmaCreateAllocator(
    9915  const VmaAllocatorCreateInfo* pCreateInfo,
    9916  VmaAllocator* pAllocator)
    9917 {
    9918  VMA_ASSERT(pCreateInfo && pAllocator);
    9919  VMA_DEBUG_LOG("vmaCreateAllocator");
    9920  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
    9921  return (*pAllocator)->Init(pCreateInfo);
    9922 }
    9923 
    9924 void vmaDestroyAllocator(
    9925  VmaAllocator allocator)
    9926 {
    9927  if(allocator != VK_NULL_HANDLE)
    9928  {
    9929  VMA_DEBUG_LOG("vmaDestroyAllocator");
    9930  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
    9931  vma_delete(&allocationCallbacks, allocator);
    9932  }
    9933 }
    9934 
    9935 void vmaGetPhysicalDeviceProperties(
    9936  VmaAllocator allocator,
    9937  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
    9938 {
    9939  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
    9940  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
    9941 }
    9942 
    9943 void vmaGetMemoryProperties(
    9944  VmaAllocator allocator,
    9945  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
    9946 {
    9947  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
    9948  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
    9949 }
    9950 
    9951 void vmaGetMemoryTypeProperties(
    9952  VmaAllocator allocator,
    9953  uint32_t memoryTypeIndex,
    9954  VkMemoryPropertyFlags* pFlags)
    9955 {
    9956  VMA_ASSERT(allocator && pFlags);
    9957  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
    9958  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
    9959 }
    9960 
    9961 void vmaSetCurrentFrameIndex(
    9962  VmaAllocator allocator,
    9963  uint32_t frameIndex)
    9964 {
    9965  VMA_ASSERT(allocator);
    9966  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
    9967 
    9968  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9969 
    9970  allocator->SetCurrentFrameIndex(frameIndex);
    9971 }
    9972 
    9973 void vmaCalculateStats(
    9974  VmaAllocator allocator,
    9975  VmaStats* pStats)
    9976 {
    9977  VMA_ASSERT(allocator && pStats);
    9978  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9979  allocator->CalculateStats(pStats);
    9980 }
    9981 
    9982 #if VMA_STATS_STRING_ENABLED
    9983 
    9984 void vmaBuildStatsString(
    9985  VmaAllocator allocator,
    9986  char** ppStatsString,
    9987  VkBool32 detailedMap)
    9988 {
    9989  VMA_ASSERT(allocator && ppStatsString);
    9990  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9991 
    9992  VmaStringBuilder sb(allocator);
    9993  {
    9994  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
    9995  json.BeginObject();
    9996 
    9997  VmaStats stats;
    9998  allocator->CalculateStats(&stats);
    9999 
    10000  json.WriteString("Total");
    10001  VmaPrintStatInfo(json, stats.total);
    10002 
    10003  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
    10004  {
    10005  json.BeginString("Heap ");
    10006  json.ContinueString(heapIndex);
    10007  json.EndString();
    10008  json.BeginObject();
    10009 
    10010  json.WriteString("Size");
    10011  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
    10012 
    10013  json.WriteString("Flags");
    10014  json.BeginArray(true);
    10015  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
    10016  {
    10017  json.WriteString("DEVICE_LOCAL");
    10018  }
    10019  json.EndArray();
    10020 
    10021  if(stats.memoryHeap[heapIndex].blockCount > 0)
    10022  {
    10023  json.WriteString("Stats");
    10024  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
    10025  }
    10026 
    10027  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
    10028  {
    10029  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
    10030  {
    10031  json.BeginString("Type ");
    10032  json.ContinueString(typeIndex);
    10033  json.EndString();
    10034 
    10035  json.BeginObject();
    10036 
    10037  json.WriteString("Flags");
    10038  json.BeginArray(true);
    10039  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
    10040  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
    10041  {
    10042  json.WriteString("DEVICE_LOCAL");
    10043  }
    10044  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    10045  {
    10046  json.WriteString("HOST_VISIBLE");
    10047  }
    10048  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
    10049  {
    10050  json.WriteString("HOST_COHERENT");
    10051  }
    10052  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
    10053  {
    10054  json.WriteString("HOST_CACHED");
    10055  }
    10056  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
    10057  {
    10058  json.WriteString("LAZILY_ALLOCATED");
    10059  }
    10060  json.EndArray();
    10061 
    10062  if(stats.memoryType[typeIndex].blockCount > 0)
    10063  {
    10064  json.WriteString("Stats");
    10065  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
    10066  }
    10067 
    10068  json.EndObject();
    10069  }
    10070  }
    10071 
    10072  json.EndObject();
    10073  }
    10074  if(detailedMap == VK_TRUE)
    10075  {
    10076  allocator->PrintDetailedMap(json);
    10077  }
    10078 
    10079  json.EndObject();
    10080  }
    10081 
    10082  const size_t len = sb.GetLength();
    10083  char* const pChars = vma_new_array(allocator, char, len + 1);
    10084  if(len > 0)
    10085  {
    10086  memcpy(pChars, sb.GetData(), len);
    10087  }
    10088  pChars[len] = '\0';
    10089  *ppStatsString = pChars;
    10090 }
    10091 
    10092 void vmaFreeStatsString(
    10093  VmaAllocator allocator,
    10094  char* pStatsString)
    10095 {
    10096  if(pStatsString != VMA_NULL)
    10097  {
    10098  VMA_ASSERT(allocator);
    10099  size_t len = strlen(pStatsString);
    10100  vma_delete_array(allocator, pStatsString, len + 1);
    10101  }
    10102 }
    10103 
    10104 #endif // #if VMA_STATS_STRING_ENABLED
    10105 
    10106 /*
    10107 This function is not protected by any mutex because it just reads immutable data.
    10108 */
    10109 VkResult vmaFindMemoryTypeIndex(
    10110  VmaAllocator allocator,
    10111  uint32_t memoryTypeBits,
    10112  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10113  uint32_t* pMemoryTypeIndex)
    10114 {
    10115  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10116  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10117  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10118 
    10119  if(pAllocationCreateInfo->memoryTypeBits != 0)
    10120  {
    10121  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
    10122  }
    10123 
    10124  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
    10125  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
    10126 
    10127  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    10128  if(mapped)
    10129  {
    10130  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10131  }
    10132 
    10133  // Convert usage to requiredFlags and preferredFlags.
    10134  switch(pAllocationCreateInfo->usage)
    10135  {
    10136  case VMA_MEMORY_USAGE_UNKNOWN:
    10137  break;
    10138  case VMA_MEMORY_USAGE_GPU_ONLY:
    10139  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10140  {
    10141  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10142  }
    10143  break;
    10144  case VMA_MEMORY_USAGE_CPU_ONLY:
    10145  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    10146  break;
    10147  case VMA_MEMORY_USAGE_CPU_TO_GPU:
    10148  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10149  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10150  {
    10151  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10152  }
    10153  break;
    10154  case VMA_MEMORY_USAGE_GPU_TO_CPU:
    10155  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10156  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
    10157  break;
    10158  default:
    10159  break;
    10160  }
    10161 
    10162  *pMemoryTypeIndex = UINT32_MAX;
    10163  uint32_t minCost = UINT32_MAX;
    10164  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
    10165  memTypeIndex < allocator->GetMemoryTypeCount();
    10166  ++memTypeIndex, memTypeBit <<= 1)
    10167  {
    10168  // This memory type is acceptable according to memoryTypeBits bitmask.
    10169  if((memTypeBit & memoryTypeBits) != 0)
    10170  {
    10171  const VkMemoryPropertyFlags currFlags =
    10172  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
    10173  // This memory type contains requiredFlags.
    10174  if((requiredFlags & ~currFlags) == 0)
    10175  {
    10176  // Calculate cost as number of bits from preferredFlags not present in this memory type.
    10177  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
    10178  // Remember memory type with lowest cost.
    10179  if(currCost < minCost)
    10180  {
    10181  *pMemoryTypeIndex = memTypeIndex;
    10182  if(currCost == 0)
    10183  {
    10184  return VK_SUCCESS;
    10185  }
    10186  minCost = currCost;
    10187  }
    10188  }
    10189  }
    10190  }
    10191  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
    10192 }
    10193 
    10194 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    10195  VmaAllocator allocator,
    10196  const VkBufferCreateInfo* pBufferCreateInfo,
    10197  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10198  uint32_t* pMemoryTypeIndex)
    10199 {
    10200  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10201  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
    10202  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10203  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10204 
    10205  const VkDevice hDev = allocator->m_hDevice;
    10206  VkBuffer hBuffer = VK_NULL_HANDLE;
    10207  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
    10208  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
    10209  if(res == VK_SUCCESS)
    10210  {
    10211  VkMemoryRequirements memReq = {};
    10212  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
    10213  hDev, hBuffer, &memReq);
    10214 
    10215  res = vmaFindMemoryTypeIndex(
    10216  allocator,
    10217  memReq.memoryTypeBits,
    10218  pAllocationCreateInfo,
    10219  pMemoryTypeIndex);
    10220 
    10221  allocator->GetVulkanFunctions().vkDestroyBuffer(
    10222  hDev, hBuffer, allocator->GetAllocationCallbacks());
    10223  }
    10224  return res;
    10225 }
    10226 
    10227 VkResult vmaFindMemoryTypeIndexForImageInfo(
    10228  VmaAllocator allocator,
    10229  const VkImageCreateInfo* pImageCreateInfo,
    10230  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10231  uint32_t* pMemoryTypeIndex)
    10232 {
    10233  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10234  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
    10235  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10236  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10237 
    10238  const VkDevice hDev = allocator->m_hDevice;
    10239  VkImage hImage = VK_NULL_HANDLE;
    10240  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
    10241  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
    10242  if(res == VK_SUCCESS)
    10243  {
    10244  VkMemoryRequirements memReq = {};
    10245  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
    10246  hDev, hImage, &memReq);
    10247 
    10248  res = vmaFindMemoryTypeIndex(
    10249  allocator,
    10250  memReq.memoryTypeBits,
    10251  pAllocationCreateInfo,
    10252  pMemoryTypeIndex);
    10253 
    10254  allocator->GetVulkanFunctions().vkDestroyImage(
    10255  hDev, hImage, allocator->GetAllocationCallbacks());
    10256  }
    10257  return res;
    10258 }
    10259 
    10260 VkResult vmaCreatePool(
    10261  VmaAllocator allocator,
    10262  const VmaPoolCreateInfo* pCreateInfo,
    10263  VmaPool* pPool)
    10264 {
    10265  VMA_ASSERT(allocator && pCreateInfo && pPool);
    10266 
    10267  VMA_DEBUG_LOG("vmaCreatePool");
    10268 
    10269  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10270 
    10271  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
    10272 
    10273 #if VMA_RECORDING_ENABLED
    10274  if(allocator->GetRecorder() != VMA_NULL)
    10275  {
    10276  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
    10277  }
    10278 #endif
    10279 
    10280  return res;
    10281 }
    10282 
    10283 void vmaDestroyPool(
    10284  VmaAllocator allocator,
    10285  VmaPool pool)
    10286 {
    10287  VMA_ASSERT(allocator);
    10288 
    10289  if(pool == VK_NULL_HANDLE)
    10290  {
    10291  return;
    10292  }
    10293 
    10294  VMA_DEBUG_LOG("vmaDestroyPool");
    10295 
    10296  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10297 
    10298 #if VMA_RECORDING_ENABLED
    10299  if(allocator->GetRecorder() != VMA_NULL)
    10300  {
    10301  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
    10302  }
    10303 #endif
    10304 
    10305  allocator->DestroyPool(pool);
    10306 }
    10307 
    10308 void vmaGetPoolStats(
    10309  VmaAllocator allocator,
    10310  VmaPool pool,
    10311  VmaPoolStats* pPoolStats)
    10312 {
    10313  VMA_ASSERT(allocator && pool && pPoolStats);
    10314 
    10315  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10316 
    10317  allocator->GetPoolStats(pool, pPoolStats);
    10318 }
    10319 
    10320 void vmaMakePoolAllocationsLost(
    10321  VmaAllocator allocator,
    10322  VmaPool pool,
    10323  size_t* pLostAllocationCount)
    10324 {
    10325  VMA_ASSERT(allocator && pool);
    10326 
    10327  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10328 
    10329  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
    10330 }
    10331 
    10332 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
    10333 {
    10334  VMA_ASSERT(allocator && pool);
    10335 
    10336  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10337 
    10338  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
    10339 
    10340  return allocator->CheckPoolCorruption(pool);
    10341 }
    10342 
    10343 VkResult vmaAllocateMemory(
    10344  VmaAllocator allocator,
    10345  const VkMemoryRequirements* pVkMemoryRequirements,
    10346  const VmaAllocationCreateInfo* pCreateInfo,
    10347  VmaAllocation* pAllocation,
    10348  VmaAllocationInfo* pAllocationInfo)
    10349 {
    10350  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
    10351 
    10352  VMA_DEBUG_LOG("vmaAllocateMemory");
    10353 
    10354  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10355 
    10356  VkResult result = allocator->AllocateMemory(
    10357  *pVkMemoryRequirements,
    10358  false, // requiresDedicatedAllocation
    10359  false, // prefersDedicatedAllocation
    10360  VK_NULL_HANDLE, // dedicatedBuffer
    10361  VK_NULL_HANDLE, // dedicatedImage
    10362  *pCreateInfo,
    10363  VMA_SUBALLOCATION_TYPE_UNKNOWN,
    10364  pAllocation);
    10365 
    10366 #if VMA_RECORDING_ENABLED
    10367  if(allocator->GetRecorder() != VMA_NULL)
    10368  {
    10369  allocator->GetRecorder()->RecordAllocateMemory(
    10370  allocator->GetCurrentFrameIndex(),
    10371  *pVkMemoryRequirements,
    10372  *pCreateInfo,
    10373  *pAllocation);
    10374  }
    10375 #endif
    10376 
    10377  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
    10378  {
    10379  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10380  }
    10381 
    10382  return result;
    10383 }
    10384 
    10385 VkResult vmaAllocateMemoryForBuffer(
    10386  VmaAllocator allocator,
    10387  VkBuffer buffer,
    10388  const VmaAllocationCreateInfo* pCreateInfo,
    10389  VmaAllocation* pAllocation,
    10390  VmaAllocationInfo* pAllocationInfo)
    10391 {
    10392  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10393 
    10394  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
    10395 
    10396  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10397 
    10398  VkMemoryRequirements vkMemReq = {};
    10399  bool requiresDedicatedAllocation = false;
    10400  bool prefersDedicatedAllocation = false;
    10401  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
    10402  requiresDedicatedAllocation,
    10403  prefersDedicatedAllocation);
    10404 
    10405  VkResult result = allocator->AllocateMemory(
    10406  vkMemReq,
    10407  requiresDedicatedAllocation,
    10408  prefersDedicatedAllocation,
    10409  buffer, // dedicatedBuffer
    10410  VK_NULL_HANDLE, // dedicatedImage
    10411  *pCreateInfo,
    10412  VMA_SUBALLOCATION_TYPE_BUFFER,
    10413  pAllocation);
    10414 
    10415 #if VMA_RECORDING_ENABLED
    10416  if(allocator->GetRecorder() != VMA_NULL)
    10417  {
    10418  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
    10419  allocator->GetCurrentFrameIndex(),
    10420  vkMemReq,
    10421  requiresDedicatedAllocation,
    10422  prefersDedicatedAllocation,
    10423  *pCreateInfo,
    10424  *pAllocation);
    10425  }
    10426 #endif
    10427 
    10428  if(pAllocationInfo && result == VK_SUCCESS)
    10429  {
    10430  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10431  }
    10432 
    10433  return result;
    10434 }
    10435 
    10436 VkResult vmaAllocateMemoryForImage(
    10437  VmaAllocator allocator,
    10438  VkImage image,
    10439  const VmaAllocationCreateInfo* pCreateInfo,
    10440  VmaAllocation* pAllocation,
    10441  VmaAllocationInfo* pAllocationInfo)
    10442 {
    10443  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10444 
    10445  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
    10446 
    10447  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10448 
    10449  VkMemoryRequirements vkMemReq = {};
    10450  bool requiresDedicatedAllocation = false;
    10451  bool prefersDedicatedAllocation = false;
    10452  allocator->GetImageMemoryRequirements(image, vkMemReq,
    10453  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10454 
    10455  VkResult result = allocator->AllocateMemory(
    10456  vkMemReq,
    10457  requiresDedicatedAllocation,
    10458  prefersDedicatedAllocation,
    10459  VK_NULL_HANDLE, // dedicatedBuffer
    10460  image, // dedicatedImage
    10461  *pCreateInfo,
    10462  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
    10463  pAllocation);
    10464 
    10465 #if VMA_RECORDING_ENABLED
    10466  if(allocator->GetRecorder() != VMA_NULL)
    10467  {
    10468  allocator->GetRecorder()->RecordAllocateMemoryForImage(
    10469  allocator->GetCurrentFrameIndex(),
    10470  vkMemReq,
    10471  requiresDedicatedAllocation,
    10472  prefersDedicatedAllocation,
    10473  *pCreateInfo,
    10474  *pAllocation);
    10475  }
    10476 #endif
    10477 
    10478  if(pAllocationInfo && result == VK_SUCCESS)
    10479  {
    10480  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10481  }
    10482 
    10483  return result;
    10484 }
    10485 
    10486 void vmaFreeMemory(
    10487  VmaAllocator allocator,
    10488  VmaAllocation allocation)
    10489 {
    10490  VMA_ASSERT(allocator);
    10491 
    10492  if(allocation == VK_NULL_HANDLE)
    10493  {
    10494  return;
    10495  }
    10496 
    10497  VMA_DEBUG_LOG("vmaFreeMemory");
    10498 
    10499  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10500 
    10501 #if VMA_RECORDING_ENABLED
    10502  if(allocator->GetRecorder() != VMA_NULL)
    10503  {
    10504  allocator->GetRecorder()->RecordFreeMemory(
    10505  allocator->GetCurrentFrameIndex(),
    10506  allocation);
    10507  }
    10508 #endif
    10509 
    10510  allocator->FreeMemory(allocation);
    10511 }
    10512 
    10513 void vmaGetAllocationInfo(
    10514  VmaAllocator allocator,
    10515  VmaAllocation allocation,
    10516  VmaAllocationInfo* pAllocationInfo)
    10517 {
    10518  VMA_ASSERT(allocator && allocation && pAllocationInfo);
    10519 
    10520  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10521 
    10522  allocator->GetAllocationInfo(allocation, pAllocationInfo);
    10523 }
    10524 
    10525 VkBool32 vmaTouchAllocation(
    10526  VmaAllocator allocator,
    10527  VmaAllocation allocation)
    10528 {
    10529  VMA_ASSERT(allocator && allocation);
    10530 
    10531  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10532 
    10533  return allocator->TouchAllocation(allocation);
    10534 }
    10535 
    10536 void vmaSetAllocationUserData(
    10537  VmaAllocator allocator,
    10538  VmaAllocation allocation,
    10539  void* pUserData)
    10540 {
    10541  VMA_ASSERT(allocator && allocation);
    10542 
    10543  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10544 
    10545  allocation->SetUserData(allocator, pUserData);
    10546 
    10547 #if VMA_RECORDING_ENABLED
    10548  if(allocator->GetRecorder() != VMA_NULL)
    10549  {
    10550  allocator->GetRecorder()->RecordSetAllocationUserData(
    10551  allocator->GetCurrentFrameIndex(),
    10552  allocation,
    10553  pUserData);
    10554  }
    10555 #endif
    10556 }
    10557 
    10558 void vmaCreateLostAllocation(
    10559  VmaAllocator allocator,
    10560  VmaAllocation* pAllocation)
    10561 {
    10562  VMA_ASSERT(allocator && pAllocation);
    10563 
    10564  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
    10565 
    10566  allocator->CreateLostAllocation(pAllocation);
    10567 
    10568 #if VMA_RECORDING_ENABLED
    10569  if(allocator->GetRecorder() != VMA_NULL)
    10570  {
    10571  allocator->GetRecorder()->RecordCreateLostAllocation(
    10572  allocator->GetCurrentFrameIndex(),
    10573  *pAllocation);
    10574  }
    10575 #endif
    10576 }
    10577 
    10578 VkResult vmaMapMemory(
    10579  VmaAllocator allocator,
    10580  VmaAllocation allocation,
    10581  void** ppData)
    10582 {
    10583  VMA_ASSERT(allocator && allocation && ppData);
    10584 
    10585  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10586 
    10587  VkResult res = allocator->Map(allocation, ppData);
    10588 
    10589 #if VMA_RECORDING_ENABLED
    10590  if(allocator->GetRecorder() != VMA_NULL)
    10591  {
    10592  allocator->GetRecorder()->RecordMapMemory(
    10593  allocator->GetCurrentFrameIndex(),
    10594  allocation);
    10595  }
    10596 #endif
    10597 
    10598  return res;
    10599 }
    10600 
    10601 void vmaUnmapMemory(
    10602  VmaAllocator allocator,
    10603  VmaAllocation allocation)
    10604 {
    10605  VMA_ASSERT(allocator && allocation);
    10606 
    10607  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10608 
    10609 #if VMA_RECORDING_ENABLED
    10610  if(allocator->GetRecorder() != VMA_NULL)
    10611  {
    10612  allocator->GetRecorder()->RecordUnmapMemory(
    10613  allocator->GetCurrentFrameIndex(),
    10614  allocation);
    10615  }
    10616 #endif
    10617 
    10618  allocator->Unmap(allocation);
    10619 }
    10620 
    10621 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10622 {
    10623  VMA_ASSERT(allocator && allocation);
    10624 
    10625  VMA_DEBUG_LOG("vmaFlushAllocation");
    10626 
    10627  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10628 
    10629  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
    10630 
    10631 #if VMA_RECORDING_ENABLED
    10632  if(allocator->GetRecorder() != VMA_NULL)
    10633  {
    10634  allocator->GetRecorder()->RecordFlushAllocation(
    10635  allocator->GetCurrentFrameIndex(),
    10636  allocation, offset, size);
    10637  }
    10638 #endif
    10639 }
    10640 
    10641 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10642 {
    10643  VMA_ASSERT(allocator && allocation);
    10644 
    10645  VMA_DEBUG_LOG("vmaInvalidateAllocation");
    10646 
    10647  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10648 
    10649  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
    10650 
    10651 #if VMA_RECORDING_ENABLED
    10652  if(allocator->GetRecorder() != VMA_NULL)
    10653  {
    10654  allocator->GetRecorder()->RecordInvalidateAllocation(
    10655  allocator->GetCurrentFrameIndex(),
    10656  allocation, offset, size);
    10657  }
    10658 #endif
    10659 }
    10660 
    10661 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
    10662 {
    10663  VMA_ASSERT(allocator);
    10664 
    10665  VMA_DEBUG_LOG("vmaCheckCorruption");
    10666 
    10667  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10668 
    10669  return allocator->CheckCorruption(memoryTypeBits);
    10670 }
    10671 
    10672 VkResult vmaDefragment(
    10673  VmaAllocator allocator,
    10674  VmaAllocation* pAllocations,
    10675  size_t allocationCount,
    10676  VkBool32* pAllocationsChanged,
    10677  const VmaDefragmentationInfo *pDefragmentationInfo,
    10678  VmaDefragmentationStats* pDefragmentationStats)
    10679 {
    10680  VMA_ASSERT(allocator && pAllocations);
    10681 
    10682  VMA_DEBUG_LOG("vmaDefragment");
    10683 
    10684  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10685 
    10686  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
    10687 }
    10688 
    10689 VkResult vmaBindBufferMemory(
    10690  VmaAllocator allocator,
    10691  VmaAllocation allocation,
    10692  VkBuffer buffer)
    10693 {
    10694  VMA_ASSERT(allocator && allocation && buffer);
    10695 
    10696  VMA_DEBUG_LOG("vmaBindBufferMemory");
    10697 
    10698  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10699 
    10700  return allocator->BindBufferMemory(allocation, buffer);
    10701 }
    10702 
    10703 VkResult vmaBindImageMemory(
    10704  VmaAllocator allocator,
    10705  VmaAllocation allocation,
    10706  VkImage image)
    10707 {
    10708  VMA_ASSERT(allocator && allocation && image);
    10709 
    10710  VMA_DEBUG_LOG("vmaBindImageMemory");
    10711 
    10712  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10713 
    10714  return allocator->BindImageMemory(allocation, image);
    10715 }
    10716 
    10717 VkResult vmaCreateBuffer(
    10718  VmaAllocator allocator,
    10719  const VkBufferCreateInfo* pBufferCreateInfo,
    10720  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10721  VkBuffer* pBuffer,
    10722  VmaAllocation* pAllocation,
    10723  VmaAllocationInfo* pAllocationInfo)
    10724 {
    10725  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
    10726 
    10727  VMA_DEBUG_LOG("vmaCreateBuffer");
    10728 
    10729  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10730 
    10731  *pBuffer = VK_NULL_HANDLE;
    10732  *pAllocation = VK_NULL_HANDLE;
    10733 
    10734  // 1. Create VkBuffer.
    10735  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
    10736  allocator->m_hDevice,
    10737  pBufferCreateInfo,
    10738  allocator->GetAllocationCallbacks(),
    10739  pBuffer);
    10740  if(res >= 0)
    10741  {
    10742  // 2. vkGetBufferMemoryRequirements.
    10743  VkMemoryRequirements vkMemReq = {};
    10744  bool requiresDedicatedAllocation = false;
    10745  bool prefersDedicatedAllocation = false;
    10746  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
    10747  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10748 
    10749  // Make sure alignment requirements for specific buffer usages reported
    10750  // in Physical Device Properties are included in alignment reported by memory requirements.
    10751  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
    10752  {
    10753  VMA_ASSERT(vkMemReq.alignment %
    10754  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
    10755  }
    10756  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
    10757  {
    10758  VMA_ASSERT(vkMemReq.alignment %
    10759  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
    10760  }
    10761  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
    10762  {
    10763  VMA_ASSERT(vkMemReq.alignment %
    10764  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
    10765  }
    10766 
    10767  // 3. Allocate memory using allocator.
    10768  res = allocator->AllocateMemory(
    10769  vkMemReq,
    10770  requiresDedicatedAllocation,
    10771  prefersDedicatedAllocation,
    10772  *pBuffer, // dedicatedBuffer
    10773  VK_NULL_HANDLE, // dedicatedImage
    10774  *pAllocationCreateInfo,
    10775  VMA_SUBALLOCATION_TYPE_BUFFER,
    10776  pAllocation);
    10777 
    10778 #if VMA_RECORDING_ENABLED
    10779  if(allocator->GetRecorder() != VMA_NULL)
    10780  {
    10781  allocator->GetRecorder()->RecordCreateBuffer(
    10782  allocator->GetCurrentFrameIndex(),
    10783  *pBufferCreateInfo,
    10784  *pAllocationCreateInfo,
    10785  *pAllocation);
    10786  }
    10787 #endif
    10788 
    10789  if(res >= 0)
    10790  {
    10791  // 3. Bind buffer with memory.
    10792  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
    10793  if(res >= 0)
    10794  {
    10795  // All steps succeeded.
    10796  #if VMA_STATS_STRING_ENABLED
    10797  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
    10798  #endif
    10799  if(pAllocationInfo != VMA_NULL)
    10800  {
    10801  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10802  }
    10803 
    10804  return VK_SUCCESS;
    10805  }
    10806  allocator->FreeMemory(*pAllocation);
    10807  *pAllocation = VK_NULL_HANDLE;
    10808  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10809  *pBuffer = VK_NULL_HANDLE;
    10810  return res;
    10811  }
    10812  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10813  *pBuffer = VK_NULL_HANDLE;
    10814  return res;
    10815  }
    10816  return res;
    10817 }
    10818 
    10819 void vmaDestroyBuffer(
    10820  VmaAllocator allocator,
    10821  VkBuffer buffer,
    10822  VmaAllocation allocation)
    10823 {
    10824  VMA_ASSERT(allocator);
    10825 
    10826  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    10827  {
    10828  return;
    10829  }
    10830 
    10831  VMA_DEBUG_LOG("vmaDestroyBuffer");
    10832 
    10833  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10834 
    10835 #if VMA_RECORDING_ENABLED
    10836  if(allocator->GetRecorder() != VMA_NULL)
    10837  {
    10838  allocator->GetRecorder()->RecordDestroyBuffer(
    10839  allocator->GetCurrentFrameIndex(),
    10840  allocation);
    10841  }
    10842 #endif
    10843 
    10844  if(buffer != VK_NULL_HANDLE)
    10845  {
    10846  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
    10847  }
    10848 
    10849  if(allocation != VK_NULL_HANDLE)
    10850  {
    10851  allocator->FreeMemory(allocation);
    10852  }
    10853 }
    10854 
    10855 VkResult vmaCreateImage(
    10856  VmaAllocator allocator,
    10857  const VkImageCreateInfo* pImageCreateInfo,
    10858  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10859  VkImage* pImage,
    10860  VmaAllocation* pAllocation,
    10861  VmaAllocationInfo* pAllocationInfo)
    10862 {
    10863  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
    10864 
    10865  VMA_DEBUG_LOG("vmaCreateImage");
    10866 
    10867  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10868 
    10869  *pImage = VK_NULL_HANDLE;
    10870  *pAllocation = VK_NULL_HANDLE;
    10871 
    10872  // 1. Create VkImage.
    10873  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
    10874  allocator->m_hDevice,
    10875  pImageCreateInfo,
    10876  allocator->GetAllocationCallbacks(),
    10877  pImage);
    10878  if(res >= 0)
    10879  {
    10880  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
    10881  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
    10882  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
    10883 
    10884  // 2. Allocate memory using allocator.
    10885  VkMemoryRequirements vkMemReq = {};
    10886  bool requiresDedicatedAllocation = false;
    10887  bool prefersDedicatedAllocation = false;
    10888  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
    10889  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10890 
    10891  res = allocator->AllocateMemory(
    10892  vkMemReq,
    10893  requiresDedicatedAllocation,
    10894  prefersDedicatedAllocation,
    10895  VK_NULL_HANDLE, // dedicatedBuffer
    10896  *pImage, // dedicatedImage
    10897  *pAllocationCreateInfo,
    10898  suballocType,
    10899  pAllocation);
    10900 
    10901 #if VMA_RECORDING_ENABLED
    10902  if(allocator->GetRecorder() != VMA_NULL)
    10903  {
    10904  allocator->GetRecorder()->RecordCreateImage(
    10905  allocator->GetCurrentFrameIndex(),
    10906  *pImageCreateInfo,
    10907  *pAllocationCreateInfo,
    10908  *pAllocation);
    10909  }
    10910 #endif
    10911 
    10912  if(res >= 0)
    10913  {
    10914  // 3. Bind image with memory.
    10915  res = allocator->BindImageMemory(*pAllocation, *pImage);
    10916  if(res >= 0)
    10917  {
    10918  // All steps succeeded.
    10919  #if VMA_STATS_STRING_ENABLED
    10920  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
    10921  #endif
    10922  if(pAllocationInfo != VMA_NULL)
    10923  {
    10924  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10925  }
    10926 
    10927  return VK_SUCCESS;
    10928  }
    10929  allocator->FreeMemory(*pAllocation);
    10930  *pAllocation = VK_NULL_HANDLE;
    10931  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    10932  *pImage = VK_NULL_HANDLE;
    10933  return res;
    10934  }
    10935  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    10936  *pImage = VK_NULL_HANDLE;
    10937  return res;
    10938  }
    10939  return res;
    10940 }
    10941 
    10942 void vmaDestroyImage(
    10943  VmaAllocator allocator,
    10944  VkImage image,
    10945  VmaAllocation allocation)
    10946 {
    10947  VMA_ASSERT(allocator);
    10948 
    10949  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    10950  {
    10951  return;
    10952  }
    10953 
    10954  VMA_DEBUG_LOG("vmaDestroyImage");
    10955 
    10956  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10957 
    10958 #if VMA_RECORDING_ENABLED
    10959  if(allocator->GetRecorder() != VMA_NULL)
    10960  {
    10961  allocator->GetRecorder()->RecordDestroyImage(
    10962  allocator->GetCurrentFrameIndex(),
    10963  allocation);
    10964  }
    10965 #endif
    10966 
    10967  if(image != VK_NULL_HANDLE)
    10968  {
    10969  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
    10970  }
    10971  if(allocation != VK_NULL_HANDLE)
    10972  {
    10973  allocator->FreeMemory(allocation);
    10974  }
    10975 }
    10976 
    10977 #endif // #ifdef VMA_IMPLEMENTATION
    VmaVulkanFunctions::vkGetPhysicalDeviceProperties
    PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
    Definition: vk_mem_alloc.h:1298
    +
    VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
    Set this flag if the allocation should have its own memory block.
    Definition: vk_mem_alloc.h:1611
    vmaUnmapMemory
    void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
    Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
    -
    VmaAllocatorCreateInfo::physicalDevice
    VkPhysicalDevice physicalDevice
    Vulkan physical device.
    Definition: vk_mem_alloc.h:1300
    +
    VmaAllocatorCreateInfo::physicalDevice
    VkPhysicalDevice physicalDevice
    Vulkan physical device.
    Definition: vk_mem_alloc.h:1367
    vmaDefragment
    VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
    Compacts memory by moving allocations.
    vmaInvalidateAllocation
    void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    Invalidates memory of given allocation.
    VmaAllocation
    Represents single memory allocation.
    -
    VmaVulkanFunctions::vkCreateBuffer
    PFN_vkCreateBuffer vkCreateBuffer
    Definition: vk_mem_alloc.h:1283
    +
    VMA_RECORD_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1329
    +
    VmaVulkanFunctions::vkCreateBuffer
    PFN_vkCreateBuffer vkCreateBuffer
    Definition: vk_mem_alloc.h:1310
    vmaFreeStatsString
    void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
    VmaStats
    struct VmaStats VmaStats
    General statistics from current state of Allocator.
    -
    VMA_MEMORY_USAGE_GPU_ONLY
    Definition: vk_mem_alloc.h:1494
    -
    VmaVulkanFunctions::vkMapMemory
    PFN_vkMapMemory vkMapMemory
    Definition: vk_mem_alloc.h:1275
    -
    VmaAllocationInfo::deviceMemory
    VkDeviceMemory deviceMemory
    Handle to Vulkan memory object.
    Definition: vk_mem_alloc.h:1883
    -
    VmaAllocatorCreateInfo::flags
    VmaAllocatorCreateFlags flags
    Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
    Definition: vk_mem_alloc.h:1297
    -
    VmaDefragmentationInfo::maxAllocationsToMove
    uint32_t maxAllocationsToMove
    Maximum number of allocations that can be moved to different place.
    Definition: vk_mem_alloc.h:2128
    -
    VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
    Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
    Definition: vk_mem_alloc.h:1713
    +
    VMA_MEMORY_USAGE_GPU_ONLY
    Definition: vk_mem_alloc.h:1568
    +
    VmaVulkanFunctions::vkMapMemory
    PFN_vkMapMemory vkMapMemory
    Definition: vk_mem_alloc.h:1302
    +
    VmaAllocationInfo::deviceMemory
    VkDeviceMemory deviceMemory
    Handle to Vulkan memory object.
    Definition: vk_mem_alloc.h:1957
    +
    VmaAllocatorCreateInfo::flags
    VmaAllocatorCreateFlags flags
    Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
    Definition: vk_mem_alloc.h:1364
    +
    VmaDefragmentationInfo::maxAllocationsToMove
    uint32_t maxAllocationsToMove
    Maximum number of allocations that can be moved to different place.
    Definition: vk_mem_alloc.h:2202
    +
    VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
    Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
    Definition: vk_mem_alloc.h:1787
    +
    VMA_RECORDING_ENABLED
    #define VMA_RECORDING_ENABLED
    Definition: vk_mem_alloc.h:1341
    vmaMakePoolAllocationsLost
    void vmaMakePoolAllocationsLost(VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
    Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInf...
    -
    VmaPoolStats::size
    VkDeviceSize size
    Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
    Definition: vk_mem_alloc.h:1767
    -
    VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
    Definition: vk_mem_alloc.h:1574
    -
    VmaAllocatorCreateFlags
    VkFlags VmaAllocatorCreateFlags
    Definition: vk_mem_alloc.h:1264
    -
    VmaAllocationCreateInfo::preferredFlags
    VkMemoryPropertyFlags preferredFlags
    Flags that preferably should be set in a memory type chosen for an allocation.
    Definition: vk_mem_alloc.h:1612
    -
    VMA_MEMORY_USAGE_MAX_ENUM
    Definition: vk_mem_alloc.h:1521
    -
    VmaAllocatorCreateInfo::pAllocationCallbacks
    const VkAllocationCallbacks * pAllocationCallbacks
    Custom CPU memory allocation callbacks. Optional.
    Definition: vk_mem_alloc.h:1309
    +
    VmaPoolStats::size
    VkDeviceSize size
    Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
    Definition: vk_mem_alloc.h:1841
    +
    VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
    Definition: vk_mem_alloc.h:1648
    +
    VmaAllocatorCreateFlags
    VkFlags VmaAllocatorCreateFlags
    Definition: vk_mem_alloc.h:1291
    +
    VmaAllocationCreateInfo::preferredFlags
    VkMemoryPropertyFlags preferredFlags
    Flags that preferably should be set in a memory type chosen for an allocation.
    Definition: vk_mem_alloc.h:1686
    +
    VMA_MEMORY_USAGE_MAX_ENUM
    Definition: vk_mem_alloc.h:1595
    +
    VmaAllocatorCreateInfo::pAllocationCallbacks
    const VkAllocationCallbacks * pAllocationCallbacks
    Custom CPU memory allocation callbacks. Optional.
    Definition: vk_mem_alloc.h:1376
    vmaCalculateStats
    void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
    Retrieves statistics from current state of the Allocator.
    -
    VmaAllocatorCreateInfo::pVulkanFunctions
    const VmaVulkanFunctions * pVulkanFunctions
    Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
    Definition: vk_mem_alloc.h:1362
    -
    VmaAllocatorCreateInfo
    Description of a Allocator to be created.
    Definition: vk_mem_alloc.h:1294
    +
    VmaAllocatorCreateInfo::pVulkanFunctions
    const VmaVulkanFunctions * pVulkanFunctions
    Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
    Definition: vk_mem_alloc.h:1429
    +
    VmaAllocatorCreateInfo
    Description of a Allocator to be created.
    Definition: vk_mem_alloc.h:1361
    vmaDestroyAllocator
    void vmaDestroyAllocator(VmaAllocator allocator)
    Destroys allocator object.
    -
    VmaAllocationCreateFlagBits
    VmaAllocationCreateFlagBits
    Flags to be passed as VmaAllocationCreateInfo::flags.
    Definition: vk_mem_alloc.h:1525
    +
    VmaAllocationCreateFlagBits
    VmaAllocationCreateFlagBits
    Flags to be passed as VmaAllocationCreateInfo::flags.
    Definition: vk_mem_alloc.h:1599
    vmaGetAllocationInfo
    void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
    Returns current information about specified allocation and atomically marks it as used in current fra...
    -
    VmaStatInfo::allocationSizeMax
    VkDeviceSize allocationSizeMax
    Definition: vk_mem_alloc.h:1427
    -
    VmaVulkanFunctions::vkBindImageMemory
    PFN_vkBindImageMemory vkBindImageMemory
    Definition: vk_mem_alloc.h:1280
    -
    VmaStatInfo::unusedBytes
    VkDeviceSize unusedBytes
    Total number of bytes occupied by unused ranges.
    Definition: vk_mem_alloc.h:1426
    -
    VmaDefragmentationStats
    Statistics returned by function vmaDefragment().
    Definition: vk_mem_alloc.h:2132
    +
    VmaStatInfo::allocationSizeMax
    VkDeviceSize allocationSizeMax
    Definition: vk_mem_alloc.h:1501
    +
    VmaVulkanFunctions::vkBindImageMemory
    PFN_vkBindImageMemory vkBindImageMemory
    Definition: vk_mem_alloc.h:1307
    +
    VmaStatInfo::unusedBytes
    VkDeviceSize unusedBytes
    Total number of bytes occupied by unused ranges.
    Definition: vk_mem_alloc.h:1500
    +
    VmaDefragmentationStats
    Statistics returned by function vmaDefragment().
    Definition: vk_mem_alloc.h:2206
    vmaFreeMemory
    void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
    Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
    -
    VmaAllocatorCreateInfo::frameInUseCount
    uint32_t frameInUseCount
    Maximum number of additional frames that are in use at the same time as current frame.
    Definition: vk_mem_alloc.h:1326
    -
    VmaStats::total
    VmaStatInfo total
    Definition: vk_mem_alloc.h:1436
    -
    VmaDefragmentationStats::deviceMemoryBlocksFreed
    uint32_t deviceMemoryBlocksFreed
    Number of empty VkDeviceMemory objects that have been released to the system.
    Definition: vk_mem_alloc.h:2140
    -
    VmaAllocationCreateInfo::flags
    VmaAllocationCreateFlags flags
    Use VmaAllocationCreateFlagBits enum.
    Definition: vk_mem_alloc.h:1596
    -
    VmaDefragmentationInfo::maxBytesToMove
    VkDeviceSize maxBytesToMove
    Maximum total numbers of bytes that can be copied while moving allocations to different places...
    Definition: vk_mem_alloc.h:2123
    -
    VmaVulkanFunctions::vkGetBufferMemoryRequirements
    PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
    Definition: vk_mem_alloc.h:1281
    -
    PFN_vmaAllocateDeviceMemoryFunction
    void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
    Callback function called after successful vkAllocateMemory.
    Definition: vk_mem_alloc.h:1206
    +
    VmaAllocatorCreateInfo::frameInUseCount
    uint32_t frameInUseCount
    Maximum number of additional frames that are in use at the same time as current frame.
    Definition: vk_mem_alloc.h:1393
    +
    VmaStats::total
    VmaStatInfo total
    Definition: vk_mem_alloc.h:1510
    +
    VmaDefragmentationStats::deviceMemoryBlocksFreed
    uint32_t deviceMemoryBlocksFreed
    Number of empty VkDeviceMemory objects that have been released to the system.
    Definition: vk_mem_alloc.h:2214
    +
    VmaAllocationCreateInfo::flags
    VmaAllocationCreateFlags flags
    Use VmaAllocationCreateFlagBits enum.
    Definition: vk_mem_alloc.h:1670
    +
    VmaDefragmentationInfo::maxBytesToMove
    VkDeviceSize maxBytesToMove
    Maximum total numbers of bytes that can be copied while moving allocations to different places...
    Definition: vk_mem_alloc.h:2197
    +
    VmaVulkanFunctions::vkGetBufferMemoryRequirements
    PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
    Definition: vk_mem_alloc.h:1308
    +
    PFN_vmaAllocateDeviceMemoryFunction
    void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
    Callback function called after successful vkAllocateMemory.
    Definition: vk_mem_alloc.h:1233
    VmaAllocator
    Represents main object of this library initialized.
    -
    VmaAllocatorCreateInfo::device
    VkDevice device
    Vulkan device.
    Definition: vk_mem_alloc.h:1303
    +
    VmaAllocatorCreateInfo::device
    VkDevice device
    Vulkan device.
    Definition: vk_mem_alloc.h:1370
    vmaBindBufferMemory
    VkResult vmaBindBufferMemory(VmaAllocator allocator, VmaAllocation allocation, VkBuffer buffer)
    Binds buffer to allocation.
    -
    VmaPoolCreateInfo
    Describes parameter of created VmaPool.
    Definition: vk_mem_alloc.h:1721
    -
    VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1715
    -
    VmaAllocationInfo::size
    VkDeviceSize size
    Size of this allocation, in bytes.
    Definition: vk_mem_alloc.h:1893
    +
    VmaPoolCreateInfo
    Describes parameter of created VmaPool.
    Definition: vk_mem_alloc.h:1795
    +
    VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1789
    +
    VmaAllocatorCreateInfo::pRecordSettings
    const VmaRecordSettings * pRecordSettings
    Parameters for recording of VMA calls. Can be null.
    Definition: vk_mem_alloc.h:1436
    +
    VmaAllocationInfo::size
    VkDeviceSize size
    Size of this allocation, in bytes.
    Definition: vk_mem_alloc.h:1967
    vmaGetMemoryTypeProperties
    void vmaGetMemoryTypeProperties(VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
    Given Memory Type Index, returns Property Flags of this memory type.
    -
    VmaVulkanFunctions::vkUnmapMemory
    PFN_vkUnmapMemory vkUnmapMemory
    Definition: vk_mem_alloc.h:1276
    -
    VmaAllocationCreateInfo::pUserData
    void * pUserData
    Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
    Definition: vk_mem_alloc.h:1633
    -
    VmaPoolCreateInfo::minBlockCount
    size_t minBlockCount
    Minimum number of blocks to be always allocated in this pool, even if they stay empty.
    Definition: vk_mem_alloc.h:1737
    -
    VmaPoolStats::allocationCount
    size_t allocationCount
    Number of VmaAllocation objects created from this pool that were not destroyed or lost...
    Definition: vk_mem_alloc.h:1773
    +
    VmaVulkanFunctions::vkUnmapMemory
    PFN_vkUnmapMemory vkUnmapMemory
    Definition: vk_mem_alloc.h:1303
    +
    VMA_RECORD_FLUSH_AFTER_CALL_BIT
    Enables flush after recording every function call.
    Definition: vk_mem_alloc.h:1327
    +
    VmaAllocationCreateInfo::pUserData
    void * pUserData
    Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
    Definition: vk_mem_alloc.h:1707
    +
    VmaPoolCreateInfo::minBlockCount
    size_t minBlockCount
    Minimum number of blocks to be always allocated in this pool, even if they stay empty.
    Definition: vk_mem_alloc.h:1811
    +
    VmaPoolStats::allocationCount
    size_t allocationCount
    Number of VmaAllocation objects created from this pool that were not destroyed or lost...
    Definition: vk_mem_alloc.h:1847
    VmaVulkanFunctions
    struct VmaVulkanFunctions VmaVulkanFunctions
    Pointers to some Vulkan functions - a subset used by the library.
    -
    VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1262
    -
    VmaPoolCreateInfo::memoryTypeIndex
    uint32_t memoryTypeIndex
    Vulkan memory type index to allocate this pool from.
    Definition: vk_mem_alloc.h:1724
    +
    VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1289
    +
    VmaPoolCreateInfo::memoryTypeIndex
    uint32_t memoryTypeIndex
    Vulkan memory type index to allocate this pool from.
    Definition: vk_mem_alloc.h:1798
    vmaFindMemoryTypeIndex
    VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
    Helps to find memoryTypeIndex, given memoryTypeBits and VmaAllocationCreateInfo.
    -
    VmaMemoryUsage
    VmaMemoryUsage
    Definition: vk_mem_alloc.h:1472
    +
    VmaMemoryUsage
    VmaMemoryUsage
    Definition: vk_mem_alloc.h:1546
    VmaAllocationInfo
    struct VmaAllocationInfo VmaAllocationInfo
    Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
    vmaFlushAllocation
    void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    Flushes memory of given allocation.
    -
    VmaDefragmentationInfo
    Optional configuration parameters to be passed to function vmaDefragment().
    Definition: vk_mem_alloc.h:2118
    +
    VmaDefragmentationInfo
    Optional configuration parameters to be passed to function vmaDefragment().
    Definition: vk_mem_alloc.h:2192
    VmaPoolCreateInfo
    struct VmaPoolCreateInfo VmaPoolCreateInfo
    Describes parameter of created VmaPool.
    vmaDestroyPool
    void vmaDestroyPool(VmaAllocator allocator, VmaPool pool)
    Destroys VmaPool object and frees Vulkan device memory.
    -
    VmaDefragmentationStats::bytesFreed
    VkDeviceSize bytesFreed
    Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
    Definition: vk_mem_alloc.h:2136
    -
    VMA_MEMORY_USAGE_CPU_TO_GPU
    Definition: vk_mem_alloc.h:1511
    -
    VmaAllocationCreateInfo::memoryTypeBits
    uint32_t memoryTypeBits
    Bitmask containing one bit set for every memory type acceptable for this allocation.
    Definition: vk_mem_alloc.h:1620
    -
    VmaVulkanFunctions::vkBindBufferMemory
    PFN_vkBindBufferMemory vkBindBufferMemory
    Definition: vk_mem_alloc.h:1279
    +
    VmaDefragmentationStats::bytesFreed
    VkDeviceSize bytesFreed
    Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
    Definition: vk_mem_alloc.h:2210
    +
    VMA_MEMORY_USAGE_CPU_TO_GPU
    Definition: vk_mem_alloc.h:1585
    +
    VmaAllocationCreateInfo::memoryTypeBits
    uint32_t memoryTypeBits
    Bitmask containing one bit set for every memory type acceptable for this allocation.
    Definition: vk_mem_alloc.h:1694
    +
    VmaVulkanFunctions::vkBindBufferMemory
    PFN_vkBindBufferMemory vkBindBufferMemory
    Definition: vk_mem_alloc.h:1306
    VmaPool
    Represents custom memory pool.
    vmaGetPoolStats
    void vmaGetPoolStats(VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
    Retrieves statistics of existing VmaPool object.
    VmaDefragmentationInfo
    struct VmaDefragmentationInfo VmaDefragmentationInfo
    Optional configuration parameters to be passed to function vmaDefragment().
    -
    VmaStats
    General statistics from current state of Allocator.
    Definition: vk_mem_alloc.h:1432
    -
    PFN_vmaFreeDeviceMemoryFunction
    void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
    Callback function called before vkFreeMemory.
    Definition: vk_mem_alloc.h:1212
    +
    VmaStats
    General statistics from current state of Allocator.
    Definition: vk_mem_alloc.h:1506
    +
    PFN_vmaFreeDeviceMemoryFunction
    void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
    Callback function called before vkFreeMemory.
    Definition: vk_mem_alloc.h:1239
    vmaSetAllocationUserData
    void vmaSetAllocationUserData(VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
    Sets pUserData in given allocation to new value.
    vmaCreatePool
    VkResult vmaCreatePool(VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
    Allocates Vulkan device memory and creates VmaPool object.
    -
    VmaAllocatorCreateFlagBits
    VmaAllocatorCreateFlagBits
    Flags for created VmaAllocator.
    Definition: vk_mem_alloc.h:1233
    +
    VmaAllocatorCreateFlagBits
    VmaAllocatorCreateFlagBits
    Flags for created VmaAllocator.
    Definition: vk_mem_alloc.h:1260
    vmaBindImageMemory
    VkResult vmaBindImageMemory(VmaAllocator allocator, VmaAllocation allocation, VkImage image)
    Binds image to allocation.
    VmaStatInfo
    struct VmaStatInfo VmaStatInfo
    Calculated statistics of memory usage in entire allocator.
    -
    VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
    Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
    Definition: vk_mem_alloc.h:1238
    -
    VmaDefragmentationStats::allocationsMoved
    uint32_t allocationsMoved
    Number of allocations that have been moved to different places.
    Definition: vk_mem_alloc.h:2138
    +
    VmaRecordFlags
    VkFlags VmaRecordFlags
    Definition: vk_mem_alloc.h:1331
    +
    VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
    Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
    Definition: vk_mem_alloc.h:1265
    +
    VmaDefragmentationStats::allocationsMoved
    uint32_t allocationsMoved
    Number of allocations that have been moved to different places.
    Definition: vk_mem_alloc.h:2212
    vmaCreateLostAllocation
    void vmaCreateLostAllocation(VmaAllocator allocator, VmaAllocation *pAllocation)
    Creates new allocation that is in lost state from the beginning.
    -
    VmaAllocationCreateInfo::requiredFlags
    VkMemoryPropertyFlags requiredFlags
    Flags that must be set in a Memory Type chosen for an allocation.
    Definition: vk_mem_alloc.h:1607
    -
    VmaPoolStats::unusedRangeSizeMax
    VkDeviceSize unusedRangeSizeMax
    Size of the largest continuous free memory region.
    Definition: vk_mem_alloc.h:1783
    +
    VmaAllocationCreateInfo::requiredFlags
    VkMemoryPropertyFlags requiredFlags
    Flags that must be set in a Memory Type chosen for an allocation.
    Definition: vk_mem_alloc.h:1681
    +
    VmaPoolStats::unusedRangeSizeMax
    VkDeviceSize unusedRangeSizeMax
    Size of the largest continuous free memory region.
    Definition: vk_mem_alloc.h:1857
    vmaBuildStatsString
    void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
    Builds and returns statistics as string in JSON format.
    -
    VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
    PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
    Definition: vk_mem_alloc.h:1272
    -
    VmaStatInfo
    Calculated statistics of memory usage in entire allocator.
    Definition: vk_mem_alloc.h:1415
    -
    VmaPoolCreateInfo::blockSize
    VkDeviceSize blockSize
    Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
    Definition: vk_mem_alloc.h:1732
    -
    VmaDeviceMemoryCallbacks
    Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
    Definition: vk_mem_alloc.h:1225
    +
    VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
    PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
    Definition: vk_mem_alloc.h:1299
    +
    VmaStatInfo
    Calculated statistics of memory usage in entire allocator.
    Definition: vk_mem_alloc.h:1489
    +
    VmaPoolCreateInfo::blockSize
    VkDeviceSize blockSize
    Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
    Definition: vk_mem_alloc.h:1806
    +
    VmaDeviceMemoryCallbacks
    Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
    Definition: vk_mem_alloc.h:1252
    vmaCreateBuffer
    VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    -
    VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
    Definition: vk_mem_alloc.h:1581
    -
    VmaStatInfo::unusedRangeSizeMin
    VkDeviceSize unusedRangeSizeMin
    Definition: vk_mem_alloc.h:1428
    -
    VmaDeviceMemoryCallbacks::pfnFree
    PFN_vmaFreeDeviceMemoryFunction pfnFree
    Optional, can be null.
    Definition: vk_mem_alloc.h:1229
    -
    VmaPoolCreateInfo::flags
    VmaPoolCreateFlags flags
    Use combination of VmaPoolCreateFlagBits.
    Definition: vk_mem_alloc.h:1727
    -
    VMA_MEMORY_USAGE_GPU_TO_CPU
    Definition: vk_mem_alloc.h:1520
    -
    VmaVulkanFunctions::vkInvalidateMappedMemoryRanges
    PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges
    Definition: vk_mem_alloc.h:1278
    +
    VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
    Definition: vk_mem_alloc.h:1655
    +
    VmaStatInfo::unusedRangeSizeMin
    VkDeviceSize unusedRangeSizeMin
    Definition: vk_mem_alloc.h:1502
    +
    VmaDeviceMemoryCallbacks::pfnFree
    PFN_vmaFreeDeviceMemoryFunction pfnFree
    Optional, can be null.
    Definition: vk_mem_alloc.h:1256
    +
    VmaPoolCreateInfo::flags
    VmaPoolCreateFlags flags
    Use combination of VmaPoolCreateFlagBits.
    Definition: vk_mem_alloc.h:1801
    +
    VMA_MEMORY_USAGE_GPU_TO_CPU
    Definition: vk_mem_alloc.h:1594
    +
    VmaVulkanFunctions::vkInvalidateMappedMemoryRanges
    PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges
    Definition: vk_mem_alloc.h:1305
    VmaPoolStats
    struct VmaPoolStats VmaPoolStats
    Describes parameter of existing VmaPool.
    vmaCreateImage
    VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    Function similar to vmaCreateBuffer().
    -
    VmaAllocationCreateInfo::usage
    VmaMemoryUsage usage
    Intended usage of memory.
    Definition: vk_mem_alloc.h:1602
    -
    VmaAllocationCreateInfo
    Definition: vk_mem_alloc.h:1593
    +
    VmaAllocationCreateInfo::usage
    VmaMemoryUsage usage
    Intended usage of memory.
    Definition: vk_mem_alloc.h:1676
    +
    VmaAllocationCreateInfo
    Definition: vk_mem_alloc.h:1667
    vmaFindMemoryTypeIndexForImageInfo
    VkResult vmaFindMemoryTypeIndexForImageInfo(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
    Helps to find memoryTypeIndex, given VkImageCreateInfo and VmaAllocationCreateInfo.
    -
    VmaStatInfo::blockCount
    uint32_t blockCount
    Number of VkDeviceMemory Vulkan memory blocks allocated.
    Definition: vk_mem_alloc.h:1418
    -
    VmaVulkanFunctions::vkFreeMemory
    PFN_vkFreeMemory vkFreeMemory
    Definition: vk_mem_alloc.h:1274
    -
    VmaPoolCreateInfo::maxBlockCount
    size_t maxBlockCount
    Maximum number of blocks that can be allocated in this pool. Optional.
    Definition: vk_mem_alloc.h:1745
    -
    VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
    const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
    Informative callbacks for vkAllocateMemory, vkFreeMemory. Optional.
    Definition: vk_mem_alloc.h:1312
    -
    VmaPoolStats::unusedRangeCount
    size_t unusedRangeCount
    Number of continuous memory ranges in the pool not used by any VmaAllocation.
    Definition: vk_mem_alloc.h:1776
    -
    VmaAllocationCreateFlags
    VkFlags VmaAllocationCreateFlags
    Definition: vk_mem_alloc.h:1591
    -
    VmaAllocationCreateInfo::pool
    VmaPool pool
    Pool that this allocation should be created in.
    Definition: vk_mem_alloc.h:1626
    +
    VmaStatInfo::blockCount
    uint32_t blockCount
    Number of VkDeviceMemory Vulkan memory blocks allocated.
    Definition: vk_mem_alloc.h:1492
    +
    VmaVulkanFunctions::vkFreeMemory
    PFN_vkFreeMemory vkFreeMemory
    Definition: vk_mem_alloc.h:1301
    +
    VmaPoolCreateInfo::maxBlockCount
    size_t maxBlockCount
    Maximum number of blocks that can be allocated in this pool. Optional.
    Definition: vk_mem_alloc.h:1819
    +
    VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
    const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
    Informative callbacks for vkAllocateMemory, vkFreeMemory. Optional.
    Definition: vk_mem_alloc.h:1379
    +
    VmaPoolStats::unusedRangeCount
    size_t unusedRangeCount
    Number of continuous memory ranges in the pool not used by any VmaAllocation.
    Definition: vk_mem_alloc.h:1850
    +
    VmaAllocationCreateFlags
    VkFlags VmaAllocationCreateFlags
    Definition: vk_mem_alloc.h:1665
    +
    VmaAllocationCreateInfo::pool
    VmaPool pool
    Pool that this allocation should be created in.
    Definition: vk_mem_alloc.h:1700
    vmaGetMemoryProperties
    void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
    -
    VmaAllocatorCreateInfo::pHeapSizeLimit
    const VkDeviceSize * pHeapSizeLimit
    Either null or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
    Definition: vk_mem_alloc.h:1350
    -
    VmaStats::memoryType
    VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
    Definition: vk_mem_alloc.h:1434
    -
    VMA_ALLOCATION_CREATE_MAPPED_BIT
    Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
    Definition: vk_mem_alloc.h:1561
    -
    VmaStatInfo::allocationSizeMin
    VkDeviceSize allocationSizeMin
    Definition: vk_mem_alloc.h:1427
    +
    VmaAllocatorCreateInfo::pHeapSizeLimit
    const VkDeviceSize * pHeapSizeLimit
    Either null or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
    Definition: vk_mem_alloc.h:1417
    +
    VmaStats::memoryType
    VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
    Definition: vk_mem_alloc.h:1508
    +
    VMA_ALLOCATION_CREATE_MAPPED_BIT
    Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
    Definition: vk_mem_alloc.h:1635
    +
    VmaStatInfo::allocationSizeMin
    VkDeviceSize allocationSizeMin
    Definition: vk_mem_alloc.h:1501
    vmaFindMemoryTypeIndexForBufferInfo
    VkResult vmaFindMemoryTypeIndexForBufferInfo(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
    Helps to find memoryTypeIndex, given VkBufferCreateInfo and VmaAllocationCreateInfo.
    -
    VmaVulkanFunctions::vkCreateImage
    PFN_vkCreateImage vkCreateImage
    Definition: vk_mem_alloc.h:1285
    -
    VmaDeviceMemoryCallbacks::pfnAllocate
    PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
    Optional, can be null.
    Definition: vk_mem_alloc.h:1227
    -
    VmaVulkanFunctions::vkDestroyBuffer
    PFN_vkDestroyBuffer vkDestroyBuffer
    Definition: vk_mem_alloc.h:1284
    +
    VmaVulkanFunctions::vkCreateImage
    PFN_vkCreateImage vkCreateImage
    Definition: vk_mem_alloc.h:1312
    +
    VmaRecordSettings::flags
    VmaRecordFlags flags
    Flags for recording. Use VmaRecordFlagBits enum.
    Definition: vk_mem_alloc.h:1349
    +
    VmaDeviceMemoryCallbacks::pfnAllocate
    PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
    Optional, can be null.
    Definition: vk_mem_alloc.h:1254
    +
    VmaVulkanFunctions::vkDestroyBuffer
    PFN_vkDestroyBuffer vkDestroyBuffer
    Definition: vk_mem_alloc.h:1311
    vmaMapMemory
    VkResult vmaMapMemory(VmaAllocator allocator, VmaAllocation allocation, void **ppData)
    Maps memory represented by given allocation and returns pointer to it.
    -
    VmaPoolCreateInfo::frameInUseCount
    uint32_t frameInUseCount
    Maximum number of additional frames that are in use at the same time as current frame.
    Definition: vk_mem_alloc.h:1759
    -
    VmaVulkanFunctions::vkFlushMappedMemoryRanges
    PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges
    Definition: vk_mem_alloc.h:1277
    +
    VmaPoolCreateInfo::frameInUseCount
    uint32_t frameInUseCount
    Maximum number of additional frames that are in use at the same time as current frame.
    Definition: vk_mem_alloc.h:1833
    +
    VmaVulkanFunctions::vkFlushMappedMemoryRanges
    PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges
    Definition: vk_mem_alloc.h:1304
    vmaAllocateMemoryForImage
    VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    Function similar to vmaAllocateMemoryForBuffer().
    VmaAllocatorCreateInfo
    struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
    Description of a Allocator to be created.
    -
    VmaAllocationInfo::pUserData
    void * pUserData
    Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
    Definition: vk_mem_alloc.h:1907
    -
    VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
    VkDeviceSize preferredLargeHeapBlockSize
    Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
    Definition: vk_mem_alloc.h:1306
    -
    VmaStatInfo::allocationSizeAvg
    VkDeviceSize allocationSizeAvg
    Definition: vk_mem_alloc.h:1427
    -
    VmaStatInfo::usedBytes
    VkDeviceSize usedBytes
    Total number of bytes occupied by all allocations.
    Definition: vk_mem_alloc.h:1424
    +
    VmaRecordSettings::pFilePath
    const char * pFilePath
    Path to the file that should be written by the recording.
    Definition: vk_mem_alloc.h:1357
    +
    VmaAllocationInfo::pUserData
    void * pUserData
    Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
    Definition: vk_mem_alloc.h:1981
    +
    VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
    VkDeviceSize preferredLargeHeapBlockSize
    Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
    Definition: vk_mem_alloc.h:1373
    +
    VmaStatInfo::allocationSizeAvg
    VkDeviceSize allocationSizeAvg
    Definition: vk_mem_alloc.h:1501
    +
    VmaStatInfo::usedBytes
    VkDeviceSize usedBytes
    Total number of bytes occupied by all allocations.
    Definition: vk_mem_alloc.h:1498
    VmaDeviceMemoryCallbacks
    struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
    Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
    vmaCheckCorruption
    VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
    Checks magic number in margins around all allocations in given memory types (in both default and cust...
    -
    VmaPoolStats
    Describes parameter of existing VmaPool.
    Definition: vk_mem_alloc.h:1764
    +
    VmaPoolStats
    Describes parameter of existing VmaPool.
    Definition: vk_mem_alloc.h:1838
    vmaCheckPoolCorruption
    VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
    Checks magic number in margins around all allocations in given memory pool in search for corruptions...
    -
    VmaAllocationInfo::offset
    VkDeviceSize offset
    Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
    Definition: vk_mem_alloc.h:1888
    -
    VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1589
    -
    VmaDefragmentationStats::bytesMoved
    VkDeviceSize bytesMoved
    Total number of bytes that have been copied while moving allocations to different places...
    Definition: vk_mem_alloc.h:2134
    -
    VmaVulkanFunctions
    Pointers to some Vulkan functions - a subset used by the library.
    Definition: vk_mem_alloc.h:1270
    +
    VmaAllocationInfo::offset
    VkDeviceSize offset
    Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
    Definition: vk_mem_alloc.h:1962
    +
    VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1663
    +
    VmaDefragmentationStats::bytesMoved
    VkDeviceSize bytesMoved
    Total number of bytes that have been copied while moving allocations to different places...
    Definition: vk_mem_alloc.h:2208
    +
    VmaVulkanFunctions
    Pointers to some Vulkan functions - a subset used by the library.
    Definition: vk_mem_alloc.h:1297
    vmaCreateAllocator
    VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
    Creates Allocator object.
    -
    VmaStatInfo::unusedRangeCount
    uint32_t unusedRangeCount
    Number of free ranges of memory between allocations.
    Definition: vk_mem_alloc.h:1422
    -
    VMA_MEMORY_USAGE_UNKNOWN
    Definition: vk_mem_alloc.h:1477
    -
    VmaPoolCreateFlags
    VkFlags VmaPoolCreateFlags
    Definition: vk_mem_alloc.h:1717
    +
    VmaStatInfo::unusedRangeCount
    uint32_t unusedRangeCount
    Number of free ranges of memory between allocations.
    Definition: vk_mem_alloc.h:1496
    +
    VMA_MEMORY_USAGE_UNKNOWN
    Definition: vk_mem_alloc.h:1551
    +
    VmaPoolCreateFlags
    VkFlags VmaPoolCreateFlags
    Definition: vk_mem_alloc.h:1791
    vmaGetPhysicalDeviceProperties
    void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
    -
    VmaStatInfo::allocationCount
    uint32_t allocationCount
    Number of VmaAllocation allocation objects allocated.
    Definition: vk_mem_alloc.h:1420
    -
    VmaVulkanFunctions::vkGetImageMemoryRequirements
    PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
    Definition: vk_mem_alloc.h:1282
    -
    VmaVulkanFunctions::vkDestroyImage
    PFN_vkDestroyImage vkDestroyImage
    Definition: vk_mem_alloc.h:1286
    -
    VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
    Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
    Definition: vk_mem_alloc.h:1548
    -
    VMA_MEMORY_USAGE_CPU_ONLY
    Definition: vk_mem_alloc.h:1504
    -
    VmaAllocationInfo::pMappedData
    void * pMappedData
    Pointer to the beginning of this allocation as mapped data.
    Definition: vk_mem_alloc.h:1902
    +
    VmaRecordSettings
    Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSetting...
    Definition: vk_mem_alloc.h:1346
    +
    VmaStatInfo::allocationCount
    uint32_t allocationCount
    Number of VmaAllocation allocation objects allocated.
    Definition: vk_mem_alloc.h:1494
    +
    VmaVulkanFunctions::vkGetImageMemoryRequirements
    PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
    Definition: vk_mem_alloc.h:1309
    +
    VmaVulkanFunctions::vkDestroyImage
    PFN_vkDestroyImage vkDestroyImage
    Definition: vk_mem_alloc.h:1313
    +
    VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
    Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
    Definition: vk_mem_alloc.h:1622
    +
    VMA_MEMORY_USAGE_CPU_ONLY
    Definition: vk_mem_alloc.h:1578
    +
    VmaAllocationInfo::pMappedData
    void * pMappedData
    Pointer to the beginning of this allocation as mapped data.
    Definition: vk_mem_alloc.h:1976
    vmaDestroyImage
    void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
    Destroys Vulkan image and frees allocated memory.
    -
    VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
    Enables usage of VK_KHR_dedicated_allocation extension.
    Definition: vk_mem_alloc.h:1260
    +
    VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
    Enables usage of VK_KHR_dedicated_allocation extension.
    Definition: vk_mem_alloc.h:1287
    VmaDefragmentationStats
    struct VmaDefragmentationStats VmaDefragmentationStats
    Statistics returned by function vmaDefragment().
    -
    VmaVulkanFunctions::vkAllocateMemory
    PFN_vkAllocateMemory vkAllocateMemory
    Definition: vk_mem_alloc.h:1273
    -
    VmaAllocationInfo
    Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
    Definition: vk_mem_alloc.h:1869
    +
    VmaVulkanFunctions::vkAllocateMemory
    PFN_vkAllocateMemory vkAllocateMemory
    Definition: vk_mem_alloc.h:1300
    +
    VmaAllocationInfo
    Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
    Definition: vk_mem_alloc.h:1943
    vmaAllocateMemory
    VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    General purpose memory allocation.
    vmaSetCurrentFrameIndex
    void vmaSetCurrentFrameIndex(VmaAllocator allocator, uint32_t frameIndex)
    Sets index of the current frame.
    VmaAllocationCreateInfo
    struct VmaAllocationCreateInfo VmaAllocationCreateInfo
    vmaAllocateMemoryForBuffer
    VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    -
    VmaPoolCreateFlagBits
    VmaPoolCreateFlagBits
    Flags to be passed as VmaPoolCreateInfo::flags.
    Definition: vk_mem_alloc.h:1695
    -
    VmaStatInfo::unusedRangeSizeAvg
    VkDeviceSize unusedRangeSizeAvg
    Definition: vk_mem_alloc.h:1428
    +
    VmaPoolCreateFlagBits
    VmaPoolCreateFlagBits
    Flags to be passed as VmaPoolCreateInfo::flags.
    Definition: vk_mem_alloc.h:1769
    +
    VmaStatInfo::unusedRangeSizeAvg
    VkDeviceSize unusedRangeSizeAvg
    Definition: vk_mem_alloc.h:1502
    vmaTouchAllocation
    VkBool32 vmaTouchAllocation(VmaAllocator allocator, VmaAllocation allocation)
    Returns VK_TRUE if allocation is not lost and atomically marks it as used in current frame...
    -
    VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
    Definition: vk_mem_alloc.h:1587
    -
    VmaStats::memoryHeap
    VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
    Definition: vk_mem_alloc.h:1435
    +
    VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
    Definition: vk_mem_alloc.h:1661
    +
    VmaRecordFlagBits
    VmaRecordFlagBits
    Flags to be used in VmaRecordSettings::flags.
    Definition: vk_mem_alloc.h:1321
    +
    VmaStats::memoryHeap
    VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
    Definition: vk_mem_alloc.h:1509
    vmaDestroyBuffer
    void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
    Destroys Vulkan buffer and frees allocated memory.
    -
    VmaPoolStats::unusedSize
    VkDeviceSize unusedSize
    Total number of bytes in the pool not used by any VmaAllocation.
    Definition: vk_mem_alloc.h:1770
    -
    VmaStatInfo::unusedRangeSizeMax
    VkDeviceSize unusedRangeSizeMax
    Definition: vk_mem_alloc.h:1428
    -
    VmaAllocationInfo::memoryType
    uint32_t memoryType
    Memory type index that this allocation was allocated from.
    Definition: vk_mem_alloc.h:1874
    +
    VmaPoolStats::unusedSize
    VkDeviceSize unusedSize
    Total number of bytes in the pool not used by any VmaAllocation.
    Definition: vk_mem_alloc.h:1844
    +
    VmaStatInfo::unusedRangeSizeMax
    VkDeviceSize unusedRangeSizeMax
    Definition: vk_mem_alloc.h:1502
    +
    VmaRecordSettings
    struct VmaRecordSettings VmaRecordSettings
    Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSetting...
    +
    VmaAllocationInfo::memoryType
    uint32_t memoryType
    Memory type index that this allocation was allocated from.
    Definition: vk_mem_alloc.h:1948
    diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index d9a8029..b6d62c0 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -1317,8 +1317,15 @@ typedef struct VmaVulkanFunctions { #endif } VmaVulkanFunctions; +/// Flags to be used in VmaRecordSettings::flags. typedef enum VmaRecordFlagBits { + /** \brief Enables flush after recording every function call. + + Enable it if you expect your application to crash, which may leave recording file truncated. + It may degrade performance though. + */ VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001, + VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF } VmaRecordFlagBits; typedef VkFlags VmaRecordFlags; @@ -1335,10 +1342,18 @@ available through VmaAllocatorCreateInfo::pRecordSettings. #endif #endif -/// TODO +/// Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSettings. typedef struct VmaRecordSettings { + /// Flags for recording. Use #VmaRecordFlagBits enum. VmaRecordFlags flags; + /** \brief Path to the file that should be written by the recording. + + Suggested extension: "csv". + If the file already exists, it will be overwritten. + It will be opened for the whole time #VmaAllocator object is alive. + If opening this file fails, creation of the whole allocator object fails. + */ const char* pFilePath; } VmaRecordSettings; @@ -1412,7 +1427,11 @@ typedef struct VmaAllocatorCreateInfo e.g. fetched using `vkGetInstanceProcAddr()` and `vkGetDeviceProcAddr()`. */ const VmaVulkanFunctions* pVulkanFunctions; - /** TODO + /** \brief Parameters for recording of VMA calls. Can be null. + + If not null, it enables recording of calls to VMA functions to a file. + If support for recording is not enabled using `VMA_RECORDING_ENABLED` macro, + creation of the allocator object fails with `VK_ERROR_FEATURE_NOT_PRESENT`. */ const VmaRecordSettings* pRecordSettings; } VmaAllocatorCreateInfo; From b3ea2c62f1658be558e0ae37ba7677effd4642d9 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 10:45:23 +0200 Subject: [PATCH 32/51] "Recording file format.md": added example file. --- docs/Recording file format.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/Recording file format.md b/docs/Recording file format.md index 882a294..65c0b9e 100644 --- a/docs/Recording file format.md +++ b/docs/Recording file format.md @@ -174,3 +174,27 @@ If `pUserData` was a pointer, it is encoded as hexadecimal string. If `VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT` was used with the allocation, the string is written as-is. It may contain additional commas. It should not contain end-of-line characters - results are then undefined. + +# Example file + + Vulkan Memory Allocator,Calls recording + 1,2 + 12552,0.000,0,vmaCreateAllocator + 12552,0.010,0,vmaCreateImage,0,1,37,128,128,1,1,1,1,1,1,0,8,36,2,0,0,0,0000000000000000,000001D85B8B1A80, + 12552,0.010,0,vmaSetAllocationUserData,000001D85B8B1A80,Ala ma kota + 12552,0.015,0,vmaCreateImage,0,1,37,128,128,1,1,1,1,0,6,0,0,0,1,0,0,0,0000000000000000,000001D85B8B1620, + 12552,0.017,0,vmaDestroyImage,000001D85B8B1A80 + 12552,0.017,0,vmaCreateBuffer,0,768,1,0,4,2,0,0,0,0000000000000000,000001D85B8B19E0, + 12552,0.017,0,vmaCreateBuffer,0,768,130,0,0,1,0,0,0,0000000000000000,000001D85B8B1A80, + 12552,0.017,0,vmaCreateBuffer,0,60,1,0,4,2,0,0,0,0000000000000000,000001D85B8B1DA0, + 12552,0.017,0,vmaCreateBuffer,0,60,66,0,0,1,0,0,0,0000000000000000,000001D85B8B16C0, + 12552,0.017,0,vmaDestroyBuffer,000001D85B8B1DA0 + 12552,0.017,0,vmaDestroyBuffer,000001D85B8B19E0 + 12552,0.022,0,vmaCreateImage,0,1,126,1424,704,1,1,1,1,0,32,0,0,0,1,0,0,0,0000000000000000,000001D85B8B1EE0, + 12552,0.048,0,vmaDestroyImage,000001D85B8B1EE0 + 12552,0.053,0,vmaCreateImage,0,1,126,1424,704,1,1,1,1,0,32,0,0,0,1,0,0,0,0000000000000000,000001D85B8B1EE0, + 12552,0.662,0,vmaDestroyImage,000001D85B8B1EE0 + 12552,0.695,0,vmaDestroyImage,000001D85B8B1620 + 12552,0.695,0,vmaDestroyBuffer,000001D85B8B16C0 + 12552,0.695,0,vmaDestroyBuffer,000001D85B8B1A80 + 12552,0.695,0,vmaDestroyAllocator \ No newline at end of file From 80cb2365c59937e299090ced04e9c772b2735f7b Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 10:59:53 +0200 Subject: [PATCH 33/51] Added recording and replaying of calls to functions: vmaTouchAllocation, vmaGetAllocationInfo. VmaReplay: fixed handling of null allocation. --- docs/Recording file format.md | 4 + src/VmaReplay/VmaReplay.cpp | 141 ++++++++++++++++++++++++++++++++-- src/vk_mem_alloc.h | 46 +++++++++++ 3 files changed, 184 insertions(+), 7 deletions(-) diff --git a/docs/Recording file format.md b/docs/Recording file format.md index 65c0b9e..02e414b 100644 --- a/docs/Recording file format.md +++ b/docs/Recording file format.md @@ -154,6 +154,10 @@ No parameters. - offset : uint64 - size : uint64 +**vmaTouchAllocation, vmaGetAllocationInfo** (min format version 1.2) + +- allocation : pointer + ## Data types **bool** diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index d3dc579..7ab8df3 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -517,6 +517,8 @@ private: void ExecuteUnmapMemory(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteFlushAllocation(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteInvalidateAllocation(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteTouchAllocation(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteGetAllocationInfo(size_t lineNumber, const CsvSplit& csvSplit); void DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit); }; @@ -644,6 +646,10 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) ExecuteFlushAllocation(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaInvalidateAllocation")) ExecuteInvalidateAllocation(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaTouchAllocation")) + ExecuteTouchAllocation(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaGetAllocationInfo")) + ExecuteGetAllocationInfo(lineNumber, csvSplit); else { if(IssueWarning()) @@ -1620,11 +1626,21 @@ void Player::ExecuteMapMemory(size_t lineNumber, const CsvSplit& csvSplit) const auto it = m_Allocations.find(origPtr); if(it != m_Allocations.end()) { - void* pData; - VkResult res = vmaMapMemory(m_Allocator, it->second.allocation, &pData); - if(res != VK_SUCCESS) + if(it->second.allocation) { - printf("Line %zu: vmaMapMemory failed (%d)\n", lineNumber, res); + void* pData; + VkResult res = vmaMapMemory(m_Allocator, it->second.allocation, &pData); + if(res != VK_SUCCESS) + { + printf("Line %zu: vmaMapMemory failed (%d)\n", lineNumber, res); + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Cannot call vmaMapMemory - allocation is null.\n", lineNumber); + } } } else @@ -1659,7 +1675,17 @@ void Player::ExecuteUnmapMemory(size_t lineNumber, const CsvSplit& csvSplit) const auto it = m_Allocations.find(origPtr); if(it != m_Allocations.end()) { - vmaUnmapMemory(m_Allocator, it->second.allocation); + if(it->second.allocation) + { + vmaUnmapMemory(m_Allocator, it->second.allocation); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Cannot call vmaUnmapMemory - allocation is null.\n", lineNumber); + } + } } else { @@ -1697,7 +1723,17 @@ void Player::ExecuteFlushAllocation(size_t lineNumber, const CsvSplit& csvSplit) const auto it = m_Allocations.find(origPtr); if(it != m_Allocations.end()) { - vmaFlushAllocation(m_Allocator, it->second.allocation, offset, size); + if(it->second.allocation) + { + vmaFlushAllocation(m_Allocator, it->second.allocation, offset, size); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Cannot call vmaFlushAllocation - allocation is null.\n", lineNumber); + } + } } else { @@ -1735,7 +1771,17 @@ void Player::ExecuteInvalidateAllocation(size_t lineNumber, const CsvSplit& csvS const auto it = m_Allocations.find(origPtr); if(it != m_Allocations.end()) { - vmaInvalidateAllocation(m_Allocator, it->second.allocation, offset, size); + if(it->second.allocation) + { + vmaInvalidateAllocation(m_Allocator, it->second.allocation, offset, size); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Cannot call vmaInvalidateAllocation - allocation is null.\n", lineNumber); + } + } } else { @@ -1756,6 +1802,87 @@ void Player::ExecuteInvalidateAllocation(size_t lineNumber, const CsvSplit& csvS } } +void Player::ExecuteTouchAllocation(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origPtr = 0; + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + const auto it = m_Allocations.find(origPtr); + if(it != m_Allocations.end()) + { + if(it->second.allocation) + { + vmaTouchAllocation(m_Allocator, it->second.allocation); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Cannot call vmaTouchAllocation - allocation is null.\n", lineNumber); + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origPtr); + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaTouchAllocation.\n", lineNumber); + } + } + } +} + +void Player::ExecuteGetAllocationInfo(size_t lineNumber, const CsvSplit& csvSplit) +{ + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origPtr = 0; + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + const auto it = m_Allocations.find(origPtr); + if(it != m_Allocations.end()) + { + if(it->second.allocation) + { + VmaAllocationInfo allocInfo; + vmaGetAllocationInfo(m_Allocator, it->second.allocation, &allocInfo); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Cannot call vmaGetAllocationInfo - allocation is null.\n", lineNumber); + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Allocation %llX not found.\n", lineNumber, origPtr); + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaGetAllocationInfo.\n", lineNumber); + } + } + } +} + //////////////////////////////////////////////////////////////////////////////// // Main functions diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index b6d62c0..8fda710 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -4803,6 +4803,10 @@ public: VmaAllocation allocation); void RecordDestroyImage(uint32_t frameIndex, VmaAllocation allocation); + void RecordTouchAllocation(uint32_t frameIndex, + VmaAllocation allocation); + void RecordGetAllocationInfo(uint32_t frameIndex, + VmaAllocation allocation); private: struct CallParams @@ -8432,6 +8436,30 @@ void VmaRecorder::RecordDestroyImage(uint32_t frameIndex, Flush(); } +void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex, + allocation); + Flush(); +} + +void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex, + VmaAllocation allocation) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex, + allocation); + Flush(); +} + VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData) { if(pUserData != VMA_NULL) @@ -10519,6 +10547,15 @@ void vmaGetAllocationInfo( VMA_DEBUG_GLOBAL_MUTEX_LOCK +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordGetAllocationInfo( + allocator->GetCurrentFrameIndex(), + allocation); + } +#endif + allocator->GetAllocationInfo(allocation, pAllocationInfo); } @@ -10530,6 +10567,15 @@ VkBool32 vmaTouchAllocation( VMA_DEBUG_GLOBAL_MUTEX_LOCK +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordTouchAllocation( + allocator->GetCurrentFrameIndex(), + allocation); + } +#endif + return allocator->TouchAllocation(allocation); } From 6ea177877a3b60c083f2295d730710c2b736505f Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 11:09:50 +0200 Subject: [PATCH 34/51] Added binary of VmaReplay. --- bin/VmaReplay_Release_vs2015.exe | Bin 0 -> 100864 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 bin/VmaReplay_Release_vs2015.exe diff --git a/bin/VmaReplay_Release_vs2015.exe b/bin/VmaReplay_Release_vs2015.exe new file mode 100644 index 0000000000000000000000000000000000000000..05a6da46851e2ea0173f02569a0ac32c51fb736e GIT binary patch literal 100864 zcmdqK3w%`7^*=hvBoKsfhDR_e$|$i$V>KEdNr%)K$iO}@1F4Fj5{a$SP+w^>gFyv? zlR(DfRH{~~TA{7|iG4|bS_QR|5J#VN1>Blu!-{Abw)i>UFQ^5JdYn?U08=doSbe7Gy#CgL_*Iav6cJ?uO zR?$zy9&0}EaQi*+zr*icde1?;Z*Sgo_m}c_&34VQ!M1sOZoM-8ZJ+HF z+tBpkHk$(tyiKL9J_gSY`RvA%dQP|52C2`9Ut1Txl9>2sw^c5U16i<0RfWWVn9oY# zqwU@6(ro+q(3~cVvaLw7txe6)nPw};=W=hFt(y6c@UI!)?@j&YOtYcG{g{|cm{Oxqm%j>d1wKR4LtES1gX9*ht5_zqgx zs{7?@KC4<~lx;*?Z6lGdre8kySwEgP|C(Bxtr(weYpwDc2FO=aTk|7i>EfqmE8nt# z@?CrVO~^=E+B&U#P5tss`r0Kb@+@n z;d;}jO>FgpDnrchAd)9I7>PCM3;e?HiUeM9y2f9V5y|6a|KG&g|yTN_n^K#N? zwlD&Dm6GGyiKozbMHG1QMN5xeG&XAu z`H;fG_$fyG^bD|2nax&KJS1=;I;>E{Hj08t`6ASkC9}IaYyQ#Q!-^v3?eRD6*BYXD z@f0u4)EZW@(uhbCCDW>dXNkxYE07?JZNk`Ne)Ux>)_|&tP?fvs*4Z9oP&m|t??uhs z_dubV5pt_=1_i4ZnR7}@yzbqDu^`;^OSa}+Ge^OzY(JnbU z&in<)>@%*+*CW$1%8ExQw$~vHQ`p|;Y6jb7I4qIPm`;^Ej%>7q5bU zNkLO_+Yj*_PwBy#sz*@7aB}nxDK#iyCgp?bo-|9!1itGtrh&+M2K7N%ahle!UY0zd zjCso!@uRg82S@?Qn7Kf};^>b+T5`1Hqb{V&E_oE?p-b|$hF{^y7x5+-+sy{-g)B&+ zY>6Zyg~lorkTUXx5pT2)lDTNRx#3e*{7~L%;v3pj?YV01RjvNWdAnq-7o%s%2!Vn` zkD@l&m-Bkp*v*$@|W)uZbFa}q3dT@Gz#HyD-S4cC^m;#z?s(3l@oj8eCfWCnm z`FQ~O-3QNf^LZ(!q^!87Cxx=hfIr$8ys$UA9PlE1s~m;NLBAOT!XirGS0qR@gPjp@ z^&SXA0v(7!cLk=~Tf+rgp8RRJ~nV{yt$G(q=VlXYC6fA7{jzzYReSW2gBBda}EU z!zcVu9tC01>)P4-YR8sECZS)TEpqNhPV*^b(`KDN?vylJG)IJPA=$2wqb;37rM)Tl zGtH5`_heYqvzkBGu4vO{?VhKd-GawGwG%$mF3))q-%y&L^+x>7^K@--r}+u&@8Sw5 z)14fH5pS~yZG#2eT0bEd7I61^q@vu+CcvV!&RRE$MIjCp!jf4K%HE|4`%S#C+fxc# zkt_|4r%gNi{7ERRc8V%!XI8I*I#ofJ#|tV+DJX7_Cd)fPm3IJ6V(ovs1gVB|tDe*45+kPv)1(IJf2n$m@_ z3nl=}M5j5M(g?+eHWgl(aLlZ6LH`W)bs*0nO=@9W4)9NXuGZQry~D!kU0QTkQg@4j zHk!bNw0?_e{myMw@AT!8X^BxQ+vkMo3diDAs2xYm3}FnR%TmfXNzH`)E#m@%V>uc2 zG~0G$>k+P&8oYl2bhfi6LI<;I@&2xS4;?&h0bX8_FGh#?{SSeCJKPzzLQ1#xNhlTN zVw9fHW~bD5_M(}aWGWQxpeuw+jO24eU=TUiR;_{7pG20JmJ>}N+Z)p>Q2B}!OsC5C zB&Oe!?-r)tRhc`?d!sf$#Zk4eR2CQ%u$>*#+Qzz}8@8GEfrC(oB(0nMJ~!U)8>N?? zATQ+MoIHng&xLUudY&CShe8rPv@lD!-U@(Mr)8M|3gu91AyR@L3uC$?>GX(#;q-Xm zZh`rAxLZE^Mq!LGe=XS|B*K%yXOkW5ZQ1cxNt8g%e4nAi&1wcIkBJ;MXA!zYj-vq9!BFhWK9S4&u8qUBWb)Sy`ox@gBXeGRU55;gb{+O-^+wCG&&Iph#@32Gy2dDf5z&2bYjXdYb8VX8+_YX zwJhM;H?wx^d8N>{G2U3I?2&D zlLywNkxV0HQ#cZJ$WReykO$?*FTakr6VCc8=6f_Sf#nUV?~G}V8gPeQaYvL5Q$z%J zuyv9<X5ehArY^^_;%{#gAXVo=?;H~R)V2p zASrj)8Y3=KyL~7>^-3o9>vlxYHfzhE0sDLv1*XrnS@O)iHqz|9ugF@^&IOw>GQnN@ zEYiB#BxSFXxz@>CkqL!@{#==?)Z(I2nc;qB5U!oIZ98jh97Bg#ejj<2Jl zCzF9bq*YAJwc9sH4Ztzj{egUZQRc-^8cHDzX4((+NI_wL86v^F2nJ7rzS`}NEAZ~$ z<5xDGrbef*@P8O`A(qo~h^>s{-#~~scPx?_USS3)KE_%N*i1to40466DuwkwQJ8Z) zfc1%X^a6|Z&q~EsO0SpApMRc$Exkue^GNw{J;Q*l2?^s)2#G%YpQTI_YxY!Sms7h(9YFt|}KC1H391~up(o|2o;hP@Q`2;kJl%yYoc zQc5MHUR-9SptGI&N35Hv{$;}5r+x4=={GHs;bt%{p}sdtL*qe7TVo>R)OVmh znW%rLd{5T@)j{^XRsYBG#YogYy?_1G|0=W!R0^fMK8SG|%qV(2I#aBRV}{{G%0zlt zdSuwk60@ZTr49BS1Q}0*-E}iETrV@gw0iU%bV6kIX(IHGE)ky86U{R3Wsh4sDEL6Me1q8J?f@);DGpx&w0V!++(YXllJPlpP`TuRNJf-V zR5xB=R*YkJ3y?Tz=vY*i^2jo6Aq8os9g+g$N06YgoWDZf!E8ieXd{%=&F6~n4=hoc z15w$Q;Q(e9M@#PE(8w5-%3X{KU)PW-%L@Z}XFYSsN>}_)3 z^sB+ur`jaT_+Zh4`!8ju`SP+Zu|mVe35!%JT}&e!5}}=%lnWT8hoa|0&mipGA$D~q zEVti_1MiDnAHXbHFh;buOM8|mw}1_A&VZSpCL%M^MB^4P!+wYbtGiE)clV%J&LV0< z^!EBL_I5RT`#tpbWcK#|Fv-{##iYlayhu9`}wihWpz;6h-Wl0`so6eZ(}OdIN_ zm9)b={SyLPa(0HaqlYF2qO$l$s;)~|{OR&NX-9ts)G!c~#s5$F0z`ep>gZ;4xCnAF z*B7n@H&8YLzc(kgWfVcuZYpLNDDD)<@r&Uyq-YBaPFj;<=^;5PP8X4>7!_MNDlYpH zxS038MIs3h>1_}Rh{qv}t!5p+DZ@0?T!6iRu9ltdqd$rMR>Sl)5s|TrgXSIY(hUjZ z!HnMKS?9E|ePwz4MS<`JvSK+q(R##35b+npu`r1~c)z7jeVDJi46`h<9=~xA%G+nW zgMs63+!B~4xrn+Cy{0T+pR1>4hR!6=NLs+UYv+PenUc^CzJeZLCycVZYAO=&a6Xia zkFgl@8L3j|fH8oXXRc=sCjh58bk}w{PM?K9iRENNOM({HtGSbEAKF_fjLkkcEC)LM z1z+f{{kk?~zbM#2IYc9>R>HjieFe8*KaJDb!u4^jE5TJjZm818R(q5xBNvVl_FZ7a zsgYq$Vc#R{FT`>-p&+G?dqlxuVgE)neiR%c4TvqIbX);{4K74~vs$Tu(N5`+l`%j| z3*tfMc70hpCxOVR2|M)t{>u0!OBw%|%D5k*$3nE6+y~R+<0?9-o`GvXV{3fIW*z3@ zHaL?BnV~&p;8KAH&z=&Gs4Bg80XO{S39!}Il-zev5uFSMvW-7TZw>Dch zUItywM-O2x^8`P7bc`>2oi*eW%u8#+^-b{HzWzfZgY4JGV56Wn5Q%LEr3<_7!8=@= zKy9L*kH~Pb0Q>PAzk-BaDC`KRT+HBeV$LHWD=l-GtaL%|D$Ost;S`-L>;0xSc(i(H zV2vRESQy80y7r;QxX?W^wpzpAFk~%^fi0U0jJto0RKM%Jn~z$vA2kou;>vGPzod1k zsy~i-_i1>C2YD7aium8WAU;wC9^eDl{!XgOLF<)LikFl^_6;1t1+v+?{UuOO*k6$o z`0>00)yg4gQF}897R#yWQ{O;PN21gP-vSnh`3g1%Mzf27q3R__vO^z}ub|Y(>dC>+ zA&0RJiOj({8_F&p%+TtC2;k_?_gr;V1(cq*Xp7I-?L#Ccv~jTB@_`*1&aX|ujL7c5 z33@>X&7r|ljf^xsjG4gZEIo$k%hn)7L$lN~(rP)9QaebcHk)B=NQ{w^8}^#NeX)e zi_K`!PA|(V1Y6_9hiRH}SPkj5Zk;L`LJVxi&*SQu8K2SugFW>VY-a|tXqRwphYlYl z)}n`699GKnEFlPcCjcizO=Dr&-?I+W)xgIGQ`P>au~q&tGgeEb!L7tY4&gi5pI3I;nP#%d7P{Nv^7rHe1)W&z1a2onadiQHCvJay(D zqMgCXkefNeA)$WC)z57C^K7Mht5QGJ>ZeZq)XSe|o7B$=^|Mm_tX4m3_@ngcKn_ax zS2LlqlK)gZKVi*-4589hQ&Gm^L1tx+jOuPPr;NlysB}5f(-$2NQ3Zd`yy#Y`D0!k zeXyk;HmM=?cikFNjiiW2qoZ{2c@Q0}Cg9#3GTJz*%JqV*fY%p+;iuM@h|pnY?Rk3G zjiJ?2i+I$f9Bzll1qNXmSA0md)O-l>A!2QiAZkdp)WB+#cF5rj?e?b`(!(&S;y3{> zJe1cAtm4CK4vft3lX>+>qU}I-Fc{{iy38s)L4>of#GiAm+qoY3{MV)fJhlt$Og<8yeu4YwbdG2<%Tz(n-Zm4t~ z7&?8?NP5~4%*TfZLoB&#N5krkq25=(D<44_m{l-;*b#Loi!93#sL0sv}XiAsI;;A&sN=X&Vy$l!N`0rivlQ{)U{ z6o8Jd%nb0C;W?t`Es|H?y5>kt-q#zzgOrGMoAp@bPgdV1z0_3l=E) z{S?_qza?;4LBC!#W{(9x7bFF0bOaHbW{u@67jZe+LDTS}H85zI+9ofvs5VXHW4uiz zeCbugm8YPF#i|A^I7Xo~2HA+xhi?+4!09MDJ6No0KZZq|DakdMk0&ci$DpEmm;6&F zO3qUy9~*D%->Jz^^4+rJ-T;NC1K@LLG*mYt_&rq}^&!^4sm4Cf7gp%_Ugk6XDy%O6 z{1XcP_;^zuv?@GVHDxTyJ}!>TD(hS30LV-#L}Bw(VZ(xFDrA1fI-RoZM!)JmJs%L~ zDToAfUC(T)Bj)ol-Vv4k!0)~g;9o+`0K<`S<|<{jICB-p3*cBR>Q}(kB`DxYRX~E? zqmhmDKIM)6%vJ9KK&VfRU%{76)VQB7M42^y%9mceb5|(}n5YUE9Uo4+neDq)hrPfT z5X|Aq+}aP^b`iks0GDE7d~dO;g}SjsP)XJM0owgCULj{8WSj7c58bUu36<`eNTHnH z7n&ax^g?7eU-|-T#~x15%s$L^^_HtB8Mfg+-dP)IIa zMqjejn*OD>nbRJUV7*IyB~fa%EOmbXgL4lj*KWyTp=ZPk9RSVaR}gC%M0xzrd}LrB zdR^fjQrOYL*`UiPrEciNe^k7cs31L#5yMNImJpWV6(h zSUv_lc@nh|0=HOci%uL5!@-L{o$GTdOPCc>&?-aSvONp^L@%_4rI$cI2MR-_|8QF> zc&JjrnAb+p$FgWlmqQ(8r=wuB&wS@IXcY?nUS*EGl3&XhHbwYPhP+{GX(O;g-60&> z2px>X&^IuGHO%<87tu&xsPrfCiif0DjAhO#ZF(9S+x!XYrS?Y++sr>f9P}+PRC?04 zDCs$Jvlhae}je9{k$X{6av(bmuh;!GW$(hv` z#cfT8)U*dh*xwl{Z9=kb@i35Z7VG}ar*ge099T>dbFp14=ka&oSaA_yB+`tc&ACDv z(7)nK2{K~F9p0qM8$wQLs&{Ytg=L=6`CT15C8|x^gl9 zvn(Jb?4L8(bJ_}7VD?$>qPFYM94MpVSTZZKV4nN!XeOt-PlOdR6j^}!lO}S$zYi|% zx%h%Tsz|m*V!ZYVMBM+Py86=JFwhfNg694SJ**ou9I>2#bm1vH zr5K6p5{Wxhq8o|NCKA`HMEX*dJzqTFV)HqW@V{yxU61tY;iw1A-j5DodYwNqqnyhK zVdhq{K2K~`G02{0rR6CcW}sxWEf3bu7@%GMKF}Tyw7K3o)_o>uhN=7oBW0T&j*#kAm%Sc}KiQ9CI5t|vUh zDT`|NaUJXwCsvco2AxKnFa9{*pe`^tSKbvL;%L*1msT3t|NHLH9^a3xn)OI}!{x_0?g5XV~$p{T}Ood`G9 z6TNUFlp%jss-M;TF}#pE6-}a`RgS$zndqum0iOmAL_@RC%C?DcWDWARH#Xq^q$EZz z;wp_z^0ASfgSWMK!w67(H?vw1^bTo>1t@24{wM63-y>7e?y|_la){L~yqHhI(a{Y| z$cGy-fcXfYyb&S=L-zqEIWR&*e$jc^~iule{=_x?7y zAm9%9(U^>YMjICd+#=r#_Q9=`3j%&3-$RGxf`AH@8OF#Cv!#_S39jah(C?n6}+dne2sKOw)dDjWV)rzcIjvy;-L|0MLL`g6v%)dwJ8FA#R&Mh zb>minpdaSd)WSCl!81 zX)=4BR{L5OR|LbzyRlGq2Ls>V_m%vPwkmAGU(j@wT^3=AxgU69nSX9s@wE=WR6WZctj_YAEs1L=6fjVKSHkWt)$J6uPo6y2#YbA$bK*~{|VhCIFmp-9YZ3LecjvS|M%H5_sKXg z@S&{y5z~jO^!hXa+wYS>WBYy8>a7YtSdOHV{YE9px+2*RG3hRqG&@aJU#@u()k73dfmBgCZStQP%nf~cN zWwTbKq(8;EAO~$&}TsSV@-{E1K&~=bO-}}gZ))G ztwP1@MEo?C^Hbf*s!nRcv)) zsPvU{=nJ3fGt%Uy%rM;S9=8_SK=l>LzM~7(zwr+k(0^t^42zGl7cgxL(;h&YZV1dJ z&D^QmvuMjB@XET3*_FCIw=Cj51{UXF{9##{_gMUm!{2H6TOz_0m`|_SjqRDpP*Sl| zYs`d-gsiGYanKyotE{;L`$@t+9NAxqnO@fi;;mTD zaezQ|8bF``xR(wflsN!-AN|>c$39T+!=GW4mqC$XGA zDX4LP`b2BIP(iI#PzM0Dra#nySk6s=B27vv4rq<(3aD8Dq0@i1P69cY5GO<|mNQXR zwFFQxt#LbtVQe`EEfjEjQL``nTwVu2e8v(CmYJAW*oo$}n=h^+A8e3X$k-QfdW`#3 z{HdZHfXH?iKVemndHF4{1lOQ5l}hGrB9HFjR8^tafCf24_mG8sSlJ#2W| zOuu=Xrre?>C>43$_aLiz=zW~eq)++827N?53 zP!@H{G#2H(5W$IDA9muf(K7lkf1k|jtq$0X-5BN;P$oNDG}T4HzPL4iPLPKsM4;Nb^LiobNJuKQv|Bs&vZ* zYJw5~QIh2FFAJ6K{Vqv?1v0&dKW1Y%{0;bt5HDixRo#2vVl{I7Clgzds008esvrR< zW7#lZp@tB)#V7@PMGm6+DRfSoD za5ROTQ@G|<1%4Q41ZYouMHM}D>%g~;wyYW(+@mtY$Ub=SibgIdF3{2vBIXt5(xTlO?_V+|T2YV)dn z_U}j!-3N2pXaD~GNx9Adwq!5JLa{4Zdcgq^xh&0m?>x#FnS_81ybZHy6H}Hkle=lc zAl-PLkI{=2`mDey*$Uz?mJ{2CI!llg#F*&e%*mh*T8U0>Ugnc9OeVuGx!se!|U+uOGNRuLZ^dWhNO* z!@er45r74)7^^%~`poI1$eDc^t2qN$MY3<$i=U;N2ruVSyPteLE+a1P`{|eXWSI&} z5~jjF)|@mIII-O$IpL3aHgsIOH#Q8f&|Cvq5N9L(QT#oQzo+rH0e|f<6PCns#%`rb zrOscXHSUmB#avnyON22aVO5M1b{Hkh{^b@~3AH`)KP>aFs%Om#BwYHbpZLS<^2RP-hZI_Ox0rWluKi&l2bf1yrekx{^TPgaOc7D1n|< zKm`hj?4U?~mnnf9Oh~E82Oyu(rGWN>Lxi!4jA8+icPvSj0(2~Zz#6Y8sAgp6LX>9Y z^%m{-g|A(L==TPHG0=~0lUFD)%QJ5>g!)7UI{YWt( zEHNR?l|c*%MY6BoNxpu?d}J@iLuiRZ)0RDs`h3QtSqf#p6ksTcw z#4b-{IVV*5;i&^5F3lduO0GP45zdeK@hgyT;b6Pr9rqZ?6+B^0ejv;9JRWUPO?%2a zE^qYyz5WyS>!CLL)L6s*Dz+x#|3t)=HOWWtsxh60Czq9ly>H>~ef)ibKZtHVBbZed zzTqvi<~Jk@Iw#^sE(9d733nm)7KwhW+fN&8#7kedUYz!Da;5OKbMTW#Qyx z;yp-=+e=lJjxFz_W4Y)a_R{QFP6bM}>?J4dr7Eov@l|OrO{Z>k(r5yiIZYV3Yc)QZ z6D8NM2-m7LNC;dYA~*;OCQy~M#O16+B6r3)qzB7@t|+G&#OX?44>M@Pc#Bzj88xY9 z&`DqxnnBOMtjwT@ggAm3^mCcHcP-rwfK+BsjVh}z9^F4gj_qo_+og17P3Ws!@R%?% zKRLyU`gr%Tqp*q++6#LkEbRr?_G{#TYs(!Fat2xIb1n247plSjfeU4(qx|l-nIKKu z2buQqOGtYGX_yX!KRcT&9sXk;$1K=%C4DP0P`h^pC#o28H>W&9l;&1SV+TrRHu&BW zL6~$=ZZKAXk?#fnhj}`LSZOU|w(Z7N3f z!ctfYsE@{yK!YXF8VhKR0$PC;iZO~Ey%z^xk??lcLv&VuWpiDInkOo4+Q7^uEaTG+nlEVva4 zt}6v@@4#?ROSonWP95j4YF`SRIWXK33Af3DbE^8JL{dF|7p2tzbeSUIIxIL-nGcF7 zWvAao-7)~2CgHI01-+s*u2pbOV5m?U`51 z*PddEG-7($GmwnY%i{(ONGADIx@^AZ0YRR^Nq?Zk?C80n(qjk2K5o>!1ndvP(KC+& z>ZH!>(4tM4ZilHt=7^qScstfOgQu+NOjtN$VBvJ^@5Hi`eV)*J8G2|(mfrGlIt?8+ zTN^iW{7p1+sE3QWHeu!pqEyZQA>QGw$6pht-0>ofo$K?eP3|+}(j_QhAG0Rxo!46> zffM%5w%3%s!@i}xbMDxIx+t}g^It$C&)Y#TOI5+%Aog_5XwCrJHh* zgwqYN02Vfnj2ClNR%Jhv=VHu%;EkMH6eaNx!@aUObL9~xy|rp&TCQXa&Q4*StD4Kkeup8?2=HrJ zWa@gT>t;-74VN=}k*RLj$tWIcCJ=-=U9^)@7-a&HEaAZJ1jN>2?e4KmZg;!+78Yfk z)@W&SkHz9`;ly7){tDaN#W)YbTHA*%7yVYvHJf?Yq43yp4se=yPV7I@8bTaNwC2d? z`Ef&T;dj{R<6yn{T+quTQ*lxinVJSZqDfY;9gGk!69Y&Yhu5CRuaF6x z?Q?4mjDKwec3_MU`O)f7=?ljVRJ>aL?TcjoxJ8zYF3&+NX2&-4GB9@2xEu;Yn|yl5 zfJI0?g}=gZ*g>#_c_(32cmh4#i+iLgF5?rI(cZ&(Q6CnQTt;RLEGuXEyg0P6&M{ot zdx&Kw?Y&{L_I~!>BPv?40sfFV5;e;cE)|3RfoX@gB5esW#VvD2GSys09jdm>a|S1< z{bYz6nnW*n? z=~dLna;{VWS@7fPwZ@;>Nr-r40^(;zA3Lw2DVE~|2vI4iXwn+zD2Ns0B1$O4tMS-d z4OXm(~>R$DFA;c{1V@mQHTV7#x3;JRH3uu9GV| zfPRP6$j?xRmsE5pgeh0=Q#xP(($c|OzAOz7PCk@{ukA8tEWvk6FdIFA zym&y?Z>nvDj4>eUgNo%MZ1b+Gkw4EI2W0(*R zr$U}KE2wJ#1$nx&muRFsHT%~Ed3uL}g6~3s`bz~x2?Y$B0Y#bBCv*#eItNfn3PYe? zqktR=r~^P*c1Jl$SmxSC@4 z8Ps;DbtmV_vcflAM;CmTe^v&+mh*t$=y`{%iYia!lse) zB|2yH^}l|>>LS^u^p4~%+)eJe-RA%)CChjA{*a3gaRayNsG;X~{>-*WO$+HML(dcvSZxCL=n<4sCa2w(HM6`u=JXBFPZ1=CN{`QGPhrDyFOMj|af!^) zAF1GSkk7L)|1r$!*Iv0X-HL3M|0LIZ?J=XzD^7T3KnTm9iDb{QFnSKzQ$p8>s8=G6 z?oKSqO8D1>1NHArSxLYC-KzRG>shuMVI7oMIJf%;tV-(YPh`wF@u3(W;{7|@>R)%i zsRfLU>+?h`C;d+>HQB$jVe+#+_AmDJg0cGcuX!vXQu`N;M~duUt)WrAV69AinN0s4 z9fTg$o|-l`)SVV^Nxg?f>`0k((wZe&10wIXGg_J^odjJf%eo{VM?w|i5tYoyirw!V zg=IB=z^j4?WFGE7Zfujw=SHo^CgQ7NicFkXI51QE9TG3j6rFqqQ@nD_(U{@_*#mpb zJ_T|BwC50Qk;^%-=xU!lh|yY-H#t;VCM&>M znY{xpa=%g}`@lMsAKFw7%dmSA>yX^=c1r4Zk*ZkXUj$hg46Eb(28|$EZuR($^JJ*G@6tR(O7osiFNq^!g*u_ zD%Kj`M{cZ?)*8@*sVwMb5yHGShnamKW?aNHuy{+v;MNa_7%sus8Mptrcnq>80xaTu zsWosWwwJW=^BCAxj!~z1@tg!3JrDC@U^aRKk~PjoyZ8(?S|M}y-PNyT3)jd;vOi%D zVJXL5I69YF=Bfc95^S_iitm@rN@V98U7ZY=P_LjhaR#s^E|0=bUkqkTj>J!o(uK45 z>2qwdQ2g}sl+-tn`k(UCFJbTgXZ$o*LjIflbOwOFH9zG*qI_$90!L!AgXE{u#iW>8 zMR^o{8VSq85=%D@Iubu+L%_uODUHwIC%4S~ulQ*ddkFk=5srf-KMfxcBEe7lq`G+7 zbRhfD_~~EktKVg^A^r4Kg&eclv+1QJ_0_MRj&sySy+1VJ!?)L07c*)NAu*duOA1w( zaZarfG)NT`P6~?I=7bT%ftH{W6oE@{NHpw_-GN-_5||*p1akL@OKD5&z=pE)MTi=w zS_fJP-%vyLLD>1xYGqG+#OG^5r4LDnzI}8_iaqg_Y%^B(^{(*Fr?@z@VnFajhqa>? z>XEs~+@}C**e6WLM1Kt!zyH$ChV_fz->Uj+4c(>GmE_$Y|25uaclP_X8{dv93gTLY zwnkkn=RTNlNn0Zx#NRwX&*hu9Ys{OpG6FXVV?KKBBkoO~3+uvKR@Zmq{AN?kg`iOe zraV5*tWJzP9y>pydBa(v|7&u5IhHkrF`uclgwr^4N4&uo5Djl6pYgh+JcGAa1? znD-&O6yoeni1R1&>PdrstM@S&bE;Y943m`wywN#-EH$Q|e>62=J_oZI(nU|{UUIp=)%r8w}FnBQUB`hZK! z-za}%Qe^N8-Yi%xJ&n>=@ez+S0K<HiRF5jMLa$|aAua#%-P4Tp?2|AG>*1=U0WW{_ey;;%Nh z|B8wJLi4xB$drBN1L|R|`Exw1@EDt-#{-8m5kQX)#)BOX<~$U|qX;l))CN`f(Hp}Y zw6BgW?L6zIT|78^^`^q0$y4kxV(i z#sexomlc1sR_=AeDrYpX^kFN@iHq@od9c%vg~06YSk4IW6IpL13grgeDd`kn_^@s@ zv&qN5$;Z8jdSFG7N_v1v@Qcf=lJd=-EV&mN05C=|VNOW_&j#D>7D#X!-UT>iZ=3lh zP8zZLag+lD!v@^!2cQVe^MUsUDIHWL6#z8aqiP%fO6C}a9P3mLk|ge6*-Kj0}+aT*@*KYIlvT*Y!L zfEt;(JeKq9uh86*isixap&onfN#V@9zeWNvdiyQ}CvG6qhDztiT6x^+vrweweSbz> zgUz{MX)OOhW#*G|==u+se^3vvnCTC()iXEHR&gL?wKIO$_Wa*+6B!6Fh2f-Um#JaE z-oh}z5WbbC2qyTS_s=+SOr2QTvI1GmX|jA=)i+T+xXoWa$co@&qoR{=hHi1ZA{#1V z_fE>kUm^aA!6xy80WFIIr6qM7S*zhwfAq0_nkaHAVv(x-5Q?aHrjJG`^JS_|7`r=$vJb|r;ny1hONV^rO1Kpz$ zH?Y1H{#rvlNG;Z>D=&Y4OuP+0Ud1-N`T*My$+%cVCeZX5upyh6$xI)xAs@om{TB^+ zY~;UaNc_GYoIsA#k+z3n8ioqiKlGwNqOT#efQQhhA_{w8U`a+OFVFc(^!j$%)^ja|Nc4bogWP z-#Ff(qr>7t2AotPPhl>AoB{tsx!`UVUZW#$jShq8D2MmxV3jG(g>QjY@M)7ee6FoJ zuF^3VVf0~a;~)pz>;dk>3s$TQLy-L#5t@_*W=}MS2okd!`e0V!GlnKHBbq?0&mBKM zdx*%{{!Qe%sQGcW75NamY6i2j`yi$6&vNooVlV?F!-?_$YwOy9xe(d^lA9oZra_s; z2QPlV)%d0SS==?4|0U$lAHc$~oD=USpUpok18@V#pKN|Nr!7Up`mx#hO#K(~hyKpD zlRt8N;8rGdAMU4t@I0g&TX489y03N=4qA4|(UsD*xGH54uF`~Z=#(9Z8xUH2khl-E)%Hh_EvFVm+so8xU3mRh=a>I z%>zSJM;7h&8Fn9vwzr!XAp)BiMX7w2(iwQ{zH)H?{z;7gL;Y35jOHgG$vOmw%e>J*)aVIiJ!uK297QaY*c)3*3>)0<%HXI8!%#Z8Iy;Pv#xw-gr z;x8W?TG3#LH`I$K9MynSjI9~gxe`?pqixv7h;t2EVXv4{a)>MWqL)PITW228Jwnj#8^}9NFT5AVS zYn_V54dI^rQ+3Gj5+M(4wMGAqb*>J*WGK#UmFfp9jdo|X`aq%hGg~g@L*i&Ur{Cy6 z!_j_tPy@6cBhOdVjHY@gqPQ(G=Qyu1>+sG^9cw~Q4YDh--&-w3e$}>>+Nc=Z3-APVTNuQ(9-K;e`*a-H5R_{8{%<|pu!jaa}wWFTN?O2 z36o42(Jfu}K8Jj)l;4sFH_7)T!gcc9LijqBxx?JOa6p86i^G45-q=i>#6Q)Z0RCUx zrRXgUiQe>nRenmMzeT<$(O)m$E%g7WGIyAx21dURy#p1}-iMXnyUuI-)H$YhAKbD4>+#RE-n`9t_maq;ZWyfL!ghr>8m&J<_3k4V;R5Phh4X z$odLm3AyH7_i`Lg#xfL49WazxjzyZ7Czy=V;3rOY%9Vybyz`^eT!PRO5vutNo)h#$~B-y?TQ(Sb~YSKmjz(5nt{J!W*7ky;B1vn=IEts6rf`0%EM=#e zd$NNLYj0M5KvBqp^WNl5vv$Pd?`qBX1L@PsF92U2IkRl@In|1 z3)paI2zQF(XR#qXRR&H{@0Ut?aR^;ts2{vzfIdAF`w&49c_-UmnE`z|N+KcpT|i|X z3b65zxP5LzRrZQQ+AF*WfU^X+B@P~%yy^#o&eRJ`?&RmKZMgBXaf{ZF4X#yJx?V0V zf`V;yC2=pR7V5%)gx#M!U{7!P3UDoYK`TzrlODovf5u^xLzIhl<67||e?bRtbXE7c z;znQG=c>E5AYRs~yY|2r{fukkUPchn2QpS8^x`W3EA90e4SDsbjy2%!lf9k<*e~&l&~64=4VSZnH=$JOSUB(qqh&DF8RD;$ z9x~VRvwEB)hz6bqYg(kum7xP}=#lufV%7xikREErRV*e*+~L!vY?MbFH_2*Q7Cd?Q zy%9Z2$DIS8Mr*iktX?8` z4@<8_kAllF;d-NQsvvhvod-5j+VZL| zwdA;$z?Jr~CCV_=rB8ke!`C=5*0WAp53o@t(!*KWDqlmNqy^t(`Ke)(M48f=;%LsM zBcBZRKpczE2Wi1~tvG%h;9u7-wJGY~$Z2`&Sl^@6PyHbW4V@gy<^jz7UQSLY?@fSd zKCqL6V>YdC{2|Ba<9E>xpwogQP&1nS#;{I*WNat8qztU)+QLIecF}b~n8p~19tj-h zF-De@Tb|1sM!CaNGE!*XmMF&}`c!K~&==wgb3{}vLKx?8T!B~d4#h;tY5q_~S31S| z$CT-Wg9|Vi!1-8jrSA2Tk+W$iYP&ow4H^y^!t@U5S~xHt;RF7DV~9WEahN}Zy-p#B z+Au!pGfH!rCO$4*oRo!+iNttsDf#r@q;Hb$-^z)EQ1~zO2TsczDH0t5SF#+gXFL-phu*=^k3xn|1bB)T7*Lc+`9Fg;le*i z!QTDxw2b+kj_~le>W@xo4*q*1gz_&(20(ROGQ$99OO_OT(WL#T?&Dpbi!qO#9SFCW zA6yU28(+tH9yIrQxMLBHb}oWnGMI|zk|Rjmr2Jf8oGkBw|3-PvWO-BnFO-++?s+>Sq|r##!OXGE9^{o`A7Tq3L+SLEVWG`sGa zm=(kcC|6`fv!)tzaEuwxi`^R#9%HEGpIn*cn&t=$Mr!c0XrT(<$64}^bwIT*UrF5!c8^G(q*Aj z!@hKswAFkUme2|j{=QtFZ{VzD>rwzqswV_^@A%LEz5T&H%Sh>`%#`OrDbIud{r=Mt z3{3Q&NOa(*N9n-l=l{R916#jc2f8n#Lm+UmC+6c3!iFF7Db9+|84J-Sj~s>USeWs- zHhZJBoMbB`l1$i&!$@1Sv(pwVV8k7wtrn|5M!yUm=I!5cCr88FW-V;+-Db@`%Xu!b z1tw9%YDB9u)J3L{Lk>a<~Up8+}$qz=rLKnvD3804^I^-Sx z_?wGAX9_T+l7wHL0*@{I{o#`^N4m=Lkce<=hViQT!A}Ofzw(4d=rUTkiL0cgOjPl- zXD}RW0LFg1Tlv6ftXdc$VyC{1$Y)Z1rml~$=G;fea=t;H6`Z4AQ9?_)!&BIC?euc3w|l%TahYWA|4nEDFO3AcHt1~ynnbG#>?`M1c)0{ z&HG7#Ew{+f8|}aXYxBffev#plo1yl4$4W%_@_uOv;^0BjGi7cI9;NYvYP7M6D8U8y z$TmWG;nsl_Q8+PMaZauzSY(>h{M%d^#J>b$Z5j?|Jt9P@MdHHvz2I>i684ar{wLN? zU5;xku>_G#Iw)_OhwMS=+vE?OD{@pm-K;`$Mg^8G;s6Dt&A}36oT5;OQxtIP54hHt z+RoID82WaL;XcC8?M6Y1cSo~m*DNuS%+K7xEFaWj*$y>zBkUJOEIMh`drQ1P7y}NITZWlUJS0VcNIb!=fH4*kBSrJ zBHqWfe56soU|pmeZ9*p*-kb1Khy`LlmgJU)d~yleuJE!yt`%dW`dxzvdWB# z8LVBzFnZf??EjnE?KSWMBxTEt8>WD$S_94_=7JRjtLY)=5lp6HOz|0-te84AjDrDr zrl3094h!0XK&;|XTnmKL?HXU#8vcn^hi=WcamD#1&iY%%Vw>+eq}AV2jKfdmkdE+R zyx`+RyOIr#u)K<=(a6U=IBgWkv)T5D_W;|ofe8prvtphi7V&NX-mwlnfUr~{eL@nrT6tU2tWL|D{=NI;g^eL1B7FK(*nV)ZCxAM%>8aHKTI_ja+@x+oUR zGqdErroc6*%quGi%1T%zD?qtKQVuHI%EbB*AM|#iGPC~Qsi+mstuZ|dqdMY=A$EjN zHcz&+#}HIMeK8sz-$%sGs%NJ_{`9|w#InDzi#92g5NN*CGt&w-Bkp={-fF-XA+iPG z{ZQT-ym+~j?YX?Qcr3#lxzVkM(-|oW4=!$y_dI9~$6!qQ5O{4u9!^Gw>hf(tSOIR( z!5+mmoaNS7c@2!K#tI#5xI7+vvj+>qLoImKW9f1xn>P=HHT8(-3`ofEunR$2?RK@t z+=qqsk@KI!f!kD8Rb{}m9##7ySI2zzcX&O~CUM|Xfw1&SCKtTp3&ZyKCQVzm9q;I_ zA=-#xd6E-&7fOfUlN8DILxe?eWG>BkKS8Gd7mTSv4=#J1}8 z@?0_gWyHjT)gHs0?=fEVxLOxy8l4`aHPn*{!Lri0b6N9W&=d9@v9}`=zj=%+o_S3s z(8ZFph(D&=F;~?h4tyC$ljjD6v!dgX8_Qf5yc9hqbSP6D=&ulLmMQLZm@mTTkX@aMjLbh zZ96zqF@BD~(}oZH;}B7V4= zk0XKmv7ij;?n9V`?f#OQ+#0MpQrvrZ2B%RTksOL8=80*G`QVJ7pf5L*g9|>;!)NB{ zp|8NYejUo9Bh#?Q;RVhm9rzqvtA~f>MNhHV_VpCD-RXC2 z)*5dHc_gH649g=>0XodyhR8>*wtNfXb*g!csIYlzR8ls>7KWT_)*7&)1$Sj#67xr{ z0#oev*%Ls4s( zfp5{V3Tthdhi}8<15Rr^2k(AWfAIdb2N&{y$jz1_-(-7-S}e~<#jI3h@T~- zf<(v+Q|-8`eT#NF<}c+=UrYz$GDGY&4hz_^!Wn}xN~xPR$pKp87#B|(6{b`{)IZJC za#-qbGFCY>bPKkjJcamLS%>Dh0G2Ft=xva)rw2P>QSKEbld}Tf;hmuvsux93vKQ%f z?$DBa8)jJc<65}AUaf4OmLKeF*b-O(6Gtjd=sl?<%>oQZ%`rxEYy|?ib_xjWN_MYA zg(D-)lN)1v3=RP%D`24FXk#pU%78*I7jh9;I#Ur6OOuJS1>J?=mw-aCX_dI8k(%1t z{J1Spg}Q@X=HnQMoMl9rsA0-~Qo|oFRyDAA)h3jCp=8q$7e+^EjzQ8GITa~*_Q14a z#G@M1Zt;0PEniJEmwU)BGCtr`F}=@3s4mx`aRUal^};&Y*@Yy(J9w@z7DB&WUWH1f zpAnkQwpe}!;>28FK`#Tfx#_JUv;v5t*J;aK_~jrW<91x1|Tnaa&#?k_`8O@1WT9-|MEEy8l)``ET`8(8{uMXtigIu;J^l4*{VlwfxLi~(q21`3%KD<15&qdllX>A zOkMu~uo{@nNz_`^V6I2(VJgo(_8y|xgXv9Qhua9`L{CBo|o*}jx| z3i3G?UVo=7Y5fZREN!9P6w5hYqT3pN)|4Ng-NzrY9~+&zbVT)Ee9?lNd$bB6L1CNo z0t~PuYLvwORNv;&K z+WdO5#j}&uJzAA)@e_DnVJv5hv1OnpA6uWo@(s7ywl8NR`A-=4tA*08ij7JlUj>q@ z#SCn~e;5Dnq*QY*fe_rIO+I-PD_k1{lDQ}< z?yDk*R*xWBJ(YaB*P%qs>|DR`sTUVXAXwni9Se?;dm_7s4IwcaP0+;1GFn>9^t?0% zT@XO&Y3a_ae>)B9@@M0%TNs15a3nIhxG2_uWl!E7c0wgBAc!SgE|U(l;ygZ_G{Yu> zl7~q{ZP}C3;E>(nGtS$~5G3OWh#eplfZ!5?NGK1g5T_6_w&O;)tJr2V?gKbp3xQ)7 z9{DLB&wH(Br_Y#(A zjUm9bhp@+RA9u6^6y$%9^UiA;FC^c7^e)9#+Kg0b;E_<9Ec{ zZ)`6!{zN)>;r77VBHBAXB4^1h$7SH4mTtQTT_ucBZ4+@sZ=6=?zVHvTJk9S9E^E34 z75xMUJv4U68YUtbR0Bei*b@uOnx+`@3Hn!~ydsvWCz2TQhz< z`@Obwt#K3r8h&H>N~HK*Q=D2T9SOQI*HN_JQ+)Gz!3FT)5#6k&0rGv_Sk3`9Ju<9W zk9GJXdJos?bK2IwMnt@8d{}~t+dX$-f6Yc=fS2X#i?x}64!!^t6(7$q_jYv)*VYBc zPK#t*0FSl?Yb0NdwRvLAa7^trZ&I&Tp(?n?c)+McB+>k57SOI=a-J=C8s`5_4IReT zRk16IAZX2)SX*k&XOXk9x$X_}!a1nm- zcjlcCkoxuiJm2>`9eA_$W$oMAx3%|Pdu_IzH#|tZWg^4;5A>6J$&_$}jEtW#l~GIk zk(F4by{lACD`Lmo`Q_eCUOG&jr~Ar8hZ!TM_jH)&OCsOVJ%uM}R(PbRe47`6<2}XU zMLN;jIdE)G@pT6?y@!82r8lp0(?a^n$`Ses6q3HOQ1z8E5i%YVy+GawJ$z^@J;2L` z>dr%HddOVv-0(%*3VXXZ_Nt0|4tDFX?ig!XV|MawGO52A^Omy6-#Lqz{}pLc&O1s7 zOW%|JZ?=E4ANj1&a~q@OcD`OSl0M7_s7BlwJvWuJJ3ivzDTfdC>XQec13Zk%X8A>o z2M5f}@m^s8(}iQtD~}vz3wN75qPvzY-(F{S#U>p3**<^3l*yUMcw4sM8M|CNd2cGA|pz?`-qCQPv}0E*dm@U5li>^szaB?tJJgHw#8U;6AVJ>18QfN1qnM5 z-9np6H|uNOvJVyYjLr2f*w;}oRO0->=W@e?&~G#AW7Gqd7SK{z7C#?7_m5D@Wk6QW zNL#l!LT3#AS46cQxOxS6NEFtqlbC=Vr4z{%4siwJr_u{^}W|6vhhpIw54!<=WWwEgYcG&GS5eU z{sDtn54SNQz9Q-01v1_h$&gv-o8RmXxj09r&1tvryMw1Y^Fd;-hrcq%(Z)n%mY9A4 z56c6DwQNm)gKP?iWjz5zU0&&+1{4c`)O8;l?Mu04f0Wu_&7a9Tbn28C z4cSUDarC?JCk&#S~fI$1(EFFllFX-3*WB=FVyEnkjM>rsqmx z){$L9WM+@pL0zJ~LXEC#E+T_6wUXN&3nDQ$kLmWMsLjCUxKasAoGP>|Rg~+T>8_hgV%H-Viobp2O zPVOvY{Aj3+~h7&caM+`(TnblLb;Oejk47u-5ua1~Ky zu|z&+Q%~eo>R}u{DnP8StUCjW$Yumjfof4x)qkaMdOxXT6YRiiRB_}l-Cjg^cGkvn z?zmb~ThbKE*&rsiO3QoFzaLPJv|6XGe3z-JC%)$+l8qp`+66J^Ht?ejZ%Fo)^NlKe zPNO(iC}#qy%&}dh_D9rU65ktYlJzbH;{a1_YAe|o-a#fuXoTaOhm8L2Dj97{@@mUx zEpBpNefWbi=ee?=`T*@oi)%k{es+jWQWrd`%(1Ua^UYEF2`h+%8p;&0oI1?H)H(Bi zBL{x3Db=$%&q^T>%Zz^WSFzaP#c3#*5o;{xnQ9rL;vL~M9h2IPIu$dcmt3)cTMnqR z*Z4NCwG*Vr(UUK46-A7f_s=UnU3vXPVwLws%*U4ZuWe>|fB1TGd0$tBikEkmj!BjG zY|JNFUe@b+M~Q{GH03=W(Q(EeATPytH|3((%LEoJ5V+6PHDScFfYasw0csbpJ}Z`6t7KPu(=b;86l}GBQLK zr2z^e8T`#Jv6HQot^A&xgRLgnZh0-4Y~NC}iIeR^a!&G!Y(tngi%P3b3WYA8{IbaE zg0e_Qk+ck5ln z@1F#C*TXJq!h=QsOO8!6zjVmf34)PPFiktgu zB!;FqX+c74?h|uvQ_fkj*ve-uvS?zB8H>y@W2uZ8vaaRglQm|{(1lPmIS(^Z;Bf4{ zp|cK;a)V&~Kbm1;v1)xy+)BW) zHnMmupc)x-KxJgHG(aTD`Rnm=6B1ZdrX~#9Wm$ytFj{7E)v)@@c}N|uEX|C*{R&mH zNRMAEm~*nE^(Dry>mZ0#HCJP%YVKC_6s1#(E`}w_*NGs)m zrM-R5#P(_0=a&gbu2o(9C+6d{&p)U$3I_3C^XQ{}5KmW)Fy1?_($T5jS%vwN+owmh zPTe-uR0~hx(~a}7yiTd#NV;bmKpH60DYVYA2rK}b{qOS*Mx{BhA!qZrJJG$Tcq z^-l}Eq#4;8(Mh+T;w$X!=Xacbewy~Pf>nUNPyy>5^ykWvw-N8{@Ow`G87dFEsopSLma^< ziagYxOp%vVvEwZuPp6HX=!};fMW~MUE%+TKdL3&)fWEed@y8Khqqmj5QdIT|_ zJU#C7oAmhi7n13*OA4KsMkneN6g^y+Pl6sv{LO&WPBMSv_%EC^f76SveFhN*$HSUF zmNO5cm<-NlkYYJ6l?jzJ)A0{7I3D7%s8o)5BnD>-oB0+$9zFFuvFN6y7@W_GFAUCh zDGjg_C_E0Vp5}1^n#Y0F5grFtM|d1q9pP~*6_3M4VjdPxq*Pe*BCLuzDu|Q{=QErI z7f$DVIcbvl=_rr6*f|}y)I*|*ZiF%vPgPMXW;*fRdA;ONrl{+g7-fp;7yJ6B2LMT& z4(&IozT8lMSbbf&RiaY-sDUq<8RETL>F`(ewZRv!FG&i19S5l|v7KNMdxf=#wK1fd z32N2XgP2*gm<(w~oFN^m#T|xpfas&EW{G8kvZ(rcL42vFqxcH!1Ioh?y<%63B3+ll zc4{s=vS=)(t@y)&V%nliX_#bIXPKm19wOJ6Hi=W|#h5yksfZd}8!4R!2t-N?08Nq7 z*?`(eX+EG@*L|g~d#Zjaj!k6?WIe=-k9KA+S5g_W33YCb!;@_wt-Ufxn{pQgM&BAiv;otTeP-VCd} zf8U#2-kGXi<5j*!N2kgg#QaYz@1rI5@~*)4Y0CQx!dd0rkNG&|y|C7--*LmrEA7fw zzjx^9RC#a5{7)?Ju3~$6{n$QDd4EYbtGsVvK2CYtxb~&m^XWW!X>ZTcu59JKUq`3P zdk^N5C@-f4Nm3(n$%HkWx%>E_TEUk`-ezl*XZjBJ=Q%bxA}RZ$Krj2H3)!{e5?$BL zZbKc(x^Qx%SM&7mli`hW%F*ZR>BvnsBNl}NZ%vrJ<)P6jX_&K1#fCfR)TE#+6-1NIdk=iueE|r~VvY(;7|Jc1RH*0Vjn8(W5|_tFMqaL6ihC-?^$^tvjbvkQ*6(I zCnZqu*{&xgO%9}c$WGFIgVM;y3nLr;+9X*@?d3#bmwID!y~yl(Zp!fOd@q$5Hu8{U z$H$iZbhV>M=_l>z1;xB}+2aLgeB%5-yka?>UMc5M%H-VeRa|^~^i)b7AVXIt6qYGn zW>;SqPf$A-_i@bTWwi}kk8r?El%(BN_#S`d7Mq(bGn&-m{O&6XJe%6*p+>u?X9X>! z+Cu~pUa?*+qNBg?b>FF??0#gk#N7Re%;;&yE0o6y-}4wvJBh3MpP?KT{$Q}6Les}cTCP^r;BT#C)QO8V~wQ8G)=Ysh?Z4+TvY%{dY zr)~FYTcx($pl#LK_HWu&t8E`aH}ct}ZGY0X05+*O&h>k8zQ_?2`ymoa8fQ<=cX41o zTMJSw=Y18Tt5d>@AnZ&vb&QhHWfVx7HjCA`Bz8lkvsk7K<*(_(^3=3z3hAwV!Phie zoNxKNb@&3yuIaS6$g*onE%sP;O{>K|%kI(PD=oXG*Wzl+t|_*-*0O7wEpD>xwK{yj zvTM36UTfKfa z3G=?}TYpu)EWAIGA)$1`i@fxM&Js~OzF=t0sf0)gXUj`AP23UncJ1XoT4KZ55$X`n zkSVu2L)GyHb-~>m*;`^RpoMW_u(viKSUieFrY!Mc@+W_Y^&&8uBJ!fLo)p2+P30n0 z`>b59cJoeb^i2XB`Wb&pB3){%?0#7DgIuFmlOh5Ls_XW!Yf)lec9wSt*+DPjzMkG- zgWV~Is_tkdo5$2dEi6vu;ipnUqm=CNV5ODO2zNUfsm<5NJ#w@*Qm%Kt6VG(#+g6@8 ziG)MsmFR3KBiWJ(QLbN0Hf=|YZjKi@X8dh@*6`Who8ktUbt+rB%{-N*Ed<=9N@QBp zN=81~Q`!qlq7=FSLx` z+gQQaV3%K8H@Q4gQNU|`Uf%qm3a=l_*2C^YXGl!uWC3@g^!dM4poYmtaDFzDZ~J+ZZQ$AxM+rPn{0H`638kKBDkIH&0T$>P8~h%gHB z`Jc1lHVQ7+rxIt;9vTXq`S9nJcy$y zQ}mNVT?puK&?5>eYIcd=*o()84wh~>7#b@*{dMH`a*Z&O%ACJL?sWQj7(VpDyiOrE zF0BX?J}I=1+`UIek&>h+^MQ+!Am~pLT)7J!K*B? z6v2O|>isxvCno9iaz8R(!u`=i^~+CB7h)#e^nTy;6zwG=P%ITsIkMt%wZEmp-x z)3&xaRWseVK?QkC^*NnMO%({A$ZO=;VY%(&vUn!9s>_`VcS4D97DClLz+Jd^FxC45 zmXv|$mkC9~Ejfzc!8tx^V_u8}gp@5`bJ*Dbh;#T}iCNWbv5qLkD5=$};qtO7~T zcze9gy)eQFe$1@Uq^#3i!st5Fl;*OLEHwTvXFiO^AF6%HNEtszXMDqz)Pwn=m8QLn zduncDLC`65NNRS4y@RhL@lPn`K(`)4T6nyu8{iPxlu~s*ez|(7*oo%M@VP>(!wg7+ zw2{l*@P6S5Eb{Xsn{-llzaTSM^g}saFP`oQ&4x8`35STj9?eV~M_NOZNlpkWB1g&$XBH&l<=Ye znwh$uiIpyRCN!sH!Ly+&xdlDs{K2)k;Y_b;u!9*Tk-n11Yo!~171_mAU!?r|yr_zO zu{5Edi9%=kB2)9qx;?wCH7*KQkq#(8U%_}ag>Ck zS)Bc)8(-v>f4n@q!oQ418k(T8kRfy#{J)c1A(#*=k>+$s)45T^Z)r;uN%RKUs91FQ zH0Opb%6B2XdDD-_w`1{UX*=2 zr#Cuc+r)Rd9zc(GenP3Cstq2Mr+4IS?3q%4UhAE-Lpz~1;y8Wi%S?g zn1tfo^6u4co<_;XjUU-k#W(1q&oQ_-Ce0gtbe%e!_>S?V~^~A$N9|eDh=Ka5Gbe_2Cxe5aJ$m%$H zi1FFSGn&(Vk&P<$o{2|&vyPwv3be~J%OksEIqBq!>nP%l)G>9GX1i6}&v7E{MR76v zFe}ak-3pv4Bk{C4i8WCiMWQ?G^O&a-$yX+;g|V(xYnDU4p7laPb**@^rl9C;i1S4Y zZK*Gn)J-ClbDfNE+~9hOcU43Yi>MTbOlQl6pmm?zq?bsAA4{*azm%b%dy!oGTJU`E zQs092gY!+T=4Cx=krLn2F_}tuip$|klf=Vcfvs0p!Zn5O94Fa|=XeXg7QG(CjMrgN zHu`+@XWTnMj|@%AE>8_p$n>Jy81mHF6jCOZYJG*4eFvNT-O1daJ*uVrnLg1_vQtKg zr9JcZ79NJHt7aX3r!=Xa^#=fv-xQ5R@)tOdvv8)x3t_5)z%cbXci z`;=nK-7HdK{Zd4v3~MA;Y-Un7vY?Ue2L{|DYQXK8_X4*Ss$;aBo;!t*iuq@#v}6){ z*hnH;B?tFa%sxk3b)tcMUi+#Jb&J|^qBtI@ZlCNUR*Jy5d9POQnkG7w+7B>wioT9W zuJiFK-oo0?i@o8AowvX>jqh2=vQTS2QTjV?&GVqh-MQgAWD~Sd&USQ3GT-5D!*f1f zJdL|nsDSNt?id=?b@cOGNQ@LSCmJ6YPor~kkv=)Z&n22;ozbx-`n}j7k~N_^KVl4q z?rv}Fo?M8i#$PV2b}wNXnC4(VeOSaWZ29F##rX<(-;vN+(6S|+GPv_}#fqc}B5Ac_A{m|lt;TIN+RGk=#jxUvWB zj~u2yF!Z4ZN?o%l>-vZezTWiT&Ep(pJ!Rw3Om*seDNiCBPmqm*6xi}+Q0%=K^WyGvP z%PR{DcPeKDIXcOVH0oN0yctr!0&e2;KndA0jNPkuxEu54ipaxFO+$$4Mr8Qyi5B`pq0(qn3DcRZ=RnftVWE#Jq3{51wYf;v7BNNc> zj>)_u(OW*Nj|u3xqtbEJ{iGFSfT5`@@?12FEZ8QWGi}+NF4=TGe#KM<2&geeMhW=) zcGbQX!_jUP_A1`=<}ltC4CZuxewqBhSer zH{QsrTm#aT#0VxDHk7&t?+rDdJ=DsgH>t--e}gq_$%7l_BuIL<8%=hi6)N&kH%Vw)Vmjnv=Ni*U}9+s_l9{wr3Q?j4|Ms_AGMnsC`G-`km5?OE&HbUF(fF zj`Vvs?h03W&wMJkGf3d~8vet-`Zn89iFIu;N-) zrCNQF^sn6x5V5V&BkbDUR9g66MfW)@orGs#n5Ma~wN>6dr=*BuNv4|$j!I+q%C5X@ zRo+HMxTPq$W<+7hBC|Acy~+2#e19w|c7#@A_#V5QKL&o0?<;(J_+H>c-;X?Wz2Xse zK)-?g=X*-OQJMDj1$#>G%w=1Zhqjkti`^@G_$BjO|DCI|d%SEpV*Uzixr$e@>|{Mx zf!@y2HwRrL^JeT`AK0aHdrRlCVnfn<1=*2v!^pGD4Mm(EBL>hQ+h8qvf^bKtaxs!c zOZohVHW4)Bj@&DI2<$?le?dlYH(RTmy^Ckm^Qz42r z|4n2>>itw?_`x+d%1QhnbkWcBfOP|F0Uul_N9;IlVdg#vmt3BQ^#(s9+zF zmY`S#bvHgb0gVAC+Y)}y?$=4S1P_@Q-V&~c;ZZAP)e?%1X$gyIl*elca?CYyOZd|0 z6Sf3BAi#+w-ZNL)CE0jm=w#Z`J8?|4N~sN#(KAdnMI6ZvOebQD!Yg|iPKblni!M|c z;W8R2FVO8@bG}FssdIm38iu6dVpQpbDbJXoq7=l<%iR2Ndl*0Gq z#mce;M`4ooh$<2e?U!XTyZ?feKzS}&ajHP}>QEj>B5YS`y*#+E3DGa=#erOqj82oh zt%f3!zwQoZQ?^{XI5$k_O#f%oV zSaiW!(4A!=*FBc4c5_09&oIdoe_zv^FFgoWMHoL$=uosLFW_E@3L_-ws)o4V9t$tP zGyI3BOkCv=dVMCEgwSIl`^EE%ec`z*9$6}scNf#8i+$20Fe@pU>vpj~f(jm~=<-w; zJEMDH=HOE_dJmxlloO3CG-AcRL@34osP!5|rF$cFjq=&=s=6V|l3%Kh`kX8Ip2_7u zoq!hwgPn?aqU^^rCoMCecd+$PL)27S*Z7vJZ+9ak1) zz9Wa!Q}JY9zdI{A_I9UwMURaNq1L~$pFq;r@bPx!uR=@oy|aikCzjb90``1`W^J3jJTA!f5lLDL!V>cz-nz~ z`Hxu-kRJ|Q8S~#)Bdl3&8Cp1^<=BI|ofscu$|6~M$Q>MGjxVorvWj|mw@RDNO!sBD zp+DyPdX_n&Q5GetRHEM!Kc&}FR8GKNz1njH8>eYi2xg`BM)r4?|ALqnWlF1Ga5VU~ zGE)|hl&xA-Nb24_CC_mwN8(&cI#7%8*vBh>kW4mnlr~B=PSDS^zDzYdMt5*t8GV%H z8@dP%iWxO4q;?>B88f9q=}wI*?=biKrT_NlR0^bGkUdZ$S3YvTGEe~39fS|YY*@qt^WU0=8B2_n()@6*Oc z9H=XKJLF7^3NIhE{n!U3l`p7h37pTFvyzWWjDs>A7w7^nV4y$^+87;awnXZgd)Tvy z^-lD+bi%t1M8@566JsmRn2mFaE1`1os?;C6MS8NjfF==e@25GOpksAAih`QEB%wh zD``rHDkRhU_8s~Y9R#S_%$;LZoz3w3B55-Vq{@&sMz=U?(TRn08|M$k-Abmyf8Z$t z+SD#yBsdb!hx76Dui&|HuR1AZi*{4`o!FzFla7&2YLGQ;Ztvj4=uwivGP-bY^gWD& z@7!g3Cr0)mR%j*7p2%yoIOm-*jYJ1ouaNy&3}9!|#{L$5a zgR6XzocB67IN`J|^3)R^V(A*VY*ybkN7+k#<%MjJl@9RHyA>Rg)!~pcJN-Y2`dGj0 zabLyV!uhf?XOH_jQ0lr8dU$Tw^#uLji1yQbA&by7w>+|Qd&RX#4$M!#=1(8T!s~qX z>33tGGn6SX*0p|qWhq3h**=EW)i~dq8(z-#o%6@uiR>xm@+dh*cBsa7miLTh-*RGJ z*@FF{+{oo)*)R-dG3=)9L_K{l6IHtTgKsjX%PYe|=&qadjN8PNMMY%*Q&hd>YJQyD zx$k-{dN0+(rUF~-8-5iY<-0TIYmiL05#7EzFZnqx#$B(4(hxrzPxmaMDgfsL(B`e0 zQ@kza+!7Ruyv8D+Fz0FlcD;3(?>EN~b5iSTUi<^*9bc+>GBe zZ7gWC-S8NO(5h$L)H1$198 zPtByiwMOuJ()TL}Ef|-29?tW?Cn5hQ(tVNFo?uOAM5i+z(^}FZ&pvlf*Uar?ipj2G zNN#@gQ@7D2%oXAg%UQuJjNZ%@8KOm`g^}<1K+$7n4 zOwE`&1iV-+%c;s7<~-nXnbDEMXQmWECKlnuC+7)0=wH2$w!^Ai68))6rKILTk1a-~ zpRB{bI<0p``eV>hDMy1$Y%>Y9)b3JYukPCtIH2a?K1!{|tA8^GN=NtuU%mHYU+T{F zMQUXH?Gt)r^yP*RbZy^2&c?5Kp=x=3;S_i^Y zQ4pUNo)q!9d)5#9rRDeXCWv$WM~3ICoFbWAQ*;^R(EGU3kIltQ&*|6Bmm*%U3@`}q z!P+VoVkCPZ+Dv+}SAAW(I3)ND$NV%)79p~bqeZIzDs3hyu4y{1Q;&&@h4LrF6`Q$V z8ghKj7Xn@;bGnE>T#6x7N(Soj%m6F+G?+BXK;~J_oP3K zN!rey^yf{}b~=aQTir{9<96fb+mq&yloRfeaf@1T?=DYsTaHMajhk;d67UAf9u&AF~|5g?Pb~7Gwv;?WO@usmrMzmrCt{0rP=688Et#g z-&g)@#8~qY8S6%NyC?mBl@BZ0p7ednNO5<@=Qi0Plla#kFMiRVOVB5d8~-DT`0qP@ z{4-As8qGY@s+Z9nhmNq<#*FB;{Iny&G+PdCGIZ+0zASx9+2%x zZ<6GlrQS>#_o+aa%&fWA#MR}G!e129Hizi3gfpDqEQ##glm2V0(p1;KU+=9bDTzE& z5_!?P{(XO2sM)*zgXT7WnRoqATgcy1I;OuQ@~e_a`qhl=-t~X-hC@xIV_u8AR5IqZ z^&c$uw}lJ$Mvj(@c~MP5&Tk^HU4o2#cg4|2U-u0kcCYvw0(RjlU)Q)WNyfZgeQ7g? zE{i!!y3^B%@seI%{R!n?Qk#*&!#I}&1WZZa_R-&BsH42;G#~u5{F5?%YgY%)>hvMy zpDy~=5JL=4UjLTi9FoDzS!)EvHx1wa#&4eS6;@&8R5B+^t%bs)t9*^{T2+gRpW4J)ZLvP#uhrItIxeBeI-UlA)`CJ7LzoJ zB07iK|5CI0zI`$mSL4U{POa|S7n_+i8rsdAsQyk;e?98&1e^QE%51<<#%6v^nbVXx zL;cNEf3wuzvFh)5_1C5TPEmg!Q=z`A{%%%(Z&!aq>hFc>ue?_to2l_dA$n})?=$6Z zrZO*7f9I*c|403;RDbWHI`-|`=FszBMa0Yg6A}&kqvIfiYB>xwaJaIrD~+&O@w&I` zw{UcOy@w6n0NEcpMK*~l>MPTZ94JrS@2l@CKXZRs{jSR+6QO>wOD=|{F3CLXO?8H{ zoR51`y9S2JQ(5Sc$lmJ?J@md_=P4eNF};ai5`bA-%Q3Gk1i+ z9eSi*Wf|<2?abKBAasb$45`0s)L)^e)bYN3iS=)6K)YCf)&B4b$*2;dTG2$KYeM~I zv@n~Yp}TY~;{f;WWPX4fWEfyOd70qD1-#>n+)xXNa(!J#UBT<+)e*MwIXzv&oxLup z7T+ZX6HM^p!#k(TJ-&A37V#eCxxXOx8fBlw66B-I4(wMbd)G&2O9Trwa$ofOLnOba zIG15HA+U_JqbtZ9aN6!vmRYQLuVjg`nQbCKyM8L-?M+T-{t{9{d~T?n42p9*2VVzQap&rG%o_9nVxlDbfsewYyU zT8ani8Fz)?-DBcCpTLv!^RnbO@bzwYgQ%`dMpNnQn#74{M3(c= z8W0msB87VB%!4W;5g`ix|^; zo%f;Rmo6C3Zs%=mj7>sRa4q!`;Cc@nSVe908M=!t4zp)>S9nnM&*)Lg$1qjF7*blK z+4#`dJ*93kC5y+wjOe$LGod4hCl(+v8sUm+603~fU`tGqT#XD}mmF;3GJLQGZgUr= z==~0%Px=d(Wt2!B6|P|_qaDUag)(JNj0*FW-5eEWYvjJ@y`0JV_N1>RN7`!ULI zcluH>LB{+;r<1U9{MVmPM}c|kghMin$_@=%OqhAIwtr~)518<26Mol(cbRaF30IoX zXTpoj_@NCfs1cfC)cqLZ1m|n{bK=vrKsKMjijlChRrg789;D zVS@=PO*q$tV@>#BK*!T(!pBYcZ4+)bp;c~VL_89rf3ibC%YE!<)9%v#mb(Q#ncDwv zCbZm77FI9P;iKa;tX!|*kO})u|3Hp*|Imah+E%x<-_+*zudVk7Le1@MW`e%O8gDrM z%DM)3T}w-QeVs<`I}u(mShvpI=D*3^;%{3OQqzPmS?BT1oKx87ZtKu6Y{FI(vW-5o zr6qjP%!0W9cWZN--EKLn_}2LAL+!zfmtX1*wYR%l+S^v?0(69e&26g^j?Har>ROr` z+@^b=y`xzbVC_s@PD59-an+KVa_=>zRW;=+mU^oe*K~yH+8XMD4K*sbS!j}284kJE ztX}L7Eetm{`hykz*7o4KD*uh)=AgfoC3i>uIqs&q4tHC7$i334kM(|MJ@@%YMG>sI;y;pmzrl*Cg}7YInf>)PDP&3#jIsL9<@x2`?R(N5x54?@u2 z5pD^&o7>#swg!JAI|L2nwW)4Rb9>O;+(<^7JKU|RWNu914)I@yv)|p|U(;MK3C`?r z?{LrTV6CZtS9`0LmMvX;k$dLNR9t(|@4l%n*hY>tDLR^HY;(XaCpE#j$X(V}-yRH7 zjYJ&u)4a+`Kp4rtF<95?C%X;)wGxH;t!b?*CRE7p)s3>9;~M$lUgD>++t(#~#PKWJ z+u4t^;b{l_m-$0x7`nrX4u7zuE>x!?)%ZF+UGKK=rh|zXPe;|KJss_U|DvGZtY~|@ z5oO+~jeGrkpVtqB+W7qx^tni7;y!6e-> z-IWqFmr(=vjrAz9nP1OH#`gJ2?NPT-txV5dr ztoek$)SH{(qFVK}#&+6rQv0EWHr6#u-^-ubaL#P^O-)$b?ZM_%&24opE$iGJ;re>N zzrioTZ0#G}|7(PW_mC56_n^M|CY7lv!ok3}%$I zF<#ZRG~XJru>bkZ$bMTD!xAMAm{-FDenH^^+I&_2sf%afX7Yceau9a3VuW9hF z46mvQ`o&LG=vA$CRsI0ugR+6AqITOi`&C3 z4Q*$K@M~*uuL{-$nwsl7+&6~(VZl;#SmLXyS>+Ga=#dR#BnIudLMQR3HIQk@S3?tt zPj-YKMu!qL4jw~=1l}>j(=gn4xf9JzkCY91^zt{Tq10*yiTW-McZAwo-2qzvi2_&^ zRw5NoO;hj16lrB${c73+L!VjWr#Y|vj2iyWQACj# z6>=Q^e0M%n2~q|NBiVnhJ3rjk+}Iv$9ofIoo!`#%7PNeI{JLCfIvt+fVd<0K2H2>k z&vbjh&fLbCGfkYvAT6|&3C&%{(7wiR&hXNh1()!esZeEQS;WXzLu%>d=mlFQWTzX7X6Ujd?RggP-{u1-hNHfLp z()*+-<9Okc@_B-(BN5)5wr!+hC=R-liXR_llIU*IFHT}BoH;^B!Q%lF=_LlVV`pbO z-HxBH5g9Qvr9?W%53j3Lb46ALPbQpMuB3n`6Hce6xGXEVlcJ*zm(2KC5u8lACZ$aV zQHM-~qk@wW;+!H(id6qGzN!Rt;WoW^VR#S*0S2nBzCIjeR%}@}$Gt*L!SU8(tLUnmS$%hS%%y>2XYyegPAr6z4_!Qj5`3bkR>afAEf zWx@JItg7bJcdWTI%e#h^OWjIYWN8E)@Vqtdi_86M{4JM~TXSo;)eXm|k>WI-s7&TW*0L&bR-xsYkiSmb{Zu@orWY}u0KZfmjZW~D`-YHa%H)YnegnHVZv83 zlEcKQwPT51I;n;t?y@M59kbOo4p5ax8bL>#)l>N8?RAP_RN0eNr;3p?=astMnfYAS z9uCSXQ3}CO=Wh*!)+ss-U$3hDqQhv4^A;Rq8b5?thc#{214#1o+1R>c!!7xgM@}GT5`yVU; zVm(g0!ZBG0DtE*2l4q)65$gowe*?rcJ5HbEd?vP-tmKmYN93?h5Z_3gijFxs-uMh< zB|D6Mw#muFNDe=2rcgPuD6GgjEWVLO^brD?v_7_8bh$Lkk*Ll%iS34Da>rv#&cBXN z@ljR-vBz;T=3(h1$DhQZDhiJ1Z)QMRVzU3Rna8YyW9f3t%n-*s;okRLwopvUC_I%9 zy`^N|BDbL#XQbad8A)HGzrvwp+J=Cc*gZ=fhTDuww&(@Vw8nwBXM*oGT|^S=%deaj z9Wy8U(w_HzwB^X|ZI}M-`|rP4v9j_q!|3D-EHsQhKBERZpZaz3H~v}qdEgo?t5$HW zLrHj51M?NS7PcFXw;g;*e*icfF9r7lS4(d7Ps$spPUPDv-$-~xCy5s&ezzIFJw7+? zVlMy+T>~TFiT}`5@>l;j@;8oRA6^c@cz77zcI<8)Or;}qk?@r^JdwI<8Uat@E59;1 z-D>P3rEA#mgwC1R-9Y(C43e&-<3II3lHa{pSU+&k%&*?m-wVU@^QC* zQg1~zyc5OO*EDi`X}DWIi7(fNXT_&5Y~>YtHmyp;XW>cy9=IVnzWkHKi#CoN-+j1S zKgr+LKT9&JdZYtdQKsl?s-Nj(Oze^Puxm)w)Y8>m0I z__m%TUf?A0iog@eC;1(ZE~>w+bQlk4v=icO2hVd7cxz7*uL!)8p|f$4cztyz7vE;^ zd?!(k%9D&Q7rad;5#NE@lgr;0@M=#YzS@(FFMk9)>Aw$(f-mzY{a_n*w`sT93*v1? z!)FtCF7XG?Wja}S%4+Lx(ysCe=N5c^R(~65+=w!cxD{5SosNXJ^*DIGVdGHs2zXMy zcdu8uR4Iww1>W%XC+U`4myB19-K|0;%3)gLKtdd4o_3oqQeOErzH-ZT0wvz-`N~lh zP=2wF-}2*rg)xzDD}Gyt`N=%;A%TB_-%p15A+n$JyD!Q1&$DE9Iy^eZ$#HH!Q@i(? zFxeNI8OR>i)9Ele7dZ@MM~*?a9K`1VoBvjuS#Tr{>xWo%Ou6w#;NIKT9YFWv$p+8>o(9?(i zTKrIU#g|sCS^AQsQ+(0&#qACL>-Cwh=Fj6x-WoY9ynbbK+x1sjJ3ctI`0G0SHETNN z7My>LF|Bkuxnq7O*2#+-tJdU@Hk z4|}HAi>sFBRTp~&r+=86I37rL<5CX%d$Joho)M=_aw}8u4GnV>$D_mCiY0KKJuhV{ zzRV=IN)?|g*-hc)Cb=zFcy8QOSykDVE4%{S^3R~WL7|DE`t^qn%buqsV`R*05V|Rx zyqj;nxnZS{OP2#R_EpD=N!hn zq@(S5YlS2SS38V1#7)L+Z;xb;IC?IZs^F7-Q6 zIC@}IAfxu-TwRtKJl~eN{1jvJDTXl$Vwa#vRMJVdrx?aCoXKbvoq*Q+aiLR)Gow=C zt0bI?Z{7rBUCIi#60H(S%6@U>?e#uxc?c*;A>%`q|4nApI#d=z=hFec7*1#~_#Bp(6C?U}~y z%QGoY7UdajIIG44CeKJWa#v0qm=MUWbrrxzP?jNh1#4`!pe|i}T9%@!A6?TBzg+e` z$dURix7oOPaI=0gPD*?ASPnqv6wOa>VL+C&#-FP_uLO!+!pQGEpqTy)lyn?~li%q; zNpEsiij=X^%Kzjssm7Rvqbf6OWpYePGbUB(k~-cPYrGK}V?1O;K$-K$=rC^s<#&%)(`^e-!mKXVS0Deogz(sZ4HUk*vsByPSfu@r z0tLTgu?`;r%I{l1$y0XzI3vGdijlW`vf-YWYnyT{4gaTw1{5nFscY9_yv?v{g!C^pUbuN^a^cvnd`BfcnLiF@D*JD7ORF;TfZ1+O3ZvR0+z{~@ z!{d|ujo|L!3jFAM(g&nJ%$S~LOblnKGtSH~&V1Qzoc8o|`r0)14GrUEz5$6#=BtCb%6LU{Oxvo z7)8gGb~=tB?^#BG72s=po@EvtGn74-1!1|kQJ3OxwBbfjN1-x|Z?Q*=DmUCo=0 zGmVVi#Mm-KIy0D`r+(-*TrZzNnP;I%5(9Fd7G}agX8Fn1N2FSs&(6Co_#xpFWHHea0AS zq~2Gt2WiG*x9fP$Ns1>w9?x0Cb2jnh6AvsDg4Y}PwwdwR?YazOjjSMLQ0-sW+W^W= zmIrD7W*EC&hsjd&f(|p9x@Ihe%bOQ6jM*!6x$SmGW}1cm#=w%E{AvqZXJ?0o#4ivi@UvJZX zr+%IW$?szz)n+W|6q{kRb?Wha04QOy6`wZ7$Ovja?E&H!_gXXVg7q5j5Kw-n+^+4D z&l|<~F~c}*p4)IbrWscL0)&4XJc%Pf9JMC??}74rsmlyMj=5^3o*#T>n&L0v1X@4I zdkS9(=NqGevQLl?>?H2tywZ2g>i;rv3Y-y$4*0`+Gq7J!RToFzt4J zN%EI{dWLa2^Ls91gvGD8&3q4-VTOQ15uNV&cdKZPOMsHX6+kJQL&o}fr0qVfzU?Wcr`~MHS4r1gP=>RQVA8eQt{8Q{aWS-D4Q9 z@NKTveamiljFWZ8SYuq3UUxV)W*Qrp3*Qf~3=gmP{#BXAs^MU75gC0+5C-z6iYEAgA_x7rzMovO#Q0>XZeZ!>Al zAZ(J|hS&cs_JqEjz_Z&Y+xm5WDeLj?==|t)Vxqlh{y4)Jz<*DcKMVE~R@~o~IQUKcp01~hf#M!-Z_GcA85zb5$LU7S&?)`dz0S%Rd6`Du!6`;= z=rqICFqOIN3}b+BbGI^|srkcV_K#$jkjQ2^IV4Op3y_2YBMTlT$^TG8yXJ}=rr6v zG>l96GOrhUq>%RbBsX(?ZPpun8hv-@QQFsI+|-sZvL0uBYlO)Aa6jmige(0flwr7j zL_6V&s&&4kIoyr@X~W5uvNeo1PKU?LWn5rxA4(h0^T&Y_bN-;+3H_x`1WJ8bc5&-} zT)WTyvDggbJ|HO@j{pTPn^WJ(j1BfZgl%Tr0W)riv){y@wp~SITnUsG(gIZRsJxCN zuVcw8>(r`5UN@QXb=uV!jT)70p;g4-ERYq#r}Zle(lrjFJKP?B|q8dz?;4t zo{;sVS_82n6S_rBDiy7{3K>6LjTepKB03|-PmI|6{DK$1~UbA2ix`p`q zYxH<#w`WV;)Ay$_UQAv-p1L1rWZa|sO)vS7-%y{9{~LRi)z|@4`9DF}2M9Z0;(hTM z9qzw?D%{y)jk9@e=FEmuDgPw+wH(9wLFSOG52XA7Gu?g@fA_OG{98a3etMQMojjjr z(r(N>qXytc743Y4b?wu5PXp!mrfHu{dv!OBH6}Wy7~_V<42^2Qo@yxiz+IE6ih35;#o#@HVw>a~Ed^6NG6?|)f`{{ir1!e^S{`%V1s{!)kE z36$`F zmzxQ@#l&0wx(?S4{G@PAgxhT5mHbMFs{wveI1k}EO}u$;=x|p6KPg-;;Q}UJ-mi7I z%YZ7}Ok`BqgP$054UHcd+n?2&84zAU`s|QLk6HcZ*!)|d)b^>TYrgSZV67R?AI*4k zf1}5*#lRGB+knE4y5G|2nS6!j&*AH3KV0!>@DLtwI{3CW=rPT1pDg@L@?56Il|J$) zzk;`Q{C@(ruY6l259+)nTiL^dl)-Xqm|%!~E8jjdufy#UM?8$|JrTxO z&P+8d4Rzi@Ma@V{G1BHy_CFw_##hj&X^?COaTrdvj1PhO+nTT4Lz)gHA87i#0~GqC|4F-NPt9b!8OuC6#u#12{AbRimBc5%0u!$bD8DHm z>hKo;B^}K>j?9nc6Gx|scNi$Y^Z%^FmjhM!6Yx)>iDQe2x8Sf2e-%)|#n%B<)_GH= z(TU$SzRp!bQ)md!Yz&^ z#;J_GwD^9tKUAAqkW#77R(c7q_Zv?$D*tX6KjN!w((NY6uJ$Csf%|)cFOFlg+t+j3 zC_}A7w->eFHdpPp73j5GmazrD5Bd7dw36(y)>xipY-3PQOHJU|?V8@|Oi+(K4H-sf zy2JQSzPx6$j*fAo{_HsBnx;R)Vf>!YI!mbM8 z#{Nl`F)o-VZas7&$q$b$M-XE z1*$&TFphrCxp9?RgT`@loDSnszL*)W&dX?H9A|-!oOJeQc5p_r-N=x9QqIvUftl>3 z26f%4ap-p3WPhLjF@gP_33J)UA&;tj1;ELyWkRD_uVup5jxr`QXH1@(Ga%)V-}O1V zOu5Jh4FA#Vi~`)Nhxxg17nvOSNk0&n%hxdR*f=cNAv<24nWluZepWuV3`pw&iDAdm?YMJCF}OgRdpwb_Z^Tn?BTcRU~@YQw_b5;OT_0Bw;2w@ZFBn^Za&;RHn+QRYr<`l&Fx{_w&Awc=C%*F zEw~-;_bnl}H^Yo_V&FTG}(4*y@*Yvw~Ehs-7DwY+x@GXH_ck;HjT><p_)0Wgv!Lr?#;}pxP34LFU^?}4I%9^-# z7w|Wj7p>~rRzcKGorW$+oT4`UE@mWY;3eNW6fN1h3>rm8%TAY$F*(<^{62zz;y&I8 zSu#EghIxZck;gC=;bOZpXKceBze8xO!v?MG*|1+DGu9}bMdzJAr=g{V9sLw=i^HfC zYYS=3C1nk)3({zG!F84Xpt?(o>LO|awLoq4>x_b9{7@r=qV%A#q_k>jX?fvX9djh* zvYKiX;Ax#eo{mt1$8(Lhy82qLn`b!Yh) zj4!3vs4Ujd9~x^KiuKlK(;$iXmNN#<4O+{!eT8vcVu4!(o;jVR< zXA-nK8jV&P8?>3b)}c5;w{{D+DU6C`R~1*DKd(TS{t~0r-`d)~#&29CruwF!VVr5y zpdqiJ23nz{kKSo4G)R*F(gSM9TSvfO-`v<-U(>`z>lR3ATq>bE{2|sHX6U%?!J7K^ zz&h?+9pi2m#Bdlj#cS89a+f9)7G70cwPNY=vWn7pUY$CI=5}Hv&uvI72jm8MV@(G= zyt$(zP*?9a<{K1TDL1K!>yN2vYnLnCfS^AV4z|e^L5dSNa(ms^p zvACDNZB28qy-idXu{UcdU0NoV721w+p-{OwBu&Klw}iRIZ>%uTrA8dIB4t>v^V-;o zP`d2$*VF`Rpm=aqJ^Kj>8~K_2CBSPaUGnJhm^*m8qwUnIO3=^4GLZ!f@HfY?-7` z&cLAkYjIt+jv8%UPNuybi%=sVCS!SOS-e>;E2qwV(rf-Ftt!+~To<6<`it8cz}JS1 zABkHzIt{{sMW%wm(zZ~LQRXM&b5(ON6s~Jo(FTc`B|ev0)}a)htrFhC-yu!>p7a%M zdhlw9k2z$7N-~V9^kx3galHe?W|pQsxCkYc4je4xR9ub9tpRCkGCs>ldWVcN_2Txg-~YcTAbSR~GO*q! zd+rAJi;XX9H`xREj61nJf@^)@2iW_~OTH&I=Q8hAG666`HMV;^Bl-_xZBzmy#iQ^< zx33x~HB;kJ+|bDB#^-knDLL!^vW-LmHnc9`^Hd=F4z%ux*zbbKosdf=a0a}@pqQH z**nHv;3Is!n70An;_Jse1iW(^cXtE_m@{3=i^zSB*Z7762bk{Wte!FotmB)CSzrra z0p|T5%N>hzXK|i`djW6*pX75h@DSg(my+k% zWP1+h2AB(g)8}&5gSi&?BffVr_W}WY- zKfx?;h;JWefkjo&1+!g_U1Z=b8Fzu#E~CvL4=%8QuK=^a)qEbz0>92ziCN$;`C2dw zJYt&da_Qma37K_~U4L)|^a5YtWmnP8F$?tZ^<%cnoZIEmMK0ZW9Wo}!Z3|q^=f*6s zj;{c-U9Q|Nb1t&yb$kur2wcs#7PG)0-zLlgzs>gmW`S?;J%(A}$EMjX8*Y~y7y0p6 zy+Nkf=LeQFP^Xv$-oPiXCE8`gMRxo*jnET!f$#EFW46nGi(L5cS2J$lE-=Qo1+!hY zTV%d>+{nF5+y%bG7sYIsxj?XuhRLOf&{rhVWp@Oi#U%yv0# zk>8%MmgI34IE8N$X1jd0$ZfB@85-g)@N;~RVYbU*i_G@4Tj)Eu3#{jR1GB)FO|!s^ zTeZ8uY`!6I>~h*7zy0!U&>a0I0^jD##XJN&cRk}X<^tdix04^tA_sl$2HKdo12^!= zm>_b>kMOy!h9K`pN!SU zUG#lE!5;uNN3=NrJZPGSfGh8&FOY`-@cJI+ooez7T>S;|hB*Lyn{O)SA>c~nh^xgN z=-jO52p6!IPx2}7#xH4eC$QiiZ5H@#)4UC6$s7y(Gv5QmDKO<;#uLmg;6r>r!MqK4 z$9;?^m^TAAd>Q!?F#}WX*JHH{7<_;^2KP?jQ+!f(V0!6lVEoQrnt;pKG{9nj_;x6!i`5wbO1Z;X3Kg1J#!(CwJOc>ln7WLNcwDlXH8}O1RC@*G_3H>`?A7+v7oc<*7ewMNT?|B-U zV-|VC^Y+jNkQ=ni42mq_rG1P=xC^|B??cSBK#>)cJP3SwFJo*iv;{u>3~^!>nZDG0 zqD>HURoJ7$r&`}XsM$1HMmUqcqIppHBM5A{PI%p&LZ`hLoeS!CCqd6D|VEb?eO zUZQ=jr2POzKCBY6K#}#@j9K6pUd0cy$Zy^M8f~MV_vJk+(nhIhg&GE58JB5%kY*=3G8a7%lec7nUWGx_>3i>y)lLE1TH7qI>Jnm%@U zA(100i4rIBIRdZY^I;abpze1V(=gj*eeCi&c3B>g@tOS|Wd>j1JiZSx3%r~!4LKcw zOZg^Zt^`j1gU*j#c1L7-?&m84N8tDP$}tO^@JGfI%mOD3X?Ky&nfWKi7TiVF=H?G6 zJ7$ra5!shaOFIoxrlc>#|n@EqN4y=fo%rI06^*ZNO}oM-jP{ zLWje+4|joA@@>Uj3;a6YcFbFVmfVTJucyE_{XgxUPe>F|9LGocC)$>wg@VK&$Oy6S zN{2*t(JoRpeubQ@}DdN3r~r76*QpN;HJI2 zf6?#R@H)`<$Om|CA6{Ylz)G))d?}&*JO_>ycY||i@cN3RgCcKA#{tIR z*+<+5X0$&{?jY+HTKog(yyOwG$T{To&*IiVx50Puq?k1K0bT{_!^`kDP^Nqnj@J71 z^uW_V`^>;kgYswa?Sq_${TJbIi2fKIg@=IJAf5|!9)1<*GW;2+PVyC5^@sfZ8{k<` zeU1ADe*{tVC;0qfzGKisumT>k`* z#mci$m2Jl+a%3q{PN<-^mOs>Xvc)@&Bi$sq2nZk^tW)BRm@`;3MzHWnMvAaMY;COE z+&jgg$~5IgBliQ4$DM?WX(4+(nRD!$=`*7mf{HsSGBG6`JF-XV!60p+$er8Mop*bG5u!H44V;?H}ht}+&3$xWX6}| gWmaF&xEx*X Date: Tue, 21 Aug 2018 11:38:32 +0200 Subject: [PATCH 35/51] Added documentation of the whole "Record and replay" feature to Doxygen docs and README.md. --- README.md | 1 + docs/html/general_considerations.html | 2 +- docs/html/index.html | 1 + docs/html/record_and_replay.html | 97 ++++++++++ docs/html/search/all_b.js | 1 + docs/html/search/pages_7.js | 1 + docs/html/vk__mem__alloc_8h_source.html | 236 ++++++++++++------------ src/vk_mem_alloc.h | 49 ++++- 8 files changed, 268 insertions(+), 120 deletions(-) create mode 100644 docs/html/record_and_replay.html diff --git a/README.md b/README.md index 10be08b..6815d7a 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ Additional features: - JSON dump: Obtain a string in JSON format with detailed map of internal state, including list of allocations and gaps between them. - Convert this JSON dump into a picture to visualize your memory. See [tools/VmaDumpVis](tools/VmaDumpVis/README.md). - Debugging incorrect memory usage: Enable initialization of all allocated memory with a bit pattern to detect usage of uninitialized or freed memory. Enable validation of a magic number before and after every allocation to detect out-of-bounds memory corruption. +- Record and replay sequence of calls to library functions to a file to check correctness, measure performance, and gather statistics. # Prequisites diff --git a/docs/html/general_considerations.html b/docs/html/general_considerations.html index 300ab1b..f631ac8 100644 --- a/docs/html/general_considerations.html +++ b/docs/html/general_considerations.html @@ -107,7 +107,7 @@ Allocation algorithm Features not supported

    Features deliberately excluded from the scope of this library:

      -
    • Data transfer - issuing commands that transfer data between buffers or images, any usage of VkCommandList or VkCommandQueue and related synchronization is responsibility of the user.
      • +
    • Data transfer - issuing commands that transfer data between buffers or images, any usage of VkCommandList or VkQueue and related synchronization is responsibility of the user.
    • Allocations for imported/exported external memory. They tend to require explicit memory type index and dedicated allocation anyway, so they don't interact with main features of this library. Such special purpose allocations should be made manually, using vkCreateBuffer() and vkAllocateMemory().
    • Support for any programming languages other than C/C++. Bindings to other languages are welcomed as external projects.
    diff --git a/docs/html/index.html b/docs/html/index.html index e4dc71d..dc62c22 100644 --- a/docs/html/index.html +++ b/docs/html/index.html @@ -112,6 +112,7 @@ Table of contents     +
  • Record and replay
  • Recommended usage patterns
      diff --git a/docs/html/record_and_replay.html b/docs/html/record_and_replay.html new file mode 100644 index 0000000..d4484be --- /dev/null +++ b/docs/html/record_and_replay.html @@ -0,0 +1,97 @@ + + + + + + + +Vulkan Memory Allocator: Record and replay + + + + + + + + + +
      +
      + + + + + + +
      +
      Vulkan Memory Allocator +
      +
      +
      + + + + + + + + +
      +
      + + +
      + +
      + + +
      +
      +
      +
      Record and replay
      +
      +
      +

      +Introduction

      +

      While using the library, sequence of calls to its functions together with their parameters can be recorded to a file and later replayed using standalone player application. It can be useful to:

      +
        +
      • Test correctness - check if same sequence of calls will not cause crash or failures on a target platform.
        • +
      • Gather statistics - see number of allocations, peak memory usage, number of calls etc.
        • +
      • Benchmark performance - see how much time it takes to replay the whole sequence.
        • +
      +

      +Usage

      +

      To record sequence of calls to a file: Fill in VmaAllocatorCreateInfo::pRecordSettings member while creating VmaAllocator object. File is opened and written during whole lifetime of the allocator.

      +

      To replay file: Use VmaReplay - standalone command-line program. Precompiled binary can be found in "bin" directory. Its source can be found in "src/VmaReplay" directory. Its project is generated by Premake. Command line syntax is printed when the program is launched without parameters. Basic usage: 

      VmaReplay.exe MyRecording.csv
+

      Documentation of file format can be found in file: "docs/Recording file format.md". It's a human-readable, text file in CSV format (Comma Separated Values).

      +

      +Additional considerations

      +
        +
      • Replaying file that was recorded on a different GPU (with different parameters like bufferImageGranularity, nonCoherentAtomSize, and especially different set of memory heaps and types) may give different performance and memory usage results, as well as issue some warnings and errors.
        • +
      • Current implementation of recording in VMA, as well as VmaReplay application, is coded and tested only on Windows. Inclusion of recording code is driven by VMA_RECORDING_ENABLED macro. Support for other platforms should be easy to add. Contributions are welcomed.
        • +
      • Currently calls to vmaDefragment() function are not recorded.
        • +
      +
      + +
      +Generated by   +doxygen + 1.8.13 +
      + + diff --git a/docs/html/search/all_b.js b/docs/html/search/all_b.js index f3376c8..f0c993f 100644 --- a/docs/html/search/all_b.js +++ b/docs/html/search/all_b.js @@ -1,5 +1,6 @@ var searchData= [ + ['record_20and_20replay',['Record and replay',['../record_and_replay.html',1,'index']]], ['requiredflags',['requiredFlags',['../struct_vma_allocation_create_info.html#a9166390303ff42d783305bc31c2b6b90',1,'VmaAllocationCreateInfo']]], ['recommended_20usage_20patterns',['Recommended usage patterns',['../usage_patterns.html',1,'index']]] ]; diff --git a/docs/html/search/pages_7.js b/docs/html/search/pages_7.js index 1e4f738..7394f7d 100644 --- a/docs/html/search/pages_7.js +++ b/docs/html/search/pages_7.js @@ -1,4 +1,5 @@ var searchData= [ + ['record_20and_20replay',['Record and replay',['../record_and_replay.html',1,'index']]], ['recommended_20usage_20patterns',['Recommended usage patterns',['../usage_patterns.html',1,'index']]] ]; diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index d5b79bf..8f7e034 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,176 +62,176 @@ $(function() {
      vk_mem_alloc.h
  • Corruption detection
    • -Go to the documentation of this file.
    1 //
    2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
    3 //
    4 // Permission is hereby granted, free of charge, to any person obtaining a copy
    5 // of this software and associated documentation files (the "Software"), to deal
    6 // in the Software without restriction, including without limitation the rights
    7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    8 // copies of the Software, and to permit persons to whom the Software is
    9 // furnished to do so, subject to the following conditions:
    10 //
    11 // The above copyright notice and this permission notice shall be included in
    12 // all copies or substantial portions of the Software.
    13 //
    14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    20 // THE SOFTWARE.
    21 //
    22 
    23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
    24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
    25 
    26 #ifdef __cplusplus
    27 extern "C" {
    28 #endif
    29 
    1211 #include <vulkan/vulkan.h>
    1212 
    1213 #if !defined(VMA_DEDICATED_ALLOCATION)
    1214  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
    1215  #define VMA_DEDICATED_ALLOCATION 1
    1216  #else
    1217  #define VMA_DEDICATED_ALLOCATION 0
    1218  #endif
    1219 #endif
    1220 
    1230 VK_DEFINE_HANDLE(VmaAllocator)
    1231 
    1232 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
    1234  VmaAllocator allocator,
    1235  uint32_t memoryType,
    1236  VkDeviceMemory memory,
    1237  VkDeviceSize size);
    1239 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
    1240  VmaAllocator allocator,
    1241  uint32_t memoryType,
    1242  VkDeviceMemory memory,
    1243  VkDeviceSize size);
    1244 
    1252 typedef struct VmaDeviceMemoryCallbacks {
    1254  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
    1256  PFN_vmaFreeDeviceMemoryFunction pfnFree;
    1257 } VmaDeviceMemoryCallbacks;
    1258 
    1260 typedef enum VmaAllocatorCreateFlagBits {
    1265  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
    1287  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
    1288 
    1289  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1290 } VmaAllocatorCreateFlagBits;
    1291 typedef VkFlags VmaAllocatorCreateFlags;
    1292 
    1297 typedef struct VmaVulkanFunctions {
    1298  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
    1299  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
    1300  PFN_vkAllocateMemory vkAllocateMemory;
    1301  PFN_vkFreeMemory vkFreeMemory;
    1302  PFN_vkMapMemory vkMapMemory;
    1303  PFN_vkUnmapMemory vkUnmapMemory;
    1304  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
    1305  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
    1306  PFN_vkBindBufferMemory vkBindBufferMemory;
    1307  PFN_vkBindImageMemory vkBindImageMemory;
    1308  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
    1309  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
    1310  PFN_vkCreateBuffer vkCreateBuffer;
    1311  PFN_vkDestroyBuffer vkDestroyBuffer;
    1312  PFN_vkCreateImage vkCreateImage;
    1313  PFN_vkDestroyImage vkDestroyImage;
    1314 #if VMA_DEDICATED_ALLOCATION
    1315  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
    1316  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
    1317 #endif
    1318 } VmaVulkanFunctions;
    1319 
    1321 typedef enum VmaRecordFlagBits {
    1327  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
    1328 
    1329  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1330 } VmaRecordFlagBits;
    1331 typedef VkFlags VmaRecordFlags;
    1332 
    1333 /*
    1334 Define this macro to 0/1 to disable/enable support for recording functionality,
    1335 available through VmaAllocatorCreateInfo::pRecordSettings.
    1336 */
    1337 #ifndef VMA_RECORDING_ENABLED
    1338  #ifdef _WIN32
    1339  #define VMA_RECORDING_ENABLED 1
    1340  #else
    1341  #define VMA_RECORDING_ENABLED 0
    1342  #endif
    1343 #endif
    1344 
    1346 typedef struct VmaRecordSettings
    1347 {
    1349  VmaRecordFlags flags;
    1357  const char* pFilePath;
    1358 } VmaRecordSettings;
    1359 
    1361 typedef struct VmaAllocatorCreateInfo
    1362 {
    1364  VmaAllocatorCreateFlags flags;
    1366 
    1367  VkPhysicalDevice physicalDevice;
    1369 
    1370  VkDevice device;
    1372 
    1373  VkDeviceSize preferredLargeHeapBlockSize;
    1375 
    1376  const VkAllocationCallbacks* pAllocationCallbacks;
    1378 
    1379  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
    1393  uint32_t frameInUseCount;
    1417  const VkDeviceSize* pHeapSizeLimit;
    1429  const VmaVulkanFunctions* pVulkanFunctions;
    1436  const VmaRecordSettings* pRecordSettings;
    1437 } VmaAllocatorCreateInfo;
    1438 
    1440 VkResult vmaCreateAllocator(
    1441  const VmaAllocatorCreateInfo* pCreateInfo,
    1442  VmaAllocator* pAllocator);
    1443 
    1445 void vmaDestroyAllocator(
    1446  VmaAllocator allocator);
    1447 
    1452 void vmaGetPhysicalDeviceProperties(
    1453  VmaAllocator allocator,
    1454  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
    1455 
    1460 void vmaGetMemoryProperties(
    1461  VmaAllocator allocator,
    1462  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
    1463 
    1470 void vmaGetMemoryTypeProperties(
    1471  VmaAllocator allocator,
    1472  uint32_t memoryTypeIndex,
    1473  VkMemoryPropertyFlags* pFlags);
    1474 
    1483 void vmaSetCurrentFrameIndex(
    1484  VmaAllocator allocator,
    1485  uint32_t frameIndex);
    1486 
    1489 typedef struct VmaStatInfo
    1490 {
    1492  uint32_t blockCount;
    1494  uint32_t allocationCount;
    1496  uint32_t unusedRangeCount;
    1498  VkDeviceSize usedBytes;
    1500  VkDeviceSize unusedBytes;
    1501  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
    1502  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
    1503 } VmaStatInfo;
    1504 
    1506 typedef struct VmaStats
    1507 {
    1508  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
    1509  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
    1510  VmaStatInfo total;
    1511 } VmaStats;
    1512 
    1514 void vmaCalculateStats(
    1515  VmaAllocator allocator,
    1516  VmaStats* pStats);
    1517 
    1518 #define VMA_STATS_STRING_ENABLED 1
    1519 
    1520 #if VMA_STATS_STRING_ENABLED
    1521 
    1523 
    1525 void vmaBuildStatsString(
    1526  VmaAllocator allocator,
    1527  char** ppStatsString,
    1528  VkBool32 detailedMap);
    1529 
    1530 void vmaFreeStatsString(
    1531  VmaAllocator allocator,
    1532  char* pStatsString);
    1533 
    1534 #endif // #if VMA_STATS_STRING_ENABLED
    1535 
    1544 VK_DEFINE_HANDLE(VmaPool)
    1545 
    1546 typedef enum VmaMemoryUsage
    1547 {
    1551  VMA_MEMORY_USAGE_UNKNOWN = 0,
    1568  VMA_MEMORY_USAGE_GPU_ONLY = 1,
    1578  VMA_MEMORY_USAGE_CPU_ONLY = 2,
    1585  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
    1594  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
    1595  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
    1596 } VmaMemoryUsage;
    1597 
    1599 typedef enum VmaAllocationCreateFlagBits {
    1611  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
    1612 
    1622  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
    1635  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
    1648  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
    1655  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
    1661  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
    1662 
    1663  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1664 } VmaAllocationCreateFlagBits;
    1665 typedef VkFlags VmaAllocationCreateFlags;
    1666 
    1667 typedef struct VmaAllocationCreateInfo
    1668 {
    1670  VmaAllocationCreateFlags flags;
    1676  VmaMemoryUsage usage;
    1681  VkMemoryPropertyFlags requiredFlags;
    1686  VkMemoryPropertyFlags preferredFlags;
    1694  uint32_t memoryTypeBits;
    1700  VmaPool pool;
    1707  void* pUserData;
    1708 } VmaAllocationCreateInfo;
    1709 
    1726 VkResult vmaFindMemoryTypeIndex(
    1727  VmaAllocator allocator,
    1728  uint32_t memoryTypeBits,
    1729  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1730  uint32_t* pMemoryTypeIndex);
    1731 
    1744 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    1745  VmaAllocator allocator,
    1746  const VkBufferCreateInfo* pBufferCreateInfo,
    1747  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1748  uint32_t* pMemoryTypeIndex);
    1749 
    1762 VkResult vmaFindMemoryTypeIndexForImageInfo(
    1763  VmaAllocator allocator,
    1764  const VkImageCreateInfo* pImageCreateInfo,
    1765  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1766  uint32_t* pMemoryTypeIndex);
    1767 
    1769 typedef enum VmaPoolCreateFlagBits {
    1787  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
    1788 
    1789  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1790 } VmaPoolCreateFlagBits;
    1791 typedef VkFlags VmaPoolCreateFlags;
    1792 
    1795 typedef struct VmaPoolCreateInfo {
    1798  uint32_t memoryTypeIndex;
    1801  VmaPoolCreateFlags flags;
    1806  VkDeviceSize blockSize;
    1811  size_t minBlockCount;
    1819  size_t maxBlockCount;
    1833  uint32_t frameInUseCount;
    1834 } VmaPoolCreateInfo;
    1835 
    1838 typedef struct VmaPoolStats {
    1841  VkDeviceSize size;
    1844  VkDeviceSize unusedSize;
    1847  size_t allocationCount;
    1850  size_t unusedRangeCount;
    1857  VkDeviceSize unusedRangeSizeMax;
    1858 } VmaPoolStats;
    1859 
    1866 VkResult vmaCreatePool(
    1867  VmaAllocator allocator,
    1868  const VmaPoolCreateInfo* pCreateInfo,
    1869  VmaPool* pPool);
    1870 
    1873 void vmaDestroyPool(
    1874  VmaAllocator allocator,
    1875  VmaPool pool);
    1876 
    1883 void vmaGetPoolStats(
    1884  VmaAllocator allocator,
    1885  VmaPool pool,
    1886  VmaPoolStats* pPoolStats);
    1887 
    1894 void vmaMakePoolAllocationsLost(
    1895  VmaAllocator allocator,
    1896  VmaPool pool,
    1897  size_t* pLostAllocationCount);
    1898 
    1913 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
    1914 
    1939 VK_DEFINE_HANDLE(VmaAllocation)
    1940 
    1941 
    1943 typedef struct VmaAllocationInfo {
    1948  uint32_t memoryType;
    1957  VkDeviceMemory deviceMemory;
    1962  VkDeviceSize offset;
    1967  VkDeviceSize size;
    1976  void* pMappedData;
    1981  void* pUserData;
    1982 } VmaAllocationInfo;
    1983 
    1994 VkResult vmaAllocateMemory(
    1995  VmaAllocator allocator,
    1996  const VkMemoryRequirements* pVkMemoryRequirements,
    1997  const VmaAllocationCreateInfo* pCreateInfo,
    1998  VmaAllocation* pAllocation,
    1999  VmaAllocationInfo* pAllocationInfo);
    2000 
    2007 VkResult vmaAllocateMemoryForBuffer(
    2008  VmaAllocator allocator,
    2009  VkBuffer buffer,
    2010  const VmaAllocationCreateInfo* pCreateInfo,
    2011  VmaAllocation* pAllocation,
    2012  VmaAllocationInfo* pAllocationInfo);
    2013 
    2015 VkResult vmaAllocateMemoryForImage(
    2016  VmaAllocator allocator,
    2017  VkImage image,
    2018  const VmaAllocationCreateInfo* pCreateInfo,
    2019  VmaAllocation* pAllocation,
    2020  VmaAllocationInfo* pAllocationInfo);
    2021 
    2023 void vmaFreeMemory(
    2024  VmaAllocator allocator,
    2025  VmaAllocation allocation);
    2026 
    2043 void vmaGetAllocationInfo(
    2044  VmaAllocator allocator,
    2045  VmaAllocation allocation,
    2046  VmaAllocationInfo* pAllocationInfo);
    2047 
    2062 VkBool32 vmaTouchAllocation(
    2063  VmaAllocator allocator,
    2064  VmaAllocation allocation);
    2065 
    2079 void vmaSetAllocationUserData(
    2080  VmaAllocator allocator,
    2081  VmaAllocation allocation,
    2082  void* pUserData);
    2083 
    2094 void vmaCreateLostAllocation(
    2095  VmaAllocator allocator,
    2096  VmaAllocation* pAllocation);
    2097 
    2132 VkResult vmaMapMemory(
    2133  VmaAllocator allocator,
    2134  VmaAllocation allocation,
    2135  void** ppData);
    2136 
    2141 void vmaUnmapMemory(
    2142  VmaAllocator allocator,
    2143  VmaAllocation allocation);
    2144 
    2157 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2158 
    2171 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2172 
    2189 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
    2190 
    2192 typedef struct VmaDefragmentationInfo {
    2197  VkDeviceSize maxBytesToMove;
    2202  uint32_t maxAllocationsToMove;
    2203 } VmaDefragmentationInfo;
    2204 
    2206 typedef struct VmaDefragmentationStats {
    2208  VkDeviceSize bytesMoved;
    2210  VkDeviceSize bytesFreed;
    2212  uint32_t allocationsMoved;
    2214  uint32_t deviceMemoryBlocksFreed;
    2215 } VmaDefragmentationStats;
    2216 
    2299 VkResult vmaDefragment(
    2300  VmaAllocator allocator,
    2301  VmaAllocation* pAllocations,
    2302  size_t allocationCount,
    2303  VkBool32* pAllocationsChanged,
    2304  const VmaDefragmentationInfo *pDefragmentationInfo,
    2305  VmaDefragmentationStats* pDefragmentationStats);
    2306 
    2319 VkResult vmaBindBufferMemory(
    2320  VmaAllocator allocator,
    2321  VmaAllocation allocation,
    2322  VkBuffer buffer);
    2323 
    2336 VkResult vmaBindImageMemory(
    2337  VmaAllocator allocator,
    2338  VmaAllocation allocation,
    2339  VkImage image);
    2340 
    2367 VkResult vmaCreateBuffer(
    2368  VmaAllocator allocator,
    2369  const VkBufferCreateInfo* pBufferCreateInfo,
    2370  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2371  VkBuffer* pBuffer,
    2372  VmaAllocation* pAllocation,
    2373  VmaAllocationInfo* pAllocationInfo);
    2374 
    2386 void vmaDestroyBuffer(
    2387  VmaAllocator allocator,
    2388  VkBuffer buffer,
    2389  VmaAllocation allocation);
    2390 
    2392 VkResult vmaCreateImage(
    2393  VmaAllocator allocator,
    2394  const VkImageCreateInfo* pImageCreateInfo,
    2395  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2396  VkImage* pImage,
    2397  VmaAllocation* pAllocation,
    2398  VmaAllocationInfo* pAllocationInfo);
    2399 
    2411 void vmaDestroyImage(
    2412  VmaAllocator allocator,
    2413  VkImage image,
    2414  VmaAllocation allocation);
    2415 
    2416 #ifdef __cplusplus
    2417 }
    2418 #endif
    2419 
    2420 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
    2421 
    2422 // For Visual Studio IntelliSense.
    2423 #if defined(__cplusplus) && defined(__INTELLISENSE__)
    2424 #define VMA_IMPLEMENTATION
    2425 #endif
    2426 
    2427 #ifdef VMA_IMPLEMENTATION
    2428 #undef VMA_IMPLEMENTATION
    2429 
    2430 #include <cstdint>
    2431 #include <cstdlib>
    2432 #include <cstring>
    2433 
    2434 /*******************************************************************************
    2435 CONFIGURATION SECTION
    2436 
    2437 Define some of these macros before each #include of this header or change them
    2438 here if you need other then default behavior depending on your environment.
    2439 */
    2440 
    2441 /*
    2442 Define this macro to 1 to make the library fetch pointers to Vulkan functions
    2443 internally, like:
    2444 
    2445  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    2446 
    2447 Define to 0 if you are going to provide you own pointers to Vulkan functions via
    2448 VmaAllocatorCreateInfo::pVulkanFunctions.
    2449 */
    2450 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
    2451 #define VMA_STATIC_VULKAN_FUNCTIONS 1
    2452 #endif
    2453 
    2454 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
    2455 //#define VMA_USE_STL_CONTAINERS 1
    2456 
    2457 /* Set this macro to 1 to make the library including and using STL containers:
    2458 std::pair, std::vector, std::list, std::unordered_map.
    2459 
    2460 Set it to 0 or undefined to make the library using its own implementation of
    2461 the containers.
    2462 */
    2463 #if VMA_USE_STL_CONTAINERS
    2464  #define VMA_USE_STL_VECTOR 1
    2465  #define VMA_USE_STL_UNORDERED_MAP 1
    2466  #define VMA_USE_STL_LIST 1
    2467 #endif
    2468 
    2469 #if VMA_USE_STL_VECTOR
    2470  #include <vector>
    2471 #endif
    2472 
    2473 #if VMA_USE_STL_UNORDERED_MAP
    2474  #include <unordered_map>
    2475 #endif
    2476 
    2477 #if VMA_USE_STL_LIST
    2478  #include <list>
    2479 #endif
    2480 
    2481 /*
    2482 Following headers are used in this CONFIGURATION section only, so feel free to
    2483 remove them if not needed.
    2484 */
    2485 #include <cassert> // for assert
    2486 #include <algorithm> // for min, max
    2487 #include <mutex> // for std::mutex
    2488 #include <atomic> // for std::atomic
    2489 
    2490 #ifndef VMA_NULL
    2491  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
    2492  #define VMA_NULL nullptr
    2493 #endif
    2494 
    2495 #if defined(__APPLE__) || defined(__ANDROID__)
    2496 #include <cstdlib>
    2497 void *aligned_alloc(size_t alignment, size_t size)
    2498 {
    2499  // alignment must be >= sizeof(void*)
    2500  if(alignment < sizeof(void*))
    2501  {
    2502  alignment = sizeof(void*);
    2503  }
    2504 
    2505  void *pointer;
    2506  if(posix_memalign(&pointer, alignment, size) == 0)
    2507  return pointer;
    2508  return VMA_NULL;
    2509 }
    2510 #endif
    2511 
    2512 // If your compiler is not compatible with C++11 and definition of
    2513 // aligned_alloc() function is missing, uncommeting following line may help:
    2514 
    2515 //#include <malloc.h>
    2516 
    2517 // Normal assert to check for programmer's errors, especially in Debug configuration.
    2518 #ifndef VMA_ASSERT
    2519  #ifdef _DEBUG
    2520  #define VMA_ASSERT(expr) assert(expr)
    2521  #else
    2522  #define VMA_ASSERT(expr)
    2523  #endif
    2524 #endif
    2525 
    2526 // Assert that will be called very often, like inside data structures e.g. operator[].
    2527 // Making it non-empty can make program slow.
    2528 #ifndef VMA_HEAVY_ASSERT
    2529  #ifdef _DEBUG
    2530  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
    2531  #else
    2532  #define VMA_HEAVY_ASSERT(expr)
    2533  #endif
    2534 #endif
    2535 
    2536 #ifndef VMA_ALIGN_OF
    2537  #define VMA_ALIGN_OF(type) (__alignof(type))
    2538 #endif
    2539 
    2540 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
    2541  #if defined(_WIN32)
    2542  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
    2543  #else
    2544  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
    2545  #endif
    2546 #endif
    2547 
    2548 #ifndef VMA_SYSTEM_FREE
    2549  #if defined(_WIN32)
    2550  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
    2551  #else
    2552  #define VMA_SYSTEM_FREE(ptr) free(ptr)
    2553  #endif
    2554 #endif
    2555 
    2556 #ifndef VMA_MIN
    2557  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
    2558 #endif
    2559 
    2560 #ifndef VMA_MAX
    2561  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
    2562 #endif
    2563 
    2564 #ifndef VMA_SWAP
    2565  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
    2566 #endif
    2567 
    2568 #ifndef VMA_SORT
    2569  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
    2570 #endif
    2571 
    2572 #ifndef VMA_DEBUG_LOG
    2573  #define VMA_DEBUG_LOG(format, ...)
    2574  /*
    2575  #define VMA_DEBUG_LOG(format, ...) do { \
    2576  printf(format, __VA_ARGS__); \
    2577  printf("\n"); \
    2578  } while(false)
    2579  */
    2580 #endif
    2581 
    2582 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
    2583 #if VMA_STATS_STRING_ENABLED
    2584  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
    2585  {
    2586  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
    2587  }
    2588  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
    2589  {
    2590  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
    2591  }
    2592  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
    2593  {
    2594  snprintf(outStr, strLen, "%p", ptr);
    2595  }
    2596 #endif
    2597 
    2598 #ifndef VMA_MUTEX
    2599  class VmaMutex
    2600  {
    2601  public:
    2602  VmaMutex() { }
    2603  ~VmaMutex() { }
    2604  void Lock() { m_Mutex.lock(); }
    2605  void Unlock() { m_Mutex.unlock(); }
    2606  private:
    2607  std::mutex m_Mutex;
    2608  };
    2609  #define VMA_MUTEX VmaMutex
    2610 #endif
    2611 
    2612 /*
    2613 If providing your own implementation, you need to implement a subset of std::atomic:
    2614 
    2615 - Constructor(uint32_t desired)
    2616 - uint32_t load() const
    2617 - void store(uint32_t desired)
    2618 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
    2619 */
    2620 #ifndef VMA_ATOMIC_UINT32
    2621  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
    2622 #endif
    2623 
    2624 #ifndef VMA_BEST_FIT
    2625 
    2637  #define VMA_BEST_FIT (1)
    2638 #endif
    2639 
    2640 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
    2641 
    2645  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
    2646 #endif
    2647 
    2648 #ifndef VMA_DEBUG_ALIGNMENT
    2649 
    2653  #define VMA_DEBUG_ALIGNMENT (1)
    2654 #endif
    2655 
    2656 #ifndef VMA_DEBUG_MARGIN
    2657 
    2661  #define VMA_DEBUG_MARGIN (0)
    2662 #endif
    2663 
    2664 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
    2665 
    2669  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
    2670 #endif
    2671 
    2672 #ifndef VMA_DEBUG_DETECT_CORRUPTION
    2673 
    2678  #define VMA_DEBUG_DETECT_CORRUPTION (0)
    2679 #endif
    2680 
    2681 #ifndef VMA_DEBUG_GLOBAL_MUTEX
    2682 
    2686  #define VMA_DEBUG_GLOBAL_MUTEX (0)
    2687 #endif
    2688 
    2689 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
    2690 
    2694  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
    2695 #endif
    2696 
    2697 #ifndef VMA_SMALL_HEAP_MAX_SIZE
    2698  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
    2700 #endif
    2701 
    2702 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
    2703  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
    2705 #endif
    2706 
    2707 #ifndef VMA_CLASS_NO_COPY
    2708  #define VMA_CLASS_NO_COPY(className) \
    2709  private: \
    2710  className(const className&) = delete; \
    2711  className& operator=(const className&) = delete;
    2712 #endif
    2713 
    2714 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
    2715 
    2716 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
    2717 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
    2718 
    2719 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
    2720 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
    2721 
    2722 /*******************************************************************************
    2723 END OF CONFIGURATION
    2724 */
    2725 
    2726 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
    2727  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
    2728 
    2729 // Returns number of bits set to 1 in (v).
    2730 static inline uint32_t VmaCountBitsSet(uint32_t v)
    2731 {
    2732  uint32_t c = v - ((v >> 1) & 0x55555555);
    2733  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
    2734  c = ((c >> 4) + c) & 0x0F0F0F0F;
    2735  c = ((c >> 8) + c) & 0x00FF00FF;
    2736  c = ((c >> 16) + c) & 0x0000FFFF;
    2737  return c;
    2738 }
    2739 
    2740 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
    2741 // Use types like uint32_t, uint64_t as T.
    2742 template <typename T>
    2743 static inline T VmaAlignUp(T val, T align)
    2744 {
    2745  return (val + align - 1) / align * align;
    2746 }
    2747 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
    2748 // Use types like uint32_t, uint64_t as T.
    2749 template <typename T>
    2750 static inline T VmaAlignDown(T val, T align)
    2751 {
    2752  return val / align * align;
    2753 }
    2754 
    2755 // Division with mathematical rounding to nearest number.
    2756 template <typename T>
    2757 inline T VmaRoundDiv(T x, T y)
    2758 {
    2759  return (x + (y / (T)2)) / y;
    2760 }
    2761 
    2762 static inline bool VmaStrIsEmpty(const char* pStr)
    2763 {
    2764  return pStr == VMA_NULL || *pStr == '\0';
    2765 }
    2766 
    2767 #ifndef VMA_SORT
    2768 
    2769 template<typename Iterator, typename Compare>
    2770 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
    2771 {
    2772  Iterator centerValue = end; --centerValue;
    2773  Iterator insertIndex = beg;
    2774  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
    2775  {
    2776  if(cmp(*memTypeIndex, *centerValue))
    2777  {
    2778  if(insertIndex != memTypeIndex)
    2779  {
    2780  VMA_SWAP(*memTypeIndex, *insertIndex);
    2781  }
    2782  ++insertIndex;
    2783  }
    2784  }
    2785  if(insertIndex != centerValue)
    2786  {
    2787  VMA_SWAP(*insertIndex, *centerValue);
    2788  }
    2789  return insertIndex;
    2790 }
    2791 
    2792 template<typename Iterator, typename Compare>
    2793 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
    2794 {
    2795  if(beg < end)
    2796  {
    2797  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
    2798  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
    2799  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
    2800  }
    2801 }
    2802 
    2803 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
    2804 
    2805 #endif // #ifndef VMA_SORT
    2806 
    2807 /*
    2808 Returns true if two memory blocks occupy overlapping pages.
    2809 ResourceA must be in less memory offset than ResourceB.
    2810 
    2811 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
    2812 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
    2813 */
    2814 static inline bool VmaBlocksOnSamePage(
    2815  VkDeviceSize resourceAOffset,
    2816  VkDeviceSize resourceASize,
    2817  VkDeviceSize resourceBOffset,
    2818  VkDeviceSize pageSize)
    2819 {
    2820  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
    2821  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
    2822  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
    2823  VkDeviceSize resourceBStart = resourceBOffset;
    2824  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
    2825  return resourceAEndPage == resourceBStartPage;
    2826 }
    2827 
    2828 enum VmaSuballocationType
    2829 {
    2830  VMA_SUBALLOCATION_TYPE_FREE = 0,
    2831  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
    2832  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
    2833  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
    2834  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
    2835  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
    2836  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
    2837 };
    2838 
    2839 /*
    2840 Returns true if given suballocation types could conflict and must respect
    2841 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
    2842 or linear image and another one is optimal image. If type is unknown, behave
    2843 conservatively.
    2844 */
    2845 static inline bool VmaIsBufferImageGranularityConflict(
    2846  VmaSuballocationType suballocType1,
    2847  VmaSuballocationType suballocType2)
    2848 {
    2849  if(suballocType1 > suballocType2)
    2850  {
    2851  VMA_SWAP(suballocType1, suballocType2);
    2852  }
    2853 
    2854  switch(suballocType1)
    2855  {
    2856  case VMA_SUBALLOCATION_TYPE_FREE:
    2857  return false;
    2858  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
    2859  return true;
    2860  case VMA_SUBALLOCATION_TYPE_BUFFER:
    2861  return
    2862  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2863  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2864  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
    2865  return
    2866  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2867  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
    2868  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2869  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
    2870  return
    2871  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2872  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
    2873  return false;
    2874  default:
    2875  VMA_ASSERT(0);
    2876  return true;
    2877  }
    2878 }
    2879 
    2880 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
    2881 {
    2882  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
    2883  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2884  for(size_t i = 0; i < numberCount; ++i, ++pDst)
    2885  {
    2886  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
    2887  }
    2888 }
    2889 
    2890 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
    2891 {
    2892  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
    2893  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2894  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
    2895  {
    2896  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
    2897  {
    2898  return false;
    2899  }
    2900  }
    2901  return true;
    2902 }
    2903 
    2904 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
    2905 struct VmaMutexLock
    2906 {
    2907  VMA_CLASS_NO_COPY(VmaMutexLock)
    2908 public:
    2909  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
    2910  m_pMutex(useMutex ? &mutex : VMA_NULL)
    2911  {
    2912  if(m_pMutex)
    2913  {
    2914  m_pMutex->Lock();
    2915  }
    2916  }
    2917 
    2918  ~VmaMutexLock()
    2919  {
    2920  if(m_pMutex)
    2921  {
    2922  m_pMutex->Unlock();
    2923  }
    2924  }
    2925 
    2926 private:
    2927  VMA_MUTEX* m_pMutex;
    2928 };
    2929 
    2930 #if VMA_DEBUG_GLOBAL_MUTEX
    2931  static VMA_MUTEX gDebugGlobalMutex;
    2932  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
    2933 #else
    2934  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
    2935 #endif
    2936 
    2937 // Minimum size of a free suballocation to register it in the free suballocation collection.
    2938 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
    2939 
    2940 /*
    2941 Performs binary search and returns iterator to first element that is greater or
    2942 equal to (key), according to comparison (cmp).
    2943 
    2944 Cmp should return true if first argument is less than second argument.
    2945 
    2946 Returned value is the found element, if present in the collection or place where
    2947 new element with value (key) should be inserted.
    2948 */
    2949 template <typename IterT, typename KeyT, typename CmpT>
    2950 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
    2951 {
    2952  size_t down = 0, up = (end - beg);
    2953  while(down < up)
    2954  {
    2955  const size_t mid = (down + up) / 2;
    2956  if(cmp(*(beg+mid), key))
    2957  {
    2958  down = mid + 1;
    2959  }
    2960  else
    2961  {
    2962  up = mid;
    2963  }
    2964  }
    2965  return beg + down;
    2966 }
    2967 
    2969 // Memory allocation
    2970 
    2971 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
    2972 {
    2973  if((pAllocationCallbacks != VMA_NULL) &&
    2974  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
    2975  {
    2976  return (*pAllocationCallbacks->pfnAllocation)(
    2977  pAllocationCallbacks->pUserData,
    2978  size,
    2979  alignment,
    2980  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    2981  }
    2982  else
    2983  {
    2984  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
    2985  }
    2986 }
    2987 
    2988 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
    2989 {
    2990  if((pAllocationCallbacks != VMA_NULL) &&
    2991  (pAllocationCallbacks->pfnFree != VMA_NULL))
    2992  {
    2993  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
    2994  }
    2995  else
    2996  {
    2997  VMA_SYSTEM_FREE(ptr);
    2998  }
    2999 }
    3000 
    3001 template<typename T>
    3002 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
    3003 {
    3004  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
    3005 }
    3006 
    3007 template<typename T>
    3008 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
    3009 {
    3010  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
    3011 }
    3012 
    3013 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
    3014 
    3015 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
    3016 
    3017 template<typename T>
    3018 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
    3019 {
    3020  ptr->~T();
    3021  VmaFree(pAllocationCallbacks, ptr);
    3022 }
    3023 
    3024 template<typename T>
    3025 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
    3026 {
    3027  if(ptr != VMA_NULL)
    3028  {
    3029  for(size_t i = count; i--; )
    3030  {
    3031  ptr[i].~T();
    3032  }
    3033  VmaFree(pAllocationCallbacks, ptr);
    3034  }
    3035 }
    3036 
    3037 // STL-compatible allocator.
    3038 template<typename T>
    3039 class VmaStlAllocator
    3040 {
    3041 public:
    3042  const VkAllocationCallbacks* const m_pCallbacks;
    3043  typedef T value_type;
    3044 
    3045  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
    3046  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
    3047 
    3048  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
    3049  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
    3050 
    3051  template<typename U>
    3052  bool operator==(const VmaStlAllocator<U>& rhs) const
    3053  {
    3054  return m_pCallbacks == rhs.m_pCallbacks;
    3055  }
    3056  template<typename U>
    3057  bool operator!=(const VmaStlAllocator<U>& rhs) const
    3058  {
    3059  return m_pCallbacks != rhs.m_pCallbacks;
    3060  }
    3061 
    3062  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
    3063 };
    3064 
    3065 #if VMA_USE_STL_VECTOR
    3066 
    3067 #define VmaVector std::vector
    3068 
    3069 template<typename T, typename allocatorT>
    3070 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
    3071 {
    3072  vec.insert(vec.begin() + index, item);
    3073 }
    3074 
    3075 template<typename T, typename allocatorT>
    3076 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
    3077 {
    3078  vec.erase(vec.begin() + index);
    3079 }
    3080 
    3081 #else // #if VMA_USE_STL_VECTOR
    3082 
    3083 /* Class with interface compatible with subset of std::vector.
    3084 T must be POD because constructors and destructors are not called and memcpy is
    3085 used for these objects. */
    3086 template<typename T, typename AllocatorT>
    3087 class VmaVector
    3088 {
    3089 public:
    3090  typedef T value_type;
    3091 
    3092  VmaVector(const AllocatorT& allocator) :
    3093  m_Allocator(allocator),
    3094  m_pArray(VMA_NULL),
    3095  m_Count(0),
    3096  m_Capacity(0)
    3097  {
    3098  }
    3099 
    3100  VmaVector(size_t count, const AllocatorT& allocator) :
    3101  m_Allocator(allocator),
    3102  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
    3103  m_Count(count),
    3104  m_Capacity(count)
    3105  {
    3106  }
    3107 
    3108  VmaVector(const VmaVector<T, AllocatorT>& src) :
    3109  m_Allocator(src.m_Allocator),
    3110  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
    3111  m_Count(src.m_Count),
    3112  m_Capacity(src.m_Count)
    3113  {
    3114  if(m_Count != 0)
    3115  {
    3116  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
    3117  }
    3118  }
    3119 
    3120  ~VmaVector()
    3121  {
    3122  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3123  }
    3124 
    3125  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
    3126  {
    3127  if(&rhs != this)
    3128  {
    3129  resize(rhs.m_Count);
    3130  if(m_Count != 0)
    3131  {
    3132  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
    3133  }
    3134  }
    3135  return *this;
    3136  }
    3137 
    3138  bool empty() const { return m_Count == 0; }
    3139  size_t size() const { return m_Count; }
    3140  T* data() { return m_pArray; }
    3141  const T* data() const { return m_pArray; }
    3142 
    3143  T& operator[](size_t index)
    3144  {
    3145  VMA_HEAVY_ASSERT(index < m_Count);
    3146  return m_pArray[index];
    3147  }
    3148  const T& operator[](size_t index) const
    3149  {
    3150  VMA_HEAVY_ASSERT(index < m_Count);
    3151  return m_pArray[index];
    3152  }
    3153 
    3154  T& front()
    3155  {
    3156  VMA_HEAVY_ASSERT(m_Count > 0);
    3157  return m_pArray[0];
    3158  }
    3159  const T& front() const
    3160  {
    3161  VMA_HEAVY_ASSERT(m_Count > 0);
    3162  return m_pArray[0];
    3163  }
    3164  T& back()
    3165  {
    3166  VMA_HEAVY_ASSERT(m_Count > 0);
    3167  return m_pArray[m_Count - 1];
    3168  }
    3169  const T& back() const
    3170  {
    3171  VMA_HEAVY_ASSERT(m_Count > 0);
    3172  return m_pArray[m_Count - 1];
    3173  }
    3174 
    3175  void reserve(size_t newCapacity, bool freeMemory = false)
    3176  {
    3177  newCapacity = VMA_MAX(newCapacity, m_Count);
    3178 
    3179  if((newCapacity < m_Capacity) && !freeMemory)
    3180  {
    3181  newCapacity = m_Capacity;
    3182  }
    3183 
    3184  if(newCapacity != m_Capacity)
    3185  {
    3186  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
    3187  if(m_Count != 0)
    3188  {
    3189  memcpy(newArray, m_pArray, m_Count * sizeof(T));
    3190  }
    3191  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3192  m_Capacity = newCapacity;
    3193  m_pArray = newArray;
    3194  }
    3195  }
    3196 
    3197  void resize(size_t newCount, bool freeMemory = false)
    3198  {
    3199  size_t newCapacity = m_Capacity;
    3200  if(newCount > m_Capacity)
    3201  {
    3202  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
    3203  }
    3204  else if(freeMemory)
    3205  {
    3206  newCapacity = newCount;
    3207  }
    3208 
    3209  if(newCapacity != m_Capacity)
    3210  {
    3211  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
    3212  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
    3213  if(elementsToCopy != 0)
    3214  {
    3215  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
    3216  }
    3217  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3218  m_Capacity = newCapacity;
    3219  m_pArray = newArray;
    3220  }
    3221 
    3222  m_Count = newCount;
    3223  }
    3224 
    3225  void clear(bool freeMemory = false)
    3226  {
    3227  resize(0, freeMemory);
    3228  }
    3229 
    3230  void insert(size_t index, const T& src)
    3231  {
    3232  VMA_HEAVY_ASSERT(index <= m_Count);
    3233  const size_t oldCount = size();
    3234  resize(oldCount + 1);
    3235  if(index < oldCount)
    3236  {
    3237  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
    3238  }
    3239  m_pArray[index] = src;
    3240  }
    3241 
    3242  void remove(size_t index)
    3243  {
    3244  VMA_HEAVY_ASSERT(index < m_Count);
    3245  const size_t oldCount = size();
    3246  if(index < oldCount - 1)
    3247  {
    3248  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
    3249  }
    3250  resize(oldCount - 1);
    3251  }
    3252 
    3253  void push_back(const T& src)
    3254  {
    3255  const size_t newIndex = size();
    3256  resize(newIndex + 1);
    3257  m_pArray[newIndex] = src;
    3258  }
    3259 
    3260  void pop_back()
    3261  {
    3262  VMA_HEAVY_ASSERT(m_Count > 0);
    3263  resize(size() - 1);
    3264  }
    3265 
    3266  void push_front(const T& src)
    3267  {
    3268  insert(0, src);
    3269  }
    3270 
    3271  void pop_front()
    3272  {
    3273  VMA_HEAVY_ASSERT(m_Count > 0);
    3274  remove(0);
    3275  }
    3276 
    3277  typedef T* iterator;
    3278 
    3279  iterator begin() { return m_pArray; }
    3280  iterator end() { return m_pArray + m_Count; }
    3281 
    3282 private:
    3283  AllocatorT m_Allocator;
    3284  T* m_pArray;
    3285  size_t m_Count;
    3286  size_t m_Capacity;
    3287 };
    3288 
    3289 template<typename T, typename allocatorT>
    3290 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
    3291 {
    3292  vec.insert(index, item);
    3293 }
    3294 
    3295 template<typename T, typename allocatorT>
    3296 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
    3297 {
    3298  vec.remove(index);
    3299 }
    3300 
    3301 #endif // #if VMA_USE_STL_VECTOR
    3302 
    3303 template<typename CmpLess, typename VectorT>
    3304 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
    3305 {
    3306  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    3307  vector.data(),
    3308  vector.data() + vector.size(),
    3309  value,
    3310  CmpLess()) - vector.data();
    3311  VmaVectorInsert(vector, indexToInsert, value);
    3312  return indexToInsert;
    3313 }
    3314 
    3315 template<typename CmpLess, typename VectorT>
    3316 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
    3317 {
    3318  CmpLess comparator;
    3319  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3320  vector.begin(),
    3321  vector.end(),
    3322  value,
    3323  comparator);
    3324  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
    3325  {
    3326  size_t indexToRemove = it - vector.begin();
    3327  VmaVectorRemove(vector, indexToRemove);
    3328  return true;
    3329  }
    3330  return false;
    3331 }
    3332 
    3333 template<typename CmpLess, typename VectorT>
    3334 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
    3335 {
    3336  CmpLess comparator;
    3337  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3338  vector.data(),
    3339  vector.data() + vector.size(),
    3340  value,
    3341  comparator);
    3342  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
    3343  {
    3344  return it - vector.begin();
    3345  }
    3346  else
    3347  {
    3348  return vector.size();
    3349  }
    3350 }
    3351 
    3353 // class VmaPoolAllocator
    3354 
    3355 /*
    3356 Allocator for objects of type T using a list of arrays (pools) to speed up
    3357 allocation. Number of elements that can be allocated is not bounded because
    3358 allocator can create multiple blocks.
    3359 */
    3360 template<typename T>
    3361 class VmaPoolAllocator
    3362 {
    3363  VMA_CLASS_NO_COPY(VmaPoolAllocator)
    3364 public:
    3365  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
    3366  ~VmaPoolAllocator();
    3367  void Clear();
    3368  T* Alloc();
    3369  void Free(T* ptr);
    3370 
    3371 private:
    3372  union Item
    3373  {
    3374  uint32_t NextFreeIndex;
    3375  T Value;
    3376  };
    3377 
    3378  struct ItemBlock
    3379  {
    3380  Item* pItems;
    3381  uint32_t FirstFreeIndex;
    3382  };
    3383 
    3384  const VkAllocationCallbacks* m_pAllocationCallbacks;
    3385  size_t m_ItemsPerBlock;
    3386  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
    3387 
    3388  ItemBlock& CreateNewBlock();
    3389 };
    3390 
    3391 template<typename T>
    3392 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
    3393  m_pAllocationCallbacks(pAllocationCallbacks),
    3394  m_ItemsPerBlock(itemsPerBlock),
    3395  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
    3396 {
    3397  VMA_ASSERT(itemsPerBlock > 0);
    3398 }
    3399 
    3400 template<typename T>
    3401 VmaPoolAllocator<T>::~VmaPoolAllocator()
    3402 {
    3403  Clear();
    3404 }
    3405 
    3406 template<typename T>
    3407 void VmaPoolAllocator<T>::Clear()
    3408 {
    3409  for(size_t i = m_ItemBlocks.size(); i--; )
    3410  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
    3411  m_ItemBlocks.clear();
    3412 }
    3413 
    3414 template<typename T>
    3415 T* VmaPoolAllocator<T>::Alloc()
    3416 {
    3417  for(size_t i = m_ItemBlocks.size(); i--; )
    3418  {
    3419  ItemBlock& block = m_ItemBlocks[i];
    3420  // This block has some free items: Use first one.
    3421  if(block.FirstFreeIndex != UINT32_MAX)
    3422  {
    3423  Item* const pItem = &block.pItems[block.FirstFreeIndex];
    3424  block.FirstFreeIndex = pItem->NextFreeIndex;
    3425  return &pItem->Value;
    3426  }
    3427  }
    3428 
    3429  // No block has free item: Create new one and use it.
    3430  ItemBlock& newBlock = CreateNewBlock();
    3431  Item* const pItem = &newBlock.pItems[0];
    3432  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
    3433  return &pItem->Value;
    3434 }
    3435 
    3436 template<typename T>
    3437 void VmaPoolAllocator<T>::Free(T* ptr)
    3438 {
    3439  // Search all memory blocks to find ptr.
    3440  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
    3441  {
    3442  ItemBlock& block = m_ItemBlocks[i];
    3443 
    3444  // Casting to union.
    3445  Item* pItemPtr;
    3446  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
    3447 
    3448  // Check if pItemPtr is in address range of this block.
    3449  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
    3450  {
    3451  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
    3452  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
    3453  block.FirstFreeIndex = index;
    3454  return;
    3455  }
    3456  }
    3457  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
    3458 }
    3459 
    3460 template<typename T>
    3461 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
    3462 {
    3463  ItemBlock newBlock = {
    3464  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
    3465 
    3466  m_ItemBlocks.push_back(newBlock);
    3467 
    3468  // Setup singly-linked list of all free items in this block.
    3469  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
    3470  newBlock.pItems[i].NextFreeIndex = i + 1;
    3471  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
    3472  return m_ItemBlocks.back();
    3473 }
    3474 
    3476 // class VmaRawList, VmaList
    3477 
    3478 #if VMA_USE_STL_LIST
    3479 
    3480 #define VmaList std::list
    3481 
    3482 #else // #if VMA_USE_STL_LIST
    3483 
    3484 template<typename T>
    3485 struct VmaListItem
    3486 {
    3487  VmaListItem* pPrev;
    3488  VmaListItem* pNext;
    3489  T Value;
    3490 };
    3491 
    3492 // Doubly linked list.
    3493 template<typename T>
    3494 class VmaRawList
    3495 {
    3496  VMA_CLASS_NO_COPY(VmaRawList)
    3497 public:
    3498  typedef VmaListItem<T> ItemType;
    3499 
    3500  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
    3501  ~VmaRawList();
    3502  void Clear();
    3503 
    3504  size_t GetCount() const { return m_Count; }
    3505  bool IsEmpty() const { return m_Count == 0; }
    3506 
    3507  ItemType* Front() { return m_pFront; }
    3508  const ItemType* Front() const { return m_pFront; }
    3509  ItemType* Back() { return m_pBack; }
    3510  const ItemType* Back() const { return m_pBack; }
    3511 
    3512  ItemType* PushBack();
    3513  ItemType* PushFront();
    3514  ItemType* PushBack(const T& value);
    3515  ItemType* PushFront(const T& value);
    3516  void PopBack();
    3517  void PopFront();
    3518 
    3519  // Item can be null - it means PushBack.
    3520  ItemType* InsertBefore(ItemType* pItem);
    3521  // Item can be null - it means PushFront.
    3522  ItemType* InsertAfter(ItemType* pItem);
    3523 
    3524  ItemType* InsertBefore(ItemType* pItem, const T& value);
    3525  ItemType* InsertAfter(ItemType* pItem, const T& value);
    3526 
    3527  void Remove(ItemType* pItem);
    3528 
    3529 private:
    3530  const VkAllocationCallbacks* const m_pAllocationCallbacks;
    3531  VmaPoolAllocator<ItemType> m_ItemAllocator;
    3532  ItemType* m_pFront;
    3533  ItemType* m_pBack;
    3534  size_t m_Count;
    3535 };
    3536 
    3537 template<typename T>
    3538 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
    3539  m_pAllocationCallbacks(pAllocationCallbacks),
    3540  m_ItemAllocator(pAllocationCallbacks, 128),
    3541  m_pFront(VMA_NULL),
    3542  m_pBack(VMA_NULL),
    3543  m_Count(0)
    3544 {
    3545 }
    3546 
    3547 template<typename T>
    3548 VmaRawList<T>::~VmaRawList()
    3549 {
    3550  // Intentionally not calling Clear, because that would be unnecessary
    3551  // computations to return all items to m_ItemAllocator as free.
    3552 }
    3553 
    3554 template<typename T>
    3555 void VmaRawList<T>::Clear()
    3556 {
    3557  if(IsEmpty() == false)
    3558  {
    3559  ItemType* pItem = m_pBack;
    3560  while(pItem != VMA_NULL)
    3561  {
    3562  ItemType* const pPrevItem = pItem->pPrev;
    3563  m_ItemAllocator.Free(pItem);
    3564  pItem = pPrevItem;
    3565  }
    3566  m_pFront = VMA_NULL;
    3567  m_pBack = VMA_NULL;
    3568  m_Count = 0;
    3569  }
    3570 }
    3571 
    3572 template<typename T>
    3573 VmaListItem<T>* VmaRawList<T>::PushBack()
    3574 {
    3575  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3576  pNewItem->pNext = VMA_NULL;
    3577  if(IsEmpty())
    3578  {
    3579  pNewItem->pPrev = VMA_NULL;
    3580  m_pFront = pNewItem;
    3581  m_pBack = pNewItem;
    3582  m_Count = 1;
    3583  }
    3584  else
    3585  {
    3586  pNewItem->pPrev = m_pBack;
    3587  m_pBack->pNext = pNewItem;
    3588  m_pBack = pNewItem;
    3589  ++m_Count;
    3590  }
    3591  return pNewItem;
    3592 }
    3593 
    3594 template<typename T>
    3595 VmaListItem<T>* VmaRawList<T>::PushFront()
    3596 {
    3597  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3598  pNewItem->pPrev = VMA_NULL;
    3599  if(IsEmpty())
    3600  {
    3601  pNewItem->pNext = VMA_NULL;
    3602  m_pFront = pNewItem;
    3603  m_pBack = pNewItem;
    3604  m_Count = 1;
    3605  }
    3606  else
    3607  {
    3608  pNewItem->pNext = m_pFront;
    3609  m_pFront->pPrev = pNewItem;
    3610  m_pFront = pNewItem;
    3611  ++m_Count;
    3612  }
    3613  return pNewItem;
    3614 }
    3615 
    3616 template<typename T>
    3617 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
    3618 {
    3619  ItemType* const pNewItem = PushBack();
    3620  pNewItem->Value = value;
    3621  return pNewItem;
    3622 }
    3623 
    3624 template<typename T>
    3625 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
    3626 {
    3627  ItemType* const pNewItem = PushFront();
    3628  pNewItem->Value = value;
    3629  return pNewItem;
    3630 }
    3631 
    3632 template<typename T>
    3633 void VmaRawList<T>::PopBack()
    3634 {
    3635  VMA_HEAVY_ASSERT(m_Count > 0);
    3636  ItemType* const pBackItem = m_pBack;
    3637  ItemType* const pPrevItem = pBackItem->pPrev;
    3638  if(pPrevItem != VMA_NULL)
    3639  {
    3640  pPrevItem->pNext = VMA_NULL;
    3641  }
    3642  m_pBack = pPrevItem;
    3643  m_ItemAllocator.Free(pBackItem);
    3644  --m_Count;
    3645 }
    3646 
    3647 template<typename T>
    3648 void VmaRawList<T>::PopFront()
    3649 {
    3650  VMA_HEAVY_ASSERT(m_Count > 0);
    3651  ItemType* const pFrontItem = m_pFront;
    3652  ItemType* const pNextItem = pFrontItem->pNext;
    3653  if(pNextItem != VMA_NULL)
    3654  {
    3655  pNextItem->pPrev = VMA_NULL;
    3656  }
    3657  m_pFront = pNextItem;
    3658  m_ItemAllocator.Free(pFrontItem);
    3659  --m_Count;
    3660 }
    3661 
    3662 template<typename T>
    3663 void VmaRawList<T>::Remove(ItemType* pItem)
    3664 {
    3665  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
    3666  VMA_HEAVY_ASSERT(m_Count > 0);
    3667 
    3668  if(pItem->pPrev != VMA_NULL)
    3669  {
    3670  pItem->pPrev->pNext = pItem->pNext;
    3671  }
    3672  else
    3673  {
    3674  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3675  m_pFront = pItem->pNext;
    3676  }
    3677 
    3678  if(pItem->pNext != VMA_NULL)
    3679  {
    3680  pItem->pNext->pPrev = pItem->pPrev;
    3681  }
    3682  else
    3683  {
    3684  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3685  m_pBack = pItem->pPrev;
    3686  }
    3687 
    3688  m_ItemAllocator.Free(pItem);
    3689  --m_Count;
    3690 }
    3691 
    3692 template<typename T>
    3693 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
    3694 {
    3695  if(pItem != VMA_NULL)
    3696  {
    3697  ItemType* const prevItem = pItem->pPrev;
    3698  ItemType* const newItem = m_ItemAllocator.Alloc();
    3699  newItem->pPrev = prevItem;
    3700  newItem->pNext = pItem;
    3701  pItem->pPrev = newItem;
    3702  if(prevItem != VMA_NULL)
    3703  {
    3704  prevItem->pNext = newItem;
    3705  }
    3706  else
    3707  {
    3708  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3709  m_pFront = newItem;
    3710  }
    3711  ++m_Count;
    3712  return newItem;
    3713  }
    3714  else
    3715  return PushBack();
    3716 }
    3717 
    3718 template<typename T>
    3719 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
    3720 {
    3721  if(pItem != VMA_NULL)
    3722  {
    3723  ItemType* const nextItem = pItem->pNext;
    3724  ItemType* const newItem = m_ItemAllocator.Alloc();
    3725  newItem->pNext = nextItem;
    3726  newItem->pPrev = pItem;
    3727  pItem->pNext = newItem;
    3728  if(nextItem != VMA_NULL)
    3729  {
    3730  nextItem->pPrev = newItem;
    3731  }
    3732  else
    3733  {
    3734  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3735  m_pBack = newItem;
    3736  }
    3737  ++m_Count;
    3738  return newItem;
    3739  }
    3740  else
    3741  return PushFront();
    3742 }
    3743 
    3744 template<typename T>
    3745 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
    3746 {
    3747  ItemType* const newItem = InsertBefore(pItem);
    3748  newItem->Value = value;
    3749  return newItem;
    3750 }
    3751 
    3752 template<typename T>
    3753 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
    3754 {
    3755  ItemType* const newItem = InsertAfter(pItem);
    3756  newItem->Value = value;
    3757  return newItem;
    3758 }
    3759 
    3760 template<typename T, typename AllocatorT>
    3761 class VmaList
    3762 {
    3763  VMA_CLASS_NO_COPY(VmaList)
    3764 public:
    3765  class iterator
    3766  {
    3767  public:
    3768  iterator() :
    3769  m_pList(VMA_NULL),
    3770  m_pItem(VMA_NULL)
    3771  {
    3772  }
    3773 
    3774  T& operator*() const
    3775  {
    3776  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3777  return m_pItem->Value;
    3778  }
    3779  T* operator->() const
    3780  {
    3781  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3782  return &m_pItem->Value;
    3783  }
    3784 
    3785  iterator& operator++()
    3786  {
    3787  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3788  m_pItem = m_pItem->pNext;
    3789  return *this;
    3790  }
    3791  iterator& operator--()
    3792  {
    3793  if(m_pItem != VMA_NULL)
    3794  {
    3795  m_pItem = m_pItem->pPrev;
    3796  }
    3797  else
    3798  {
    3799  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3800  m_pItem = m_pList->Back();
    3801  }
    3802  return *this;
    3803  }
    3804 
    3805  iterator operator++(int)
    3806  {
    3807  iterator result = *this;
    3808  ++*this;
    3809  return result;
    3810  }
    3811  iterator operator--(int)
    3812  {
    3813  iterator result = *this;
    3814  --*this;
    3815  return result;
    3816  }
    3817 
    3818  bool operator==(const iterator& rhs) const
    3819  {
    3820  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3821  return m_pItem == rhs.m_pItem;
    3822  }
    3823  bool operator!=(const iterator& rhs) const
    3824  {
    3825  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3826  return m_pItem != rhs.m_pItem;
    3827  }
    3828 
    3829  private:
    3830  VmaRawList<T>* m_pList;
    3831  VmaListItem<T>* m_pItem;
    3832 
    3833  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
    3834  m_pList(pList),
    3835  m_pItem(pItem)
    3836  {
    3837  }
    3838 
    3839  friend class VmaList<T, AllocatorT>;
    3840  };
    3841 
    3842  class const_iterator
    3843  {
    3844  public:
    3845  const_iterator() :
    3846  m_pList(VMA_NULL),
    3847  m_pItem(VMA_NULL)
    3848  {
    3849  }
    3850 
    3851  const_iterator(const iterator& src) :
    3852  m_pList(src.m_pList),
    3853  m_pItem(src.m_pItem)
    3854  {
    3855  }
    3856 
    3857  const T& operator*() const
    3858  {
    3859  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3860  return m_pItem->Value;
    3861  }
    3862  const T* operator->() const
    3863  {
    3864  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3865  return &m_pItem->Value;
    3866  }
    3867 
    3868  const_iterator& operator++()
    3869  {
    3870  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3871  m_pItem = m_pItem->pNext;
    3872  return *this;
    3873  }
    3874  const_iterator& operator--()
    3875  {
    3876  if(m_pItem != VMA_NULL)
    3877  {
    3878  m_pItem = m_pItem->pPrev;
    3879  }
    3880  else
    3881  {
    3882  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3883  m_pItem = m_pList->Back();
    3884  }
    3885  return *this;
    3886  }
    3887 
    3888  const_iterator operator++(int)
    3889  {
    3890  const_iterator result = *this;
    3891  ++*this;
    3892  return result;
    3893  }
    3894  const_iterator operator--(int)
    3895  {
    3896  const_iterator result = *this;
    3897  --*this;
    3898  return result;
    3899  }
    3900 
    3901  bool operator==(const const_iterator& rhs) const
    3902  {
    3903  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3904  return m_pItem == rhs.m_pItem;
    3905  }
    3906  bool operator!=(const const_iterator& rhs) const
    3907  {
    3908  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3909  return m_pItem != rhs.m_pItem;
    3910  }
    3911 
    3912  private:
    3913  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
    3914  m_pList(pList),
    3915  m_pItem(pItem)
    3916  {
    3917  }
    3918 
    3919  const VmaRawList<T>* m_pList;
    3920  const VmaListItem<T>* m_pItem;
    3921 
    3922  friend class VmaList<T, AllocatorT>;
    3923  };
    3924 
    3925  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
    3926 
    3927  bool empty() const { return m_RawList.IsEmpty(); }
    3928  size_t size() const { return m_RawList.GetCount(); }
    3929 
    3930  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
    3931  iterator end() { return iterator(&m_RawList, VMA_NULL); }
    3932 
    3933  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
    3934  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
    3935 
    3936  void clear() { m_RawList.Clear(); }
    3937  void push_back(const T& value) { m_RawList.PushBack(value); }
    3938  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
    3939  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
    3940 
    3941 private:
    3942  VmaRawList<T> m_RawList;
    3943 };
    3944 
    3945 #endif // #if VMA_USE_STL_LIST
    3946 
    3948 // class VmaMap
    3949 
    3950 // Unused in this version.
    3951 #if 0
    3952 
    3953 #if VMA_USE_STL_UNORDERED_MAP
    3954 
    3955 #define VmaPair std::pair
    3956 
    3957 #define VMA_MAP_TYPE(KeyT, ValueT) \
    3958  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
    3959 
    3960 #else // #if VMA_USE_STL_UNORDERED_MAP
    3961 
    3962 template<typename T1, typename T2>
    3963 struct VmaPair
    3964 {
    3965  T1 first;
    3966  T2 second;
    3967 
    3968  VmaPair() : first(), second() { }
    3969  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
    3970 };
    3971 
    3972 /* Class compatible with subset of interface of std::unordered_map.
    3973 KeyT, ValueT must be POD because they will be stored in VmaVector.
    3974 */
    3975 template<typename KeyT, typename ValueT>
    3976 class VmaMap
    3977 {
    3978 public:
    3979  typedef VmaPair<KeyT, ValueT> PairType;
    3980  typedef PairType* iterator;
    3981 
    3982  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
    3983 
    3984  iterator begin() { return m_Vector.begin(); }
    3985  iterator end() { return m_Vector.end(); }
    3986 
    3987  void insert(const PairType& pair);
    3988  iterator find(const KeyT& key);
    3989  void erase(iterator it);
    3990 
    3991 private:
    3992  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
    3993 };
    3994 
    3995 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
    3996 
    3997 template<typename FirstT, typename SecondT>
    3998 struct VmaPairFirstLess
    3999 {
    4000  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
    4001  {
    4002  return lhs.first < rhs.first;
    4003  }
    4004  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
    4005  {
    4006  return lhs.first < rhsFirst;
    4007  }
    4008 };
    4009 
    4010 template<typename KeyT, typename ValueT>
    4011 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
    4012 {
    4013  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    4014  m_Vector.data(),
    4015  m_Vector.data() + m_Vector.size(),
    4016  pair,
    4017  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
    4018  VmaVectorInsert(m_Vector, indexToInsert, pair);
    4019 }
    4020 
    4021 template<typename KeyT, typename ValueT>
    4022 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
    4023 {
    4024  PairType* it = VmaBinaryFindFirstNotLess(
    4025  m_Vector.data(),
    4026  m_Vector.data() + m_Vector.size(),
    4027  key,
    4028  VmaPairFirstLess<KeyT, ValueT>());
    4029  if((it != m_Vector.end()) && (it->first == key))
    4030  {
    4031  return it;
    4032  }
    4033  else
    4034  {
    4035  return m_Vector.end();
    4036  }
    4037 }
    4038 
    4039 template<typename KeyT, typename ValueT>
    4040 void VmaMap<KeyT, ValueT>::erase(iterator it)
    4041 {
    4042  VmaVectorRemove(m_Vector, it - m_Vector.begin());
    4043 }
    4044 
    4045 #endif // #if VMA_USE_STL_UNORDERED_MAP
    4046 
    4047 #endif // #if 0
    4048 
    4050 
    4051 class VmaDeviceMemoryBlock;
    4052 
    4053 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
    4054 
    4055 struct VmaAllocation_T
    4056 {
    4057  VMA_CLASS_NO_COPY(VmaAllocation_T)
    4058 private:
    4059  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
    4060 
    4061  enum FLAGS
    4062  {
    4063  FLAG_USER_DATA_STRING = 0x01,
    4064  };
    4065 
    4066 public:
    4067  enum ALLOCATION_TYPE
    4068  {
    4069  ALLOCATION_TYPE_NONE,
    4070  ALLOCATION_TYPE_BLOCK,
    4071  ALLOCATION_TYPE_DEDICATED,
    4072  };
    4073 
    4074  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
    4075  m_Alignment(1),
    4076  m_Size(0),
    4077  m_pUserData(VMA_NULL),
    4078  m_LastUseFrameIndex(currentFrameIndex),
    4079  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
    4080  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
    4081  m_MapCount(0),
    4082  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
    4083  {
    4084 #if VMA_STATS_STRING_ENABLED
    4085  m_CreationFrameIndex = currentFrameIndex;
    4086  m_BufferImageUsage = 0;
    4087 #endif
    4088  }
    4089 
    4090  ~VmaAllocation_T()
    4091  {
    4092  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
    4093 
    4094  // Check if owned string was freed.
    4095  VMA_ASSERT(m_pUserData == VMA_NULL);
    4096  }
    4097 
    4098  void InitBlockAllocation(
    4099  VmaPool hPool,
    4100  VmaDeviceMemoryBlock* block,
    4101  VkDeviceSize offset,
    4102  VkDeviceSize alignment,
    4103  VkDeviceSize size,
    4104  VmaSuballocationType suballocationType,
    4105  bool mapped,
    4106  bool canBecomeLost)
    4107  {
    4108  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4109  VMA_ASSERT(block != VMA_NULL);
    4110  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4111  m_Alignment = alignment;
    4112  m_Size = size;
    4113  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4114  m_SuballocationType = (uint8_t)suballocationType;
    4115  m_BlockAllocation.m_hPool = hPool;
    4116  m_BlockAllocation.m_Block = block;
    4117  m_BlockAllocation.m_Offset = offset;
    4118  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
    4119  }
    4120 
    4121  void InitLost()
    4122  {
    4123  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4124  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
    4125  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4126  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
    4127  m_BlockAllocation.m_Block = VMA_NULL;
    4128  m_BlockAllocation.m_Offset = 0;
    4129  m_BlockAllocation.m_CanBecomeLost = true;
    4130  }
    4131 
    4132  void ChangeBlockAllocation(
    4133  VmaAllocator hAllocator,
    4134  VmaDeviceMemoryBlock* block,
    4135  VkDeviceSize offset);
    4136 
    4137  // pMappedData not null means allocation is created with MAPPED flag.
    4138  void InitDedicatedAllocation(
    4139  uint32_t memoryTypeIndex,
    4140  VkDeviceMemory hMemory,
    4141  VmaSuballocationType suballocationType,
    4142  void* pMappedData,
    4143  VkDeviceSize size)
    4144  {
    4145  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4146  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
    4147  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
    4148  m_Alignment = 0;
    4149  m_Size = size;
    4150  m_SuballocationType = (uint8_t)suballocationType;
    4151  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4152  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
    4153  m_DedicatedAllocation.m_hMemory = hMemory;
    4154  m_DedicatedAllocation.m_pMappedData = pMappedData;
    4155  }
    4156 
    4157  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
    4158  VkDeviceSize GetAlignment() const { return m_Alignment; }
    4159  VkDeviceSize GetSize() const { return m_Size; }
    4160  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
    4161  void* GetUserData() const { return m_pUserData; }
    4162  void SetUserData(VmaAllocator hAllocator, void* pUserData);
    4163  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
    4164 
    4165  VmaDeviceMemoryBlock* GetBlock() const
    4166  {
    4167  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    4168  return m_BlockAllocation.m_Block;
    4169  }
    4170  VkDeviceSize GetOffset() const;
    4171  VkDeviceMemory GetMemory() const;
    4172  uint32_t GetMemoryTypeIndex() const;
    4173  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
    4174  void* GetMappedData() const;
    4175  bool CanBecomeLost() const;
    4176  VmaPool GetPool() const;
    4177 
    4178  uint32_t GetLastUseFrameIndex() const
    4179  {
    4180  return m_LastUseFrameIndex.load();
    4181  }
    4182  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
    4183  {
    4184  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
    4185  }
    4186  /*
    4187  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
    4188  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
    4189  - Else, returns false.
    4190 
    4191  If hAllocation is already lost, assert - you should not call it then.
    4192  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
    4193  */
    4194  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4195 
    4196  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
    4197  {
    4198  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
    4199  outInfo.blockCount = 1;
    4200  outInfo.allocationCount = 1;
    4201  outInfo.unusedRangeCount = 0;
    4202  outInfo.usedBytes = m_Size;
    4203  outInfo.unusedBytes = 0;
    4204  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
    4205  outInfo.unusedRangeSizeMin = UINT64_MAX;
    4206  outInfo.unusedRangeSizeMax = 0;
    4207  }
    4208 
    4209  void BlockAllocMap();
    4210  void BlockAllocUnmap();
    4211  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
    4212  void DedicatedAllocUnmap(VmaAllocator hAllocator);
    4213 
    4214 #if VMA_STATS_STRING_ENABLED
    4215  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
    4216  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
    4217 
    4218  void InitBufferImageUsage(uint32_t bufferImageUsage)
    4219  {
    4220  VMA_ASSERT(m_BufferImageUsage == 0);
    4221  m_BufferImageUsage = bufferImageUsage;
    4222  }
    4223 
    4224  void PrintParameters(class VmaJsonWriter& json) const;
    4225 #endif
    4226 
    4227 private:
    4228  VkDeviceSize m_Alignment;
    4229  VkDeviceSize m_Size;
    4230  void* m_pUserData;
    4231  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
    4232  uint8_t m_Type; // ALLOCATION_TYPE
    4233  uint8_t m_SuballocationType; // VmaSuballocationType
    4234  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
    4235  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
    4236  uint8_t m_MapCount;
    4237  uint8_t m_Flags; // enum FLAGS
    4238 
    4239  // Allocation out of VmaDeviceMemoryBlock.
    4240  struct BlockAllocation
    4241  {
    4242  VmaPool m_hPool; // Null if belongs to general memory.
    4243  VmaDeviceMemoryBlock* m_Block;
    4244  VkDeviceSize m_Offset;
    4245  bool m_CanBecomeLost;
    4246  };
    4247 
    4248  // Allocation for an object that has its own private VkDeviceMemory.
    4249  struct DedicatedAllocation
    4250  {
    4251  uint32_t m_MemoryTypeIndex;
    4252  VkDeviceMemory m_hMemory;
    4253  void* m_pMappedData; // Not null means memory is mapped.
    4254  };
    4255 
    4256  union
    4257  {
    4258  // Allocation out of VmaDeviceMemoryBlock.
    4259  BlockAllocation m_BlockAllocation;
    4260  // Allocation for an object that has its own private VkDeviceMemory.
    4261  DedicatedAllocation m_DedicatedAllocation;
    4262  };
    4263 
    4264 #if VMA_STATS_STRING_ENABLED
    4265  uint32_t m_CreationFrameIndex;
    4266  uint32_t m_BufferImageUsage; // 0 if unknown.
    4267 #endif
    4268 
    4269  void FreeUserDataString(VmaAllocator hAllocator);
    4270 };
    4271 
    4272 /*
    4273 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
    4274 allocated memory block or free.
    4275 */
    4276 struct VmaSuballocation
    4277 {
    4278  VkDeviceSize offset;
    4279  VkDeviceSize size;
    4280  VmaAllocation hAllocation;
    4281  VmaSuballocationType type;
    4282 };
    4283 
    4284 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
    4285 
    4286 // Cost of one additional allocation lost, as equivalent in bytes.
    4287 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
    4288 
    4289 /*
    4290 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
    4291 
    4292 If canMakeOtherLost was false:
    4293 - item points to a FREE suballocation.
    4294 - itemsToMakeLostCount is 0.
    4295 
    4296 If canMakeOtherLost was true:
    4297 - item points to first of sequence of suballocations, which are either FREE,
    4298  or point to VmaAllocations that can become lost.
    4299 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
    4300  the requested allocation to succeed.
    4301 */
    4302 struct VmaAllocationRequest
    4303 {
    4304  VkDeviceSize offset;
    4305  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
    4306  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
    4307  VmaSuballocationList::iterator item;
    4308  size_t itemsToMakeLostCount;
    4309 
    4310  VkDeviceSize CalcCost() const
    4311  {
    4312  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
    4313  }
    4314 };
    4315 
    4316 /*
    4317 Data structure used for bookkeeping of allocations and unused ranges of memory
    4318 in a single VkDeviceMemory block.
    4319 */
    4320 class VmaBlockMetadata
    4321 {
    4322  VMA_CLASS_NO_COPY(VmaBlockMetadata)
    4323 public:
    4324  VmaBlockMetadata(VmaAllocator hAllocator);
    4325  ~VmaBlockMetadata();
    4326  void Init(VkDeviceSize size);
    4327 
    4328  // Validates all data structures inside this object. If not valid, returns false.
    4329  bool Validate() const;
    4330  VkDeviceSize GetSize() const { return m_Size; }
    4331  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
    4332  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
    4333  VkDeviceSize GetUnusedRangeSizeMax() const;
    4334  // Returns true if this block is empty - contains only single free suballocation.
    4335  bool IsEmpty() const;
    4336 
    4337  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
    4338  void AddPoolStats(VmaPoolStats& inoutStats) const;
    4339 
    4340 #if VMA_STATS_STRING_ENABLED
    4341  void PrintDetailedMap(class VmaJsonWriter& json) const;
    4342 #endif
    4343 
    4344  // Tries to find a place for suballocation with given parameters inside this block.
    4345  // If succeeded, fills pAllocationRequest and returns true.
    4346  // If failed, returns false.
    4347  bool CreateAllocationRequest(
    4348  uint32_t currentFrameIndex,
    4349  uint32_t frameInUseCount,
    4350  VkDeviceSize bufferImageGranularity,
    4351  VkDeviceSize allocSize,
    4352  VkDeviceSize allocAlignment,
    4353  VmaSuballocationType allocType,
    4354  bool canMakeOtherLost,
    4355  VmaAllocationRequest* pAllocationRequest);
    4356 
    4357  bool MakeRequestedAllocationsLost(
    4358  uint32_t currentFrameIndex,
    4359  uint32_t frameInUseCount,
    4360  VmaAllocationRequest* pAllocationRequest);
    4361 
    4362  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4363 
    4364  VkResult CheckCorruption(const void* pBlockData);
    4365 
    4366  // Makes actual allocation based on request. Request must already be checked and valid.
    4367  void Alloc(
    4368  const VmaAllocationRequest& request,
    4369  VmaSuballocationType type,
    4370  VkDeviceSize allocSize,
    4371  VmaAllocation hAllocation);
    4372 
    4373  // Frees suballocation assigned to given memory region.
    4374  void Free(const VmaAllocation allocation);
    4375  void FreeAtOffset(VkDeviceSize offset);
    4376 
    4377 private:
    4378  VkDeviceSize m_Size;
    4379  uint32_t m_FreeCount;
    4380  VkDeviceSize m_SumFreeSize;
    4381  VmaSuballocationList m_Suballocations;
    4382  // Suballocations that are free and have size greater than certain threshold.
    4383  // Sorted by size, ascending.
    4384  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
    4385 
    4386  bool ValidateFreeSuballocationList() const;
    4387 
    4388  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
    4389  // If yes, fills pOffset and returns true. If no, returns false.
    4390  bool CheckAllocation(
    4391  uint32_t currentFrameIndex,
    4392  uint32_t frameInUseCount,
    4393  VkDeviceSize bufferImageGranularity,
    4394  VkDeviceSize allocSize,
    4395  VkDeviceSize allocAlignment,
    4396  VmaSuballocationType allocType,
    4397  VmaSuballocationList::const_iterator suballocItem,
    4398  bool canMakeOtherLost,
    4399  VkDeviceSize* pOffset,
    4400  size_t* itemsToMakeLostCount,
    4401  VkDeviceSize* pSumFreeSize,
    4402  VkDeviceSize* pSumItemSize) const;
    4403  // Given free suballocation, it merges it with following one, which must also be free.
    4404  void MergeFreeWithNext(VmaSuballocationList::iterator item);
    4405  // Releases given suballocation, making it free.
    4406  // Merges it with adjacent free suballocations if applicable.
    4407  // Returns iterator to new free suballocation at this place.
    4408  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
    4409  // Given free suballocation, it inserts it into sorted list of
    4410  // m_FreeSuballocationsBySize if it's suitable.
    4411  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
    4412  // Given free suballocation, it removes it from sorted list of
    4413  // m_FreeSuballocationsBySize if it's suitable.
    4414  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
    4415 };
    4416 
    4417 /*
    4418 Represents a single block of device memory (`VkDeviceMemory`) with all the
    4419 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
    4420 
    4421 Thread-safety: This class must be externally synchronized.
    4422 */
    4423 class VmaDeviceMemoryBlock
    4424 {
    4425  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
    4426 public:
    4427  VmaBlockMetadata m_Metadata;
    4428 
    4429  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
    4430 
    4431  ~VmaDeviceMemoryBlock()
    4432  {
    4433  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
    4434  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    4435  }
    4436 
    4437  // Always call after construction.
    4438  void Init(
    4439  uint32_t newMemoryTypeIndex,
    4440  VkDeviceMemory newMemory,
    4441  VkDeviceSize newSize,
    4442  uint32_t id);
    4443  // Always call before destruction.
    4444  void Destroy(VmaAllocator allocator);
    4445 
    4446  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
    4447  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4448  uint32_t GetId() const { return m_Id; }
    4449  void* GetMappedData() const { return m_pMappedData; }
    4450 
    4451  // Validates all data structures inside this object. If not valid, returns false.
    4452  bool Validate() const;
    4453 
    4454  VkResult CheckCorruption(VmaAllocator hAllocator);
    4455 
    4456  // ppData can be null.
    4457  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
    4458  void Unmap(VmaAllocator hAllocator, uint32_t count);
    4459 
    4460  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4461  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4462 
    4463  VkResult BindBufferMemory(
    4464  const VmaAllocator hAllocator,
    4465  const VmaAllocation hAllocation,
    4466  VkBuffer hBuffer);
    4467  VkResult BindImageMemory(
    4468  const VmaAllocator hAllocator,
    4469  const VmaAllocation hAllocation,
    4470  VkImage hImage);
    4471 
    4472 private:
    4473  uint32_t m_MemoryTypeIndex;
    4474  uint32_t m_Id;
    4475  VkDeviceMemory m_hMemory;
    4476 
    4477  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
    4478  // Also protects m_MapCount, m_pMappedData.
    4479  VMA_MUTEX m_Mutex;
    4480  uint32_t m_MapCount;
    4481  void* m_pMappedData;
    4482 };
    4483 
    4484 struct VmaPointerLess
    4485 {
    4486  bool operator()(const void* lhs, const void* rhs) const
    4487  {
    4488  return lhs < rhs;
    4489  }
    4490 };
    4491 
    4492 class VmaDefragmentator;
    4493 
    4494 /*
    4495 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
    4496 Vulkan memory type.
    4497 
    4498 Synchronized internally with a mutex.
    4499 */
    4500 struct VmaBlockVector
    4501 {
    4502  VMA_CLASS_NO_COPY(VmaBlockVector)
    4503 public:
    4504  VmaBlockVector(
    4505  VmaAllocator hAllocator,
    4506  uint32_t memoryTypeIndex,
    4507  VkDeviceSize preferredBlockSize,
    4508  size_t minBlockCount,
    4509  size_t maxBlockCount,
    4510  VkDeviceSize bufferImageGranularity,
    4511  uint32_t frameInUseCount,
    4512  bool isCustomPool);
    4513  ~VmaBlockVector();
    4514 
    4515  VkResult CreateMinBlocks();
    4516 
    4517  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4518  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
    4519  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
    4520  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
    4521 
    4522  void GetPoolStats(VmaPoolStats* pStats);
    4523 
    4524  bool IsEmpty() const { return m_Blocks.empty(); }
    4525  bool IsCorruptionDetectionEnabled() const;
    4526 
    4527  VkResult Allocate(
    4528  VmaPool hCurrentPool,
    4529  uint32_t currentFrameIndex,
    4530  VkDeviceSize size,
    4531  VkDeviceSize alignment,
    4532  const VmaAllocationCreateInfo& createInfo,
    4533  VmaSuballocationType suballocType,
    4534  VmaAllocation* pAllocation);
    4535 
    4536  void Free(
    4537  VmaAllocation hAllocation);
    4538 
    4539  // Adds statistics of this BlockVector to pStats.
    4540  void AddStats(VmaStats* pStats);
    4541 
    4542 #if VMA_STATS_STRING_ENABLED
    4543  void PrintDetailedMap(class VmaJsonWriter& json);
    4544 #endif
    4545 
    4546  void MakePoolAllocationsLost(
    4547  uint32_t currentFrameIndex,
    4548  size_t* pLostAllocationCount);
    4549  VkResult CheckCorruption();
    4550 
    4551  VmaDefragmentator* EnsureDefragmentator(
    4552  VmaAllocator hAllocator,
    4553  uint32_t currentFrameIndex);
    4554 
    4555  VkResult Defragment(
    4556  VmaDefragmentationStats* pDefragmentationStats,
    4557  VkDeviceSize& maxBytesToMove,
    4558  uint32_t& maxAllocationsToMove);
    4559 
    4560  void DestroyDefragmentator();
    4561 
    4562 private:
    4563  friend class VmaDefragmentator;
    4564 
    4565  const VmaAllocator m_hAllocator;
    4566  const uint32_t m_MemoryTypeIndex;
    4567  const VkDeviceSize m_PreferredBlockSize;
    4568  const size_t m_MinBlockCount;
    4569  const size_t m_MaxBlockCount;
    4570  const VkDeviceSize m_BufferImageGranularity;
    4571  const uint32_t m_FrameInUseCount;
    4572  const bool m_IsCustomPool;
    4573  VMA_MUTEX m_Mutex;
    4574  // Incrementally sorted by sumFreeSize, ascending.
    4575  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
    4576  /* There can be at most one allocation that is completely empty - a
    4577  hysteresis to avoid pessimistic case of alternating creation and destruction
    4578  of a VkDeviceMemory. */
    4579  bool m_HasEmptyBlock;
    4580  VmaDefragmentator* m_pDefragmentator;
    4581  uint32_t m_NextBlockId;
    4582 
    4583  VkDeviceSize CalcMaxBlockSize() const;
    4584 
    4585  // Finds and removes given block from vector.
    4586  void Remove(VmaDeviceMemoryBlock* pBlock);
    4587 
    4588  // Performs single step in sorting m_Blocks. They may not be fully sorted
    4589  // after this call.
    4590  void IncrementallySortBlocks();
    4591 
    4592  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
    4593 };
    4594 
    4595 struct VmaPool_T
    4596 {
    4597  VMA_CLASS_NO_COPY(VmaPool_T)
    4598 public:
    4599  VmaBlockVector m_BlockVector;
    4600 
    4601  VmaPool_T(
    4602  VmaAllocator hAllocator,
    4603  const VmaPoolCreateInfo& createInfo);
    4604  ~VmaPool_T();
    4605 
    4606  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
    4607  uint32_t GetId() const { return m_Id; }
    4608  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
    4609 
    4610 #if VMA_STATS_STRING_ENABLED
    4611  //void PrintDetailedMap(class VmaStringBuilder& sb);
    4612 #endif
    4613 
    4614 private:
    4615  uint32_t m_Id;
    4616 };
    4617 
    4618 class VmaDefragmentator
    4619 {
    4620  VMA_CLASS_NO_COPY(VmaDefragmentator)
    4621 private:
    4622  const VmaAllocator m_hAllocator;
    4623  VmaBlockVector* const m_pBlockVector;
    4624  uint32_t m_CurrentFrameIndex;
    4625  VkDeviceSize m_BytesMoved;
    4626  uint32_t m_AllocationsMoved;
    4627 
    4628  struct AllocationInfo
    4629  {
    4630  VmaAllocation m_hAllocation;
    4631  VkBool32* m_pChanged;
    4632 
    4633  AllocationInfo() :
    4634  m_hAllocation(VK_NULL_HANDLE),
    4635  m_pChanged(VMA_NULL)
    4636  {
    4637  }
    4638  };
    4639 
    4640  struct AllocationInfoSizeGreater
    4641  {
    4642  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
    4643  {
    4644  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
    4645  }
    4646  };
    4647 
    4648  // Used between AddAllocation and Defragment.
    4649  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4650 
    4651  struct BlockInfo
    4652  {
    4653  VmaDeviceMemoryBlock* m_pBlock;
    4654  bool m_HasNonMovableAllocations;
    4655  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4656 
    4657  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
    4658  m_pBlock(VMA_NULL),
    4659  m_HasNonMovableAllocations(true),
    4660  m_Allocations(pAllocationCallbacks),
    4661  m_pMappedDataForDefragmentation(VMA_NULL)
    4662  {
    4663  }
    4664 
    4665  void CalcHasNonMovableAllocations()
    4666  {
    4667  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
    4668  const size_t defragmentAllocCount = m_Allocations.size();
    4669  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
    4670  }
    4671 
    4672  void SortAllocationsBySizeDescecnding()
    4673  {
    4674  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
    4675  }
    4676 
    4677  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
    4678  void Unmap(VmaAllocator hAllocator);
    4679 
    4680  private:
    4681  // Not null if mapped for defragmentation only, not originally mapped.
    4682  void* m_pMappedDataForDefragmentation;
    4683  };
    4684 
    4685  struct BlockPointerLess
    4686  {
    4687  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
    4688  {
    4689  return pLhsBlockInfo->m_pBlock < pRhsBlock;
    4690  }
    4691  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4692  {
    4693  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
    4694  }
    4695  };
    4696 
    4697  // 1. Blocks with some non-movable allocations go first.
    4698  // 2. Blocks with smaller sumFreeSize go first.
    4699  struct BlockInfoCompareMoveDestination
    4700  {
    4701  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4702  {
    4703  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
    4704  {
    4705  return true;
    4706  }
    4707  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
    4708  {
    4709  return false;
    4710  }
    4711  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
    4712  {
    4713  return true;
    4714  }
    4715  return false;
    4716  }
    4717  };
    4718 
    4719  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
    4720  BlockInfoVector m_Blocks;
    4721 
    4722  VkResult DefragmentRound(
    4723  VkDeviceSize maxBytesToMove,
    4724  uint32_t maxAllocationsToMove);
    4725 
    4726  static bool MoveMakesSense(
    4727  size_t dstBlockIndex, VkDeviceSize dstOffset,
    4728  size_t srcBlockIndex, VkDeviceSize srcOffset);
    4729 
    4730 public:
    4731  VmaDefragmentator(
    4732  VmaAllocator hAllocator,
    4733  VmaBlockVector* pBlockVector,
    4734  uint32_t currentFrameIndex);
    4735 
    4736  ~VmaDefragmentator();
    4737 
    4738  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
    4739  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
    4740 
    4741  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
    4742 
    4743  VkResult Defragment(
    4744  VkDeviceSize maxBytesToMove,
    4745  uint32_t maxAllocationsToMove);
    4746 };
    4747 
    4748 #if VMA_RECORDING_ENABLED
    4749 
    4750 class VmaRecorder
    4751 {
    4752 public:
    4753  VmaRecorder();
    4754  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
    4755  ~VmaRecorder();
    4756 
    4757  void RecordCreateAllocator(uint32_t frameIndex);
    4758  void RecordDestroyAllocator(uint32_t frameIndex);
    4759  void RecordCreatePool(uint32_t frameIndex,
    4760  const VmaPoolCreateInfo& createInfo,
    4761  VmaPool pool);
    4762  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
    4763  void RecordAllocateMemory(uint32_t frameIndex,
    4764  const VkMemoryRequirements& vkMemReq,
    4765  const VmaAllocationCreateInfo& createInfo,
    4766  VmaAllocation allocation);
    4767  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    4768  const VkMemoryRequirements& vkMemReq,
    4769  bool requiresDedicatedAllocation,
    4770  bool prefersDedicatedAllocation,
    4771  const VmaAllocationCreateInfo& createInfo,
    4772  VmaAllocation allocation);
    4773  void RecordAllocateMemoryForImage(uint32_t frameIndex,
    4774  const VkMemoryRequirements& vkMemReq,
    4775  bool requiresDedicatedAllocation,
    4776  bool prefersDedicatedAllocation,
    4777  const VmaAllocationCreateInfo& createInfo,
    4778  VmaAllocation allocation);
    4779  void RecordFreeMemory(uint32_t frameIndex,
    4780  VmaAllocation allocation);
    4781  void RecordSetAllocationUserData(uint32_t frameIndex,
    4782  VmaAllocation allocation,
    4783  const void* pUserData);
    4784  void RecordCreateLostAllocation(uint32_t frameIndex,
    4785  VmaAllocation allocation);
    4786  void RecordMapMemory(uint32_t frameIndex,
    4787  VmaAllocation allocation);
    4788  void RecordUnmapMemory(uint32_t frameIndex,
    4789  VmaAllocation allocation);
    4790  void RecordFlushAllocation(uint32_t frameIndex,
    4791  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4792  void RecordInvalidateAllocation(uint32_t frameIndex,
    4793  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4794  void RecordCreateBuffer(uint32_t frameIndex,
    4795  const VkBufferCreateInfo& bufCreateInfo,
    4796  const VmaAllocationCreateInfo& allocCreateInfo,
    4797  VmaAllocation allocation);
    4798  void RecordCreateImage(uint32_t frameIndex,
    4799  const VkImageCreateInfo& imageCreateInfo,
    4800  const VmaAllocationCreateInfo& allocCreateInfo,
    4801  VmaAllocation allocation);
    4802  void RecordDestroyBuffer(uint32_t frameIndex,
    4803  VmaAllocation allocation);
    4804  void RecordDestroyImage(uint32_t frameIndex,
    4805  VmaAllocation allocation);
    4806 
    4807 private:
    4808  struct CallParams
    4809  {
    4810  uint32_t threadId;
    4811  double time;
    4812  };
    4813 
    4814  class UserDataString
    4815  {
    4816  public:
    4817  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
    4818  const char* GetString() const { return m_Str; }
    4819 
    4820  private:
    4821  char m_PtrStr[17];
    4822  const char* m_Str;
    4823  };
    4824 
    4825  bool m_UseMutex;
    4826  VmaRecordFlags m_Flags;
    4827  FILE* m_File;
    4828  VMA_MUTEX m_FileMutex;
    4829  int64_t m_Freq;
    4830  int64_t m_StartCounter;
    4831 
    4832  void GetBasicParams(CallParams& outParams);
    4833  void Flush();
    4834 };
    4835 
    4836 #endif // #if VMA_RECORDING_ENABLED
    4837 
    4838 // Main allocator object.
    4839 struct VmaAllocator_T
    4840 {
    4841  VMA_CLASS_NO_COPY(VmaAllocator_T)
    4842 public:
    4843  bool m_UseMutex;
    4844  bool m_UseKhrDedicatedAllocation;
    4845  VkDevice m_hDevice;
    4846  bool m_AllocationCallbacksSpecified;
    4847  VkAllocationCallbacks m_AllocationCallbacks;
    4848  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
    4849 
    4850  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
    4851  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
    4852  VMA_MUTEX m_HeapSizeLimitMutex;
    4853 
    4854  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
    4855  VkPhysicalDeviceMemoryProperties m_MemProps;
    4856 
    4857  // Default pools.
    4858  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
    4859 
    4860  // Each vector is sorted by memory (handle value).
    4861  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
    4862  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
    4863  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
    4864 
    4865  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
    4866  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
    4867  ~VmaAllocator_T();
    4868 
    4869  const VkAllocationCallbacks* GetAllocationCallbacks() const
    4870  {
    4871  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
    4872  }
    4873  const VmaVulkanFunctions& GetVulkanFunctions() const
    4874  {
    4875  return m_VulkanFunctions;
    4876  }
    4877 
    4878  VkDeviceSize GetBufferImageGranularity() const
    4879  {
    4880  return VMA_MAX(
    4881  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
    4882  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
    4883  }
    4884 
    4885  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
    4886  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
    4887 
    4888  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
    4889  {
    4890  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
    4891  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
    4892  }
    4893  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
    4894  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
    4895  {
    4896  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
    4897  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    4898  }
    4899  // Minimum alignment for all allocations in specific memory type.
    4900  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
    4901  {
    4902  return IsMemoryTypeNonCoherent(memTypeIndex) ?
    4903  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
    4904  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
    4905  }
    4906 
    4907  bool IsIntegratedGpu() const
    4908  {
    4909  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
    4910  }
    4911 
    4912 #if VMA_RECORDING_ENABLED
    4913  VmaRecorder* GetRecorder() const { return m_pRecorder; }
    4914 #endif
    4915 
    4916  void GetBufferMemoryRequirements(
    4917  VkBuffer hBuffer,
    4918  VkMemoryRequirements& memReq,
    4919  bool& requiresDedicatedAllocation,
    4920  bool& prefersDedicatedAllocation) const;
    4921  void GetImageMemoryRequirements(
    4922  VkImage hImage,
    4923  VkMemoryRequirements& memReq,
    4924  bool& requiresDedicatedAllocation,
    4925  bool& prefersDedicatedAllocation) const;
    4926 
    4927  // Main allocation function.
    4928  VkResult AllocateMemory(
    4929  const VkMemoryRequirements& vkMemReq,
    4930  bool requiresDedicatedAllocation,
    4931  bool prefersDedicatedAllocation,
    4932  VkBuffer dedicatedBuffer,
    4933  VkImage dedicatedImage,
    4934  const VmaAllocationCreateInfo& createInfo,
    4935  VmaSuballocationType suballocType,
    4936  VmaAllocation* pAllocation);
    4937 
    4938  // Main deallocation function.
    4939  void FreeMemory(const VmaAllocation allocation);
    4940 
    4941  void CalculateStats(VmaStats* pStats);
    4942 
    4943 #if VMA_STATS_STRING_ENABLED
    4944  void PrintDetailedMap(class VmaJsonWriter& json);
    4945 #endif
    4946 
    4947  VkResult Defragment(
    4948  VmaAllocation* pAllocations,
    4949  size_t allocationCount,
    4950  VkBool32* pAllocationsChanged,
    4951  const VmaDefragmentationInfo* pDefragmentationInfo,
    4952  VmaDefragmentationStats* pDefragmentationStats);
    4953 
    4954  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
    4955  bool TouchAllocation(VmaAllocation hAllocation);
    4956 
    4957  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
    4958  void DestroyPool(VmaPool pool);
    4959  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
    4960 
    4961  void SetCurrentFrameIndex(uint32_t frameIndex);
    4962  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
    4963 
    4964  void MakePoolAllocationsLost(
    4965  VmaPool hPool,
    4966  size_t* pLostAllocationCount);
    4967  VkResult CheckPoolCorruption(VmaPool hPool);
    4968  VkResult CheckCorruption(uint32_t memoryTypeBits);
    4969 
    4970  void CreateLostAllocation(VmaAllocation* pAllocation);
    4971 
    4972  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
    4973  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
    4974 
    4975  VkResult Map(VmaAllocation hAllocation, void** ppData);
    4976  void Unmap(VmaAllocation hAllocation);
    4977 
    4978  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
    4979  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
    4980 
    4981  void FlushOrInvalidateAllocation(
    4982  VmaAllocation hAllocation,
    4983  VkDeviceSize offset, VkDeviceSize size,
    4984  VMA_CACHE_OPERATION op);
    4985 
    4986  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
    4987 
    4988 private:
    4989  VkDeviceSize m_PreferredLargeHeapBlockSize;
    4990 
    4991  VkPhysicalDevice m_PhysicalDevice;
    4992  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
    4993 
    4994  VMA_MUTEX m_PoolsMutex;
    4995  // Protected by m_PoolsMutex. Sorted by pointer value.
    4996  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
    4997  uint32_t m_NextPoolId;
    4998 
    4999  VmaVulkanFunctions m_VulkanFunctions;
    5000 
    5001 #if VMA_RECORDING_ENABLED
    5002  VmaRecorder* m_pRecorder;
    5003 #endif
    5004 
    5005  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
    5006 
    5007  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
    5008 
    5009  VkResult AllocateMemoryOfType(
    5010  VkDeviceSize size,
    5011  VkDeviceSize alignment,
    5012  bool dedicatedAllocation,
    5013  VkBuffer dedicatedBuffer,
    5014  VkImage dedicatedImage,
    5015  const VmaAllocationCreateInfo& createInfo,
    5016  uint32_t memTypeIndex,
    5017  VmaSuballocationType suballocType,
    5018  VmaAllocation* pAllocation);
    5019 
    5020  // Allocates and registers new VkDeviceMemory specifically for single allocation.
    5021  VkResult AllocateDedicatedMemory(
    5022  VkDeviceSize size,
    5023  VmaSuballocationType suballocType,
    5024  uint32_t memTypeIndex,
    5025  bool map,
    5026  bool isUserDataString,
    5027  void* pUserData,
    5028  VkBuffer dedicatedBuffer,
    5029  VkImage dedicatedImage,
    5030  VmaAllocation* pAllocation);
    5031 
    5032  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
    5033  void FreeDedicatedMemory(VmaAllocation allocation);
    5034 };
    5035 
    5037 // Memory allocation #2 after VmaAllocator_T definition
    5038 
    5039 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
    5040 {
    5041  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
    5042 }
    5043 
    5044 static void VmaFree(VmaAllocator hAllocator, void* ptr)
    5045 {
    5046  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
    5047 }
    5048 
    5049 template<typename T>
    5050 static T* VmaAllocate(VmaAllocator hAllocator)
    5051 {
    5052  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
    5053 }
    5054 
    5055 template<typename T>
    5056 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
    5057 {
    5058  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
    5059 }
    5060 
    5061 template<typename T>
    5062 static void vma_delete(VmaAllocator hAllocator, T* ptr)
    5063 {
    5064  if(ptr != VMA_NULL)
    5065  {
    5066  ptr->~T();
    5067  VmaFree(hAllocator, ptr);
    5068  }
    5069 }
    5070 
    5071 template<typename T>
    5072 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
    5073 {
    5074  if(ptr != VMA_NULL)
    5075  {
    5076  for(size_t i = count; i--; )
    5077  ptr[i].~T();
    5078  VmaFree(hAllocator, ptr);
    5079  }
    5080 }
    5081 
    5083 // VmaStringBuilder
    5084 
    5085 #if VMA_STATS_STRING_ENABLED
    5086 
    5087 class VmaStringBuilder
    5088 {
    5089 public:
    5090  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
    5091  size_t GetLength() const { return m_Data.size(); }
    5092  const char* GetData() const { return m_Data.data(); }
    5093 
    5094  void Add(char ch) { m_Data.push_back(ch); }
    5095  void Add(const char* pStr);
    5096  void AddNewLine() { Add('\n'); }
    5097  void AddNumber(uint32_t num);
    5098  void AddNumber(uint64_t num);
    5099  void AddPointer(const void* ptr);
    5100 
    5101 private:
    5102  VmaVector< char, VmaStlAllocator<char> > m_Data;
    5103 };
    5104 
    5105 void VmaStringBuilder::Add(const char* pStr)
    5106 {
    5107  const size_t strLen = strlen(pStr);
    5108  if(strLen > 0)
    5109  {
    5110  const size_t oldCount = m_Data.size();
    5111  m_Data.resize(oldCount + strLen);
    5112  memcpy(m_Data.data() + oldCount, pStr, strLen);
    5113  }
    5114 }
    5115 
    5116 void VmaStringBuilder::AddNumber(uint32_t num)
    5117 {
    5118  char buf[11];
    5119  VmaUint32ToStr(buf, sizeof(buf), num);
    5120  Add(buf);
    5121 }
    5122 
    5123 void VmaStringBuilder::AddNumber(uint64_t num)
    5124 {
    5125  char buf[21];
    5126  VmaUint64ToStr(buf, sizeof(buf), num);
    5127  Add(buf);
    5128 }
    5129 
    5130 void VmaStringBuilder::AddPointer(const void* ptr)
    5131 {
    5132  char buf[21];
    5133  VmaPtrToStr(buf, sizeof(buf), ptr);
    5134  Add(buf);
    5135 }
    5136 
    5137 #endif // #if VMA_STATS_STRING_ENABLED
    5138 
    5140 // VmaJsonWriter
    5141 
    5142 #if VMA_STATS_STRING_ENABLED
    5143 
    5144 class VmaJsonWriter
    5145 {
    5146  VMA_CLASS_NO_COPY(VmaJsonWriter)
    5147 public:
    5148  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
    5149  ~VmaJsonWriter();
    5150 
    5151  void BeginObject(bool singleLine = false);
    5152  void EndObject();
    5153 
    5154  void BeginArray(bool singleLine = false);
    5155  void EndArray();
    5156 
    5157  void WriteString(const char* pStr);
    5158  void BeginString(const char* pStr = VMA_NULL);
    5159  void ContinueString(const char* pStr);
    5160  void ContinueString(uint32_t n);
    5161  void ContinueString(uint64_t n);
    5162  void ContinueString_Pointer(const void* ptr);
    5163  void EndString(const char* pStr = VMA_NULL);
    5164 
    5165  void WriteNumber(uint32_t n);
    5166  void WriteNumber(uint64_t n);
    5167  void WriteBool(bool b);
    5168  void WriteNull();
    5169 
    5170 private:
    5171  static const char* const INDENT;
    5172 
    5173  enum COLLECTION_TYPE
    5174  {
    5175  COLLECTION_TYPE_OBJECT,
    5176  COLLECTION_TYPE_ARRAY,
    5177  };
    5178  struct StackItem
    5179  {
    5180  COLLECTION_TYPE type;
    5181  uint32_t valueCount;
    5182  bool singleLineMode;
    5183  };
    5184 
    5185  VmaStringBuilder& m_SB;
    5186  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
    5187  bool m_InsideString;
    5188 
    5189  void BeginValue(bool isString);
    5190  void WriteIndent(bool oneLess = false);
    5191 };
    5192 
    5193 const char* const VmaJsonWriter::INDENT = " ";
    5194 
    5195 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
    5196  m_SB(sb),
    5197  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
    5198  m_InsideString(false)
    5199 {
    5200 }
    5201 
    5202 VmaJsonWriter::~VmaJsonWriter()
    5203 {
    5204  VMA_ASSERT(!m_InsideString);
    5205  VMA_ASSERT(m_Stack.empty());
    5206 }
    5207 
    5208 void VmaJsonWriter::BeginObject(bool singleLine)
    5209 {
    5210  VMA_ASSERT(!m_InsideString);
    5211 
    5212  BeginValue(false);
    5213  m_SB.Add('{');
    5214 
    5215  StackItem item;
    5216  item.type = COLLECTION_TYPE_OBJECT;
    5217  item.valueCount = 0;
    5218  item.singleLineMode = singleLine;
    5219  m_Stack.push_back(item);
    5220 }
    5221 
    5222 void VmaJsonWriter::EndObject()
    5223 {
    5224  VMA_ASSERT(!m_InsideString);
    5225 
    5226  WriteIndent(true);
    5227  m_SB.Add('}');
    5228 
    5229  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
    5230  m_Stack.pop_back();
    5231 }
    5232 
    5233 void VmaJsonWriter::BeginArray(bool singleLine)
    5234 {
    5235  VMA_ASSERT(!m_InsideString);
    5236 
    5237  BeginValue(false);
    5238  m_SB.Add('[');
    5239 
    5240  StackItem item;
    5241  item.type = COLLECTION_TYPE_ARRAY;
    5242  item.valueCount = 0;
    5243  item.singleLineMode = singleLine;
    5244  m_Stack.push_back(item);
    5245 }
    5246 
    5247 void VmaJsonWriter::EndArray()
    5248 {
    5249  VMA_ASSERT(!m_InsideString);
    5250 
    5251  WriteIndent(true);
    5252  m_SB.Add(']');
    5253 
    5254  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
    5255  m_Stack.pop_back();
    5256 }
    5257 
    5258 void VmaJsonWriter::WriteString(const char* pStr)
    5259 {
    5260  BeginString(pStr);
    5261  EndString();
    5262 }
    5263 
    5264 void VmaJsonWriter::BeginString(const char* pStr)
    5265 {
    5266  VMA_ASSERT(!m_InsideString);
    5267 
    5268  BeginValue(true);
    5269  m_SB.Add('"');
    5270  m_InsideString = true;
    5271  if(pStr != VMA_NULL && pStr[0] != '\0')
    5272  {
    5273  ContinueString(pStr);
    5274  }
    5275 }
    5276 
    5277 void VmaJsonWriter::ContinueString(const char* pStr)
    5278 {
    5279  VMA_ASSERT(m_InsideString);
    5280 
    5281  const size_t strLen = strlen(pStr);
    5282  for(size_t i = 0; i < strLen; ++i)
    5283  {
    5284  char ch = pStr[i];
    5285  if(ch == '\\')
    5286  {
    5287  m_SB.Add("\\\\");
    5288  }
    5289  else if(ch == '"')
    5290  {
    5291  m_SB.Add("\\\"");
    5292  }
    5293  else if(ch >= 32)
    5294  {
    5295  m_SB.Add(ch);
    5296  }
    5297  else switch(ch)
    5298  {
    5299  case '\b':
    5300  m_SB.Add("\\b");
    5301  break;
    5302  case '\f':
    5303  m_SB.Add("\\f");
    5304  break;
    5305  case '\n':
    5306  m_SB.Add("\\n");
    5307  break;
    5308  case '\r':
    5309  m_SB.Add("\\r");
    5310  break;
    5311  case '\t':
    5312  m_SB.Add("\\t");
    5313  break;
    5314  default:
    5315  VMA_ASSERT(0 && "Character not currently supported.");
    5316  break;
    5317  }
    5318  }
    5319 }
    5320 
    5321 void VmaJsonWriter::ContinueString(uint32_t n)
    5322 {
    5323  VMA_ASSERT(m_InsideString);
    5324  m_SB.AddNumber(n);
    5325 }
    5326 
    5327 void VmaJsonWriter::ContinueString(uint64_t n)
    5328 {
    5329  VMA_ASSERT(m_InsideString);
    5330  m_SB.AddNumber(n);
    5331 }
    5332 
    5333 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
    5334 {
    5335  VMA_ASSERT(m_InsideString);
    5336  m_SB.AddPointer(ptr);
    5337 }
    5338 
    5339 void VmaJsonWriter::EndString(const char* pStr)
    5340 {
    5341  VMA_ASSERT(m_InsideString);
    5342  if(pStr != VMA_NULL && pStr[0] != '\0')
    5343  {
    5344  ContinueString(pStr);
    5345  }
    5346  m_SB.Add('"');
    5347  m_InsideString = false;
    5348 }
    5349 
    5350 void VmaJsonWriter::WriteNumber(uint32_t n)
    5351 {
    5352  VMA_ASSERT(!m_InsideString);
    5353  BeginValue(false);
    5354  m_SB.AddNumber(n);
    5355 }
    5356 
    5357 void VmaJsonWriter::WriteNumber(uint64_t n)
    5358 {
    5359  VMA_ASSERT(!m_InsideString);
    5360  BeginValue(false);
    5361  m_SB.AddNumber(n);
    5362 }
    5363 
    5364 void VmaJsonWriter::WriteBool(bool b)
    5365 {
    5366  VMA_ASSERT(!m_InsideString);
    5367  BeginValue(false);
    5368  m_SB.Add(b ? "true" : "false");
    5369 }
    5370 
    5371 void VmaJsonWriter::WriteNull()
    5372 {
    5373  VMA_ASSERT(!m_InsideString);
    5374  BeginValue(false);
    5375  m_SB.Add("null");
    5376 }
    5377 
    5378 void VmaJsonWriter::BeginValue(bool isString)
    5379 {
    5380  if(!m_Stack.empty())
    5381  {
    5382  StackItem& currItem = m_Stack.back();
    5383  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5384  currItem.valueCount % 2 == 0)
    5385  {
    5386  VMA_ASSERT(isString);
    5387  }
    5388 
    5389  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5390  currItem.valueCount % 2 != 0)
    5391  {
    5392  m_SB.Add(": ");
    5393  }
    5394  else if(currItem.valueCount > 0)
    5395  {
    5396  m_SB.Add(", ");
    5397  WriteIndent();
    5398  }
    5399  else
    5400  {
    5401  WriteIndent();
    5402  }
    5403  ++currItem.valueCount;
    5404  }
    5405 }
    5406 
    5407 void VmaJsonWriter::WriteIndent(bool oneLess)
    5408 {
    5409  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
    5410  {
    5411  m_SB.AddNewLine();
    5412 
    5413  size_t count = m_Stack.size();
    5414  if(count > 0 && oneLess)
    5415  {
    5416  --count;
    5417  }
    5418  for(size_t i = 0; i < count; ++i)
    5419  {
    5420  m_SB.Add(INDENT);
    5421  }
    5422  }
    5423 }
    5424 
    5425 #endif // #if VMA_STATS_STRING_ENABLED
    5426 
    5428 
    5429 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
    5430 {
    5431  if(IsUserDataString())
    5432  {
    5433  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
    5434 
    5435  FreeUserDataString(hAllocator);
    5436 
    5437  if(pUserData != VMA_NULL)
    5438  {
    5439  const char* const newStrSrc = (char*)pUserData;
    5440  const size_t newStrLen = strlen(newStrSrc);
    5441  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
    5442  memcpy(newStrDst, newStrSrc, newStrLen + 1);
    5443  m_pUserData = newStrDst;
    5444  }
    5445  }
    5446  else
    5447  {
    5448  m_pUserData = pUserData;
    5449  }
    5450 }
    5451 
    5452 void VmaAllocation_T::ChangeBlockAllocation(
    5453  VmaAllocator hAllocator,
    5454  VmaDeviceMemoryBlock* block,
    5455  VkDeviceSize offset)
    5456 {
    5457  VMA_ASSERT(block != VMA_NULL);
    5458  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5459 
    5460  // Move mapping reference counter from old block to new block.
    5461  if(block != m_BlockAllocation.m_Block)
    5462  {
    5463  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
    5464  if(IsPersistentMap())
    5465  ++mapRefCount;
    5466  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
    5467  block->Map(hAllocator, mapRefCount, VMA_NULL);
    5468  }
    5469 
    5470  m_BlockAllocation.m_Block = block;
    5471  m_BlockAllocation.m_Offset = offset;
    5472 }
    5473 
    5474 VkDeviceSize VmaAllocation_T::GetOffset() const
    5475 {
    5476  switch(m_Type)
    5477  {
    5478  case ALLOCATION_TYPE_BLOCK:
    5479  return m_BlockAllocation.m_Offset;
    5480  case ALLOCATION_TYPE_DEDICATED:
    5481  return 0;
    5482  default:
    5483  VMA_ASSERT(0);
    5484  return 0;
    5485  }
    5486 }
    5487 
    5488 VkDeviceMemory VmaAllocation_T::GetMemory() const
    5489 {
    5490  switch(m_Type)
    5491  {
    5492  case ALLOCATION_TYPE_BLOCK:
    5493  return m_BlockAllocation.m_Block->GetDeviceMemory();
    5494  case ALLOCATION_TYPE_DEDICATED:
    5495  return m_DedicatedAllocation.m_hMemory;
    5496  default:
    5497  VMA_ASSERT(0);
    5498  return VK_NULL_HANDLE;
    5499  }
    5500 }
    5501 
    5502 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
    5503 {
    5504  switch(m_Type)
    5505  {
    5506  case ALLOCATION_TYPE_BLOCK:
    5507  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
    5508  case ALLOCATION_TYPE_DEDICATED:
    5509  return m_DedicatedAllocation.m_MemoryTypeIndex;
    5510  default:
    5511  VMA_ASSERT(0);
    5512  return UINT32_MAX;
    5513  }
    5514 }
    5515 
    5516 void* VmaAllocation_T::GetMappedData() const
    5517 {
    5518  switch(m_Type)
    5519  {
    5520  case ALLOCATION_TYPE_BLOCK:
    5521  if(m_MapCount != 0)
    5522  {
    5523  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
    5524  VMA_ASSERT(pBlockData != VMA_NULL);
    5525  return (char*)pBlockData + m_BlockAllocation.m_Offset;
    5526  }
    5527  else
    5528  {
    5529  return VMA_NULL;
    5530  }
    5531  break;
    5532  case ALLOCATION_TYPE_DEDICATED:
    5533  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
    5534  return m_DedicatedAllocation.m_pMappedData;
    5535  default:
    5536  VMA_ASSERT(0);
    5537  return VMA_NULL;
    5538  }
    5539 }
    5540 
    5541 bool VmaAllocation_T::CanBecomeLost() const
    5542 {
    5543  switch(m_Type)
    5544  {
    5545  case ALLOCATION_TYPE_BLOCK:
    5546  return m_BlockAllocation.m_CanBecomeLost;
    5547  case ALLOCATION_TYPE_DEDICATED:
    5548  return false;
    5549  default:
    5550  VMA_ASSERT(0);
    5551  return false;
    5552  }
    5553 }
    5554 
    5555 VmaPool VmaAllocation_T::GetPool() const
    5556 {
    5557  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5558  return m_BlockAllocation.m_hPool;
    5559 }
    5560 
    5561 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    5562 {
    5563  VMA_ASSERT(CanBecomeLost());
    5564 
    5565  /*
    5566  Warning: This is a carefully designed algorithm.
    5567  Do not modify unless you really know what you're doing :)
    5568  */
    5569  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
    5570  for(;;)
    5571  {
    5572  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    5573  {
    5574  VMA_ASSERT(0);
    5575  return false;
    5576  }
    5577  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
    5578  {
    5579  return false;
    5580  }
    5581  else // Last use time earlier than current time.
    5582  {
    5583  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
    5584  {
    5585  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
    5586  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
    5587  return true;
    5588  }
    5589  }
    5590  }
    5591 }
    5592 
    5593 #if VMA_STATS_STRING_ENABLED
    5594 
    5595 // Correspond to values of enum VmaSuballocationType.
    5596 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
    5597  "FREE",
    5598  "UNKNOWN",
    5599  "BUFFER",
    5600  "IMAGE_UNKNOWN",
    5601  "IMAGE_LINEAR",
    5602  "IMAGE_OPTIMAL",
    5603 };
    5604 
    5605 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
    5606 {
    5607  json.WriteString("Type");
    5608  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
    5609 
    5610  json.WriteString("Size");
    5611  json.WriteNumber(m_Size);
    5612 
    5613  if(m_pUserData != VMA_NULL)
    5614  {
    5615  json.WriteString("UserData");
    5616  if(IsUserDataString())
    5617  {
    5618  json.WriteString((const char*)m_pUserData);
    5619  }
    5620  else
    5621  {
    5622  json.BeginString();
    5623  json.ContinueString_Pointer(m_pUserData);
    5624  json.EndString();
    5625  }
    5626  }
    5627 
    5628  json.WriteString("CreationFrameIndex");
    5629  json.WriteNumber(m_CreationFrameIndex);
    5630 
    5631  json.WriteString("LastUseFrameIndex");
    5632  json.WriteNumber(GetLastUseFrameIndex());
    5633 
    5634  if(m_BufferImageUsage != 0)
    5635  {
    5636  json.WriteString("Usage");
    5637  json.WriteNumber(m_BufferImageUsage);
    5638  }
    5639 }
    5640 
    5641 #endif
    5642 
    5643 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
    5644 {
    5645  VMA_ASSERT(IsUserDataString());
    5646  if(m_pUserData != VMA_NULL)
    5647  {
    5648  char* const oldStr = (char*)m_pUserData;
    5649  const size_t oldStrLen = strlen(oldStr);
    5650  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
    5651  m_pUserData = VMA_NULL;
    5652  }
    5653 }
    5654 
    5655 void VmaAllocation_T::BlockAllocMap()
    5656 {
    5657  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5658 
    5659  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5660  {
    5661  ++m_MapCount;
    5662  }
    5663  else
    5664  {
    5665  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
    5666  }
    5667 }
    5668 
    5669 void VmaAllocation_T::BlockAllocUnmap()
    5670 {
    5671  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5672 
    5673  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5674  {
    5675  --m_MapCount;
    5676  }
    5677  else
    5678  {
    5679  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
    5680  }
    5681 }
    5682 
    5683 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
    5684 {
    5685  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5686 
    5687  if(m_MapCount != 0)
    5688  {
    5689  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5690  {
    5691  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
    5692  *ppData = m_DedicatedAllocation.m_pMappedData;
    5693  ++m_MapCount;
    5694  return VK_SUCCESS;
    5695  }
    5696  else
    5697  {
    5698  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
    5699  return VK_ERROR_MEMORY_MAP_FAILED;
    5700  }
    5701  }
    5702  else
    5703  {
    5704  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    5705  hAllocator->m_hDevice,
    5706  m_DedicatedAllocation.m_hMemory,
    5707  0, // offset
    5708  VK_WHOLE_SIZE,
    5709  0, // flags
    5710  ppData);
    5711  if(result == VK_SUCCESS)
    5712  {
    5713  m_DedicatedAllocation.m_pMappedData = *ppData;
    5714  m_MapCount = 1;
    5715  }
    5716  return result;
    5717  }
    5718 }
    5719 
    5720 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
    5721 {
    5722  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5723 
    5724  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5725  {
    5726  --m_MapCount;
    5727  if(m_MapCount == 0)
    5728  {
    5729  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
    5730  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
    5731  hAllocator->m_hDevice,
    5732  m_DedicatedAllocation.m_hMemory);
    5733  }
    5734  }
    5735  else
    5736  {
    5737  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
    5738  }
    5739 }
    5740 
    5741 #if VMA_STATS_STRING_ENABLED
    5742 
    5743 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
    5744 {
    5745  json.BeginObject();
    5746 
    5747  json.WriteString("Blocks");
    5748  json.WriteNumber(stat.blockCount);
    5749 
    5750  json.WriteString("Allocations");
    5751  json.WriteNumber(stat.allocationCount);
    5752 
    5753  json.WriteString("UnusedRanges");
    5754  json.WriteNumber(stat.unusedRangeCount);
    5755 
    5756  json.WriteString("UsedBytes");
    5757  json.WriteNumber(stat.usedBytes);
    5758 
    5759  json.WriteString("UnusedBytes");
    5760  json.WriteNumber(stat.unusedBytes);
    5761 
    5762  if(stat.allocationCount > 1)
    5763  {
    5764  json.WriteString("AllocationSize");
    5765  json.BeginObject(true);
    5766  json.WriteString("Min");
    5767  json.WriteNumber(stat.allocationSizeMin);
    5768  json.WriteString("Avg");
    5769  json.WriteNumber(stat.allocationSizeAvg);
    5770  json.WriteString("Max");
    5771  json.WriteNumber(stat.allocationSizeMax);
    5772  json.EndObject();
    5773  }
    5774 
    5775  if(stat.unusedRangeCount > 1)
    5776  {
    5777  json.WriteString("UnusedRangeSize");
    5778  json.BeginObject(true);
    5779  json.WriteString("Min");
    5780  json.WriteNumber(stat.unusedRangeSizeMin);
    5781  json.WriteString("Avg");
    5782  json.WriteNumber(stat.unusedRangeSizeAvg);
    5783  json.WriteString("Max");
    5784  json.WriteNumber(stat.unusedRangeSizeMax);
    5785  json.EndObject();
    5786  }
    5787 
    5788  json.EndObject();
    5789 }
    5790 
    5791 #endif // #if VMA_STATS_STRING_ENABLED
    5792 
    5793 struct VmaSuballocationItemSizeLess
    5794 {
    5795  bool operator()(
    5796  const VmaSuballocationList::iterator lhs,
    5797  const VmaSuballocationList::iterator rhs) const
    5798  {
    5799  return lhs->size < rhs->size;
    5800  }
    5801  bool operator()(
    5802  const VmaSuballocationList::iterator lhs,
    5803  VkDeviceSize rhsSize) const
    5804  {
    5805  return lhs->size < rhsSize;
    5806  }
    5807 };
    5808 
    5810 // class VmaBlockMetadata
    5811 
    5812 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
    5813  m_Size(0),
    5814  m_FreeCount(0),
    5815  m_SumFreeSize(0),
    5816  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
    5817  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
    5818 {
    5819 }
    5820 
    5821 VmaBlockMetadata::~VmaBlockMetadata()
    5822 {
    5823 }
    5824 
    5825 void VmaBlockMetadata::Init(VkDeviceSize size)
    5826 {
    5827  m_Size = size;
    5828  m_FreeCount = 1;
    5829  m_SumFreeSize = size;
    5830 
    5831  VmaSuballocation suballoc = {};
    5832  suballoc.offset = 0;
    5833  suballoc.size = size;
    5834  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    5835  suballoc.hAllocation = VK_NULL_HANDLE;
    5836 
    5837  m_Suballocations.push_back(suballoc);
    5838  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
    5839  --suballocItem;
    5840  m_FreeSuballocationsBySize.push_back(suballocItem);
    5841 }
    5842 
    5843 bool VmaBlockMetadata::Validate() const
    5844 {
    5845  if(m_Suballocations.empty())
    5846  {
    5847  return false;
    5848  }
    5849 
    5850  // Expected offset of new suballocation as calculates from previous ones.
    5851  VkDeviceSize calculatedOffset = 0;
    5852  // Expected number of free suballocations as calculated from traversing their list.
    5853  uint32_t calculatedFreeCount = 0;
    5854  // Expected sum size of free suballocations as calculated from traversing their list.
    5855  VkDeviceSize calculatedSumFreeSize = 0;
    5856  // Expected number of free suballocations that should be registered in
    5857  // m_FreeSuballocationsBySize calculated from traversing their list.
    5858  size_t freeSuballocationsToRegister = 0;
    5859  // True if previous visited suballocation was free.
    5860  bool prevFree = false;
    5861 
    5862  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5863  suballocItem != m_Suballocations.cend();
    5864  ++suballocItem)
    5865  {
    5866  const VmaSuballocation& subAlloc = *suballocItem;
    5867 
    5868  // Actual offset of this suballocation doesn't match expected one.
    5869  if(subAlloc.offset != calculatedOffset)
    5870  {
    5871  return false;
    5872  }
    5873 
    5874  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
    5875  // Two adjacent free suballocations are invalid. They should be merged.
    5876  if(prevFree && currFree)
    5877  {
    5878  return false;
    5879  }
    5880 
    5881  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
    5882  {
    5883  return false;
    5884  }
    5885 
    5886  if(currFree)
    5887  {
    5888  calculatedSumFreeSize += subAlloc.size;
    5889  ++calculatedFreeCount;
    5890  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    5891  {
    5892  ++freeSuballocationsToRegister;
    5893  }
    5894 
    5895  // Margin required between allocations - every free space must be at least that large.
    5896  if(subAlloc.size < VMA_DEBUG_MARGIN)
    5897  {
    5898  return false;
    5899  }
    5900  }
    5901  else
    5902  {
    5903  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
    5904  {
    5905  return false;
    5906  }
    5907  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
    5908  {
    5909  return false;
    5910  }
    5911 
    5912  // Margin required between allocations - previous allocation must be free.
    5913  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
    5914  {
    5915  return false;
    5916  }
    5917  }
    5918 
    5919  calculatedOffset += subAlloc.size;
    5920  prevFree = currFree;
    5921  }
    5922 
    5923  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
    5924  // match expected one.
    5925  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
    5926  {
    5927  return false;
    5928  }
    5929 
    5930  VkDeviceSize lastSize = 0;
    5931  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
    5932  {
    5933  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
    5934 
    5935  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
    5936  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
    5937  {
    5938  return false;
    5939  }
    5940  // They must be sorted by size ascending.
    5941  if(suballocItem->size < lastSize)
    5942  {
    5943  return false;
    5944  }
    5945 
    5946  lastSize = suballocItem->size;
    5947  }
    5948 
    5949  // Check if totals match calculacted values.
    5950  if(!ValidateFreeSuballocationList() ||
    5951  (calculatedOffset != m_Size) ||
    5952  (calculatedSumFreeSize != m_SumFreeSize) ||
    5953  (calculatedFreeCount != m_FreeCount))
    5954  {
    5955  return false;
    5956  }
    5957 
    5958  return true;
    5959 }
    5960 
    5961 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
    5962 {
    5963  if(!m_FreeSuballocationsBySize.empty())
    5964  {
    5965  return m_FreeSuballocationsBySize.back()->size;
    5966  }
    5967  else
    5968  {
    5969  return 0;
    5970  }
    5971 }
    5972 
    5973 bool VmaBlockMetadata::IsEmpty() const
    5974 {
    5975  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
    5976 }
    5977 
    5978 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
    5979 {
    5980  outInfo.blockCount = 1;
    5981 
    5982  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    5983  outInfo.allocationCount = rangeCount - m_FreeCount;
    5984  outInfo.unusedRangeCount = m_FreeCount;
    5985 
    5986  outInfo.unusedBytes = m_SumFreeSize;
    5987  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
    5988 
    5989  outInfo.allocationSizeMin = UINT64_MAX;
    5990  outInfo.allocationSizeMax = 0;
    5991  outInfo.unusedRangeSizeMin = UINT64_MAX;
    5992  outInfo.unusedRangeSizeMax = 0;
    5993 
    5994  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5995  suballocItem != m_Suballocations.cend();
    5996  ++suballocItem)
    5997  {
    5998  const VmaSuballocation& suballoc = *suballocItem;
    5999  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
    6000  {
    6001  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
    6002  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
    6003  }
    6004  else
    6005  {
    6006  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
    6007  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
    6008  }
    6009  }
    6010 }
    6011 
    6012 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
    6013 {
    6014  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    6015 
    6016  inoutStats.size += m_Size;
    6017  inoutStats.unusedSize += m_SumFreeSize;
    6018  inoutStats.allocationCount += rangeCount - m_FreeCount;
    6019  inoutStats.unusedRangeCount += m_FreeCount;
    6020  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
    6021 }
    6022 
    6023 #if VMA_STATS_STRING_ENABLED
    6024 
    6025 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
    6026 {
    6027  json.BeginObject();
    6028 
    6029  json.WriteString("TotalBytes");
    6030  json.WriteNumber(m_Size);
    6031 
    6032  json.WriteString("UnusedBytes");
    6033  json.WriteNumber(m_SumFreeSize);
    6034 
    6035  json.WriteString("Allocations");
    6036  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
    6037 
    6038  json.WriteString("UnusedRanges");
    6039  json.WriteNumber(m_FreeCount);
    6040 
    6041  json.WriteString("Suballocations");
    6042  json.BeginArray();
    6043  size_t i = 0;
    6044  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    6045  suballocItem != m_Suballocations.cend();
    6046  ++suballocItem, ++i)
    6047  {
    6048  json.BeginObject(true);
    6049 
    6050  json.WriteString("Offset");
    6051  json.WriteNumber(suballocItem->offset);
    6052 
    6053  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6054  {
    6055  json.WriteString("Type");
    6056  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
    6057 
    6058  json.WriteString("Size");
    6059  json.WriteNumber(suballocItem->size);
    6060  }
    6061  else
    6062  {
    6063  suballocItem->hAllocation->PrintParameters(json);
    6064  }
    6065 
    6066  json.EndObject();
    6067  }
    6068  json.EndArray();
    6069 
    6070  json.EndObject();
    6071 }
    6072 
    6073 #endif // #if VMA_STATS_STRING_ENABLED
    6074 
    6075 /*
    6076 How many suitable free suballocations to analyze before choosing best one.
    6077 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
    6078  be chosen.
    6079 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
    6080  suballocations will be analized and best one will be chosen.
    6081 - Any other value is also acceptable.
    6082 */
    6083 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
    6084 
    6085 bool VmaBlockMetadata::CreateAllocationRequest(
    6086  uint32_t currentFrameIndex,
    6087  uint32_t frameInUseCount,
    6088  VkDeviceSize bufferImageGranularity,
    6089  VkDeviceSize allocSize,
    6090  VkDeviceSize allocAlignment,
    6091  VmaSuballocationType allocType,
    6092  bool canMakeOtherLost,
    6093  VmaAllocationRequest* pAllocationRequest)
    6094 {
    6095  VMA_ASSERT(allocSize > 0);
    6096  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6097  VMA_ASSERT(pAllocationRequest != VMA_NULL);
    6098  VMA_HEAVY_ASSERT(Validate());
    6099 
    6100  // There is not enough total free space in this block to fullfill the request: Early return.
    6101  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
    6102  {
    6103  return false;
    6104  }
    6105 
    6106  // New algorithm, efficiently searching freeSuballocationsBySize.
    6107  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
    6108  if(freeSuballocCount > 0)
    6109  {
    6110  if(VMA_BEST_FIT)
    6111  {
    6112  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
    6113  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6114  m_FreeSuballocationsBySize.data(),
    6115  m_FreeSuballocationsBySize.data() + freeSuballocCount,
    6116  allocSize + 2 * VMA_DEBUG_MARGIN,
    6117  VmaSuballocationItemSizeLess());
    6118  size_t index = it - m_FreeSuballocationsBySize.data();
    6119  for(; index < freeSuballocCount; ++index)
    6120  {
    6121  if(CheckAllocation(
    6122  currentFrameIndex,
    6123  frameInUseCount,
    6124  bufferImageGranularity,
    6125  allocSize,
    6126  allocAlignment,
    6127  allocType,
    6128  m_FreeSuballocationsBySize[index],
    6129  false, // canMakeOtherLost
    6130  &pAllocationRequest->offset,
    6131  &pAllocationRequest->itemsToMakeLostCount,
    6132  &pAllocationRequest->sumFreeSize,
    6133  &pAllocationRequest->sumItemSize))
    6134  {
    6135  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6136  return true;
    6137  }
    6138  }
    6139  }
    6140  else
    6141  {
    6142  // Search staring from biggest suballocations.
    6143  for(size_t index = freeSuballocCount; index--; )
    6144  {
    6145  if(CheckAllocation(
    6146  currentFrameIndex,
    6147  frameInUseCount,
    6148  bufferImageGranularity,
    6149  allocSize,
    6150  allocAlignment,
    6151  allocType,
    6152  m_FreeSuballocationsBySize[index],
    6153  false, // canMakeOtherLost
    6154  &pAllocationRequest->offset,
    6155  &pAllocationRequest->itemsToMakeLostCount,
    6156  &pAllocationRequest->sumFreeSize,
    6157  &pAllocationRequest->sumItemSize))
    6158  {
    6159  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6160  return true;
    6161  }
    6162  }
    6163  }
    6164  }
    6165 
    6166  if(canMakeOtherLost)
    6167  {
    6168  // Brute-force algorithm. TODO: Come up with something better.
    6169 
    6170  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
    6171  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
    6172 
    6173  VmaAllocationRequest tmpAllocRequest = {};
    6174  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
    6175  suballocIt != m_Suballocations.end();
    6176  ++suballocIt)
    6177  {
    6178  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
    6179  suballocIt->hAllocation->CanBecomeLost())
    6180  {
    6181  if(CheckAllocation(
    6182  currentFrameIndex,
    6183  frameInUseCount,
    6184  bufferImageGranularity,
    6185  allocSize,
    6186  allocAlignment,
    6187  allocType,
    6188  suballocIt,
    6189  canMakeOtherLost,
    6190  &tmpAllocRequest.offset,
    6191  &tmpAllocRequest.itemsToMakeLostCount,
    6192  &tmpAllocRequest.sumFreeSize,
    6193  &tmpAllocRequest.sumItemSize))
    6194  {
    6195  tmpAllocRequest.item = suballocIt;
    6196 
    6197  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
    6198  {
    6199  *pAllocationRequest = tmpAllocRequest;
    6200  }
    6201  }
    6202  }
    6203  }
    6204 
    6205  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
    6206  {
    6207  return true;
    6208  }
    6209  }
    6210 
    6211  return false;
    6212 }
    6213 
    6214 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
    6215  uint32_t currentFrameIndex,
    6216  uint32_t frameInUseCount,
    6217  VmaAllocationRequest* pAllocationRequest)
    6218 {
    6219  while(pAllocationRequest->itemsToMakeLostCount > 0)
    6220  {
    6221  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
    6222  {
    6223  ++pAllocationRequest->item;
    6224  }
    6225  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6226  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
    6227  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
    6228  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6229  {
    6230  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
    6231  --pAllocationRequest->itemsToMakeLostCount;
    6232  }
    6233  else
    6234  {
    6235  return false;
    6236  }
    6237  }
    6238 
    6239  VMA_HEAVY_ASSERT(Validate());
    6240  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6241  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6242 
    6243  return true;
    6244 }
    6245 
    6246 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    6247 {
    6248  uint32_t lostAllocationCount = 0;
    6249  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6250  it != m_Suballocations.end();
    6251  ++it)
    6252  {
    6253  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
    6254  it->hAllocation->CanBecomeLost() &&
    6255  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6256  {
    6257  it = FreeSuballocation(it);
    6258  ++lostAllocationCount;
    6259  }
    6260  }
    6261  return lostAllocationCount;
    6262 }
    6263 
    6264 VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData)
    6265 {
    6266  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6267  it != m_Suballocations.end();
    6268  ++it)
    6269  {
    6270  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6271  {
    6272  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
    6273  {
    6274  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
    6275  return VK_ERROR_VALIDATION_FAILED_EXT;
    6276  }
    6277  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
    6278  {
    6279  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
    6280  return VK_ERROR_VALIDATION_FAILED_EXT;
    6281  }
    6282  }
    6283  }
    6284 
    6285  return VK_SUCCESS;
    6286 }
    6287 
    6288 void VmaBlockMetadata::Alloc(
    6289  const VmaAllocationRequest& request,
    6290  VmaSuballocationType type,
    6291  VkDeviceSize allocSize,
    6292  VmaAllocation hAllocation)
    6293 {
    6294  VMA_ASSERT(request.item != m_Suballocations.end());
    6295  VmaSuballocation& suballoc = *request.item;
    6296  // Given suballocation is a free block.
    6297  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6298  // Given offset is inside this suballocation.
    6299  VMA_ASSERT(request.offset >= suballoc.offset);
    6300  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
    6301  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
    6302  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
    6303 
    6304  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
    6305  // it to become used.
    6306  UnregisterFreeSuballocation(request.item);
    6307 
    6308  suballoc.offset = request.offset;
    6309  suballoc.size = allocSize;
    6310  suballoc.type = type;
    6311  suballoc.hAllocation = hAllocation;
    6312 
    6313  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
    6314  if(paddingEnd)
    6315  {
    6316  VmaSuballocation paddingSuballoc = {};
    6317  paddingSuballoc.offset = request.offset + allocSize;
    6318  paddingSuballoc.size = paddingEnd;
    6319  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6320  VmaSuballocationList::iterator next = request.item;
    6321  ++next;
    6322  const VmaSuballocationList::iterator paddingEndItem =
    6323  m_Suballocations.insert(next, paddingSuballoc);
    6324  RegisterFreeSuballocation(paddingEndItem);
    6325  }
    6326 
    6327  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
    6328  if(paddingBegin)
    6329  {
    6330  VmaSuballocation paddingSuballoc = {};
    6331  paddingSuballoc.offset = request.offset - paddingBegin;
    6332  paddingSuballoc.size = paddingBegin;
    6333  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6334  const VmaSuballocationList::iterator paddingBeginItem =
    6335  m_Suballocations.insert(request.item, paddingSuballoc);
    6336  RegisterFreeSuballocation(paddingBeginItem);
    6337  }
    6338 
    6339  // Update totals.
    6340  m_FreeCount = m_FreeCount - 1;
    6341  if(paddingBegin > 0)
    6342  {
    6343  ++m_FreeCount;
    6344  }
    6345  if(paddingEnd > 0)
    6346  {
    6347  ++m_FreeCount;
    6348  }
    6349  m_SumFreeSize -= allocSize;
    6350 }
    6351 
    6352 void VmaBlockMetadata::Free(const VmaAllocation allocation)
    6353 {
    6354  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6355  suballocItem != m_Suballocations.end();
    6356  ++suballocItem)
    6357  {
    6358  VmaSuballocation& suballoc = *suballocItem;
    6359  if(suballoc.hAllocation == allocation)
    6360  {
    6361  FreeSuballocation(suballocItem);
    6362  VMA_HEAVY_ASSERT(Validate());
    6363  return;
    6364  }
    6365  }
    6366  VMA_ASSERT(0 && "Not found!");
    6367 }
    6368 
    6369 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
    6370 {
    6371  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6372  suballocItem != m_Suballocations.end();
    6373  ++suballocItem)
    6374  {
    6375  VmaSuballocation& suballoc = *suballocItem;
    6376  if(suballoc.offset == offset)
    6377  {
    6378  FreeSuballocation(suballocItem);
    6379  return;
    6380  }
    6381  }
    6382  VMA_ASSERT(0 && "Not found!");
    6383 }
    6384 
    6385 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
    6386 {
    6387  VkDeviceSize lastSize = 0;
    6388  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
    6389  {
    6390  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
    6391 
    6392  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6393  {
    6394  VMA_ASSERT(0);
    6395  return false;
    6396  }
    6397  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6398  {
    6399  VMA_ASSERT(0);
    6400  return false;
    6401  }
    6402  if(it->size < lastSize)
    6403  {
    6404  VMA_ASSERT(0);
    6405  return false;
    6406  }
    6407 
    6408  lastSize = it->size;
    6409  }
    6410  return true;
    6411 }
    6412 
    6413 bool VmaBlockMetadata::CheckAllocation(
    6414  uint32_t currentFrameIndex,
    6415  uint32_t frameInUseCount,
    6416  VkDeviceSize bufferImageGranularity,
    6417  VkDeviceSize allocSize,
    6418  VkDeviceSize allocAlignment,
    6419  VmaSuballocationType allocType,
    6420  VmaSuballocationList::const_iterator suballocItem,
    6421  bool canMakeOtherLost,
    6422  VkDeviceSize* pOffset,
    6423  size_t* itemsToMakeLostCount,
    6424  VkDeviceSize* pSumFreeSize,
    6425  VkDeviceSize* pSumItemSize) const
    6426 {
    6427  VMA_ASSERT(allocSize > 0);
    6428  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6429  VMA_ASSERT(suballocItem != m_Suballocations.cend());
    6430  VMA_ASSERT(pOffset != VMA_NULL);
    6431 
    6432  *itemsToMakeLostCount = 0;
    6433  *pSumFreeSize = 0;
    6434  *pSumItemSize = 0;
    6435 
    6436  if(canMakeOtherLost)
    6437  {
    6438  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6439  {
    6440  *pSumFreeSize = suballocItem->size;
    6441  }
    6442  else
    6443  {
    6444  if(suballocItem->hAllocation->CanBecomeLost() &&
    6445  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6446  {
    6447  ++*itemsToMakeLostCount;
    6448  *pSumItemSize = suballocItem->size;
    6449  }
    6450  else
    6451  {
    6452  return false;
    6453  }
    6454  }
    6455 
    6456  // Remaining size is too small for this request: Early return.
    6457  if(m_Size - suballocItem->offset < allocSize)
    6458  {
    6459  return false;
    6460  }
    6461 
    6462  // Start from offset equal to beginning of this suballocation.
    6463  *pOffset = suballocItem->offset;
    6464 
    6465  // Apply VMA_DEBUG_MARGIN at the beginning.
    6466  if(VMA_DEBUG_MARGIN > 0)
    6467  {
    6468  *pOffset += VMA_DEBUG_MARGIN;
    6469  }
    6470 
    6471  // Apply alignment.
    6472  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6473 
    6474  // Check previous suballocations for BufferImageGranularity conflicts.
    6475  // Make bigger alignment if necessary.
    6476  if(bufferImageGranularity > 1)
    6477  {
    6478  bool bufferImageGranularityConflict = false;
    6479  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6480  while(prevSuballocItem != m_Suballocations.cbegin())
    6481  {
    6482  --prevSuballocItem;
    6483  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6484  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6485  {
    6486  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6487  {
    6488  bufferImageGranularityConflict = true;
    6489  break;
    6490  }
    6491  }
    6492  else
    6493  // Already on previous page.
    6494  break;
    6495  }
    6496  if(bufferImageGranularityConflict)
    6497  {
    6498  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6499  }
    6500  }
    6501 
    6502  // Now that we have final *pOffset, check if we are past suballocItem.
    6503  // If yes, return false - this function should be called for another suballocItem as starting point.
    6504  if(*pOffset >= suballocItem->offset + suballocItem->size)
    6505  {
    6506  return false;
    6507  }
    6508 
    6509  // Calculate padding at the beginning based on current offset.
    6510  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
    6511 
    6512  // Calculate required margin at the end.
    6513  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6514 
    6515  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
    6516  // Another early return check.
    6517  if(suballocItem->offset + totalSize > m_Size)
    6518  {
    6519  return false;
    6520  }
    6521 
    6522  // Advance lastSuballocItem until desired size is reached.
    6523  // Update itemsToMakeLostCount.
    6524  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
    6525  if(totalSize > suballocItem->size)
    6526  {
    6527  VkDeviceSize remainingSize = totalSize - suballocItem->size;
    6528  while(remainingSize > 0)
    6529  {
    6530  ++lastSuballocItem;
    6531  if(lastSuballocItem == m_Suballocations.cend())
    6532  {
    6533  return false;
    6534  }
    6535  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6536  {
    6537  *pSumFreeSize += lastSuballocItem->size;
    6538  }
    6539  else
    6540  {
    6541  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
    6542  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
    6543  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6544  {
    6545  ++*itemsToMakeLostCount;
    6546  *pSumItemSize += lastSuballocItem->size;
    6547  }
    6548  else
    6549  {
    6550  return false;
    6551  }
    6552  }
    6553  remainingSize = (lastSuballocItem->size < remainingSize) ?
    6554  remainingSize - lastSuballocItem->size : 0;
    6555  }
    6556  }
    6557 
    6558  // Check next suballocations for BufferImageGranularity conflicts.
    6559  // If conflict exists, we must mark more allocations lost or fail.
    6560  if(bufferImageGranularity > 1)
    6561  {
    6562  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
    6563  ++nextSuballocItem;
    6564  while(nextSuballocItem != m_Suballocations.cend())
    6565  {
    6566  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6567  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6568  {
    6569  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6570  {
    6571  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
    6572  if(nextSuballoc.hAllocation->CanBecomeLost() &&
    6573  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6574  {
    6575  ++*itemsToMakeLostCount;
    6576  }
    6577  else
    6578  {
    6579  return false;
    6580  }
    6581  }
    6582  }
    6583  else
    6584  {
    6585  // Already on next page.
    6586  break;
    6587  }
    6588  ++nextSuballocItem;
    6589  }
    6590  }
    6591  }
    6592  else
    6593  {
    6594  const VmaSuballocation& suballoc = *suballocItem;
    6595  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6596 
    6597  *pSumFreeSize = suballoc.size;
    6598 
    6599  // Size of this suballocation is too small for this request: Early return.
    6600  if(suballoc.size < allocSize)
    6601  {
    6602  return false;
    6603  }
    6604 
    6605  // Start from offset equal to beginning of this suballocation.
    6606  *pOffset = suballoc.offset;
    6607 
    6608  // Apply VMA_DEBUG_MARGIN at the beginning.
    6609  if(VMA_DEBUG_MARGIN > 0)
    6610  {
    6611  *pOffset += VMA_DEBUG_MARGIN;
    6612  }
    6613 
    6614  // Apply alignment.
    6615  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6616 
    6617  // Check previous suballocations for BufferImageGranularity conflicts.
    6618  // Make bigger alignment if necessary.
    6619  if(bufferImageGranularity > 1)
    6620  {
    6621  bool bufferImageGranularityConflict = false;
    6622  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6623  while(prevSuballocItem != m_Suballocations.cbegin())
    6624  {
    6625  --prevSuballocItem;
    6626  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6627  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6628  {
    6629  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6630  {
    6631  bufferImageGranularityConflict = true;
    6632  break;
    6633  }
    6634  }
    6635  else
    6636  // Already on previous page.
    6637  break;
    6638  }
    6639  if(bufferImageGranularityConflict)
    6640  {
    6641  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6642  }
    6643  }
    6644 
    6645  // Calculate padding at the beginning based on current offset.
    6646  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
    6647 
    6648  // Calculate required margin at the end.
    6649  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6650 
    6651  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
    6652  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
    6653  {
    6654  return false;
    6655  }
    6656 
    6657  // Check next suballocations for BufferImageGranularity conflicts.
    6658  // If conflict exists, allocation cannot be made here.
    6659  if(bufferImageGranularity > 1)
    6660  {
    6661  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
    6662  ++nextSuballocItem;
    6663  while(nextSuballocItem != m_Suballocations.cend())
    6664  {
    6665  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6666  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6667  {
    6668  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6669  {
    6670  return false;
    6671  }
    6672  }
    6673  else
    6674  {
    6675  // Already on next page.
    6676  break;
    6677  }
    6678  ++nextSuballocItem;
    6679  }
    6680  }
    6681  }
    6682 
    6683  // All tests passed: Success. pOffset is already filled.
    6684  return true;
    6685 }
    6686 
    6687 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
    6688 {
    6689  VMA_ASSERT(item != m_Suballocations.end());
    6690  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6691 
    6692  VmaSuballocationList::iterator nextItem = item;
    6693  ++nextItem;
    6694  VMA_ASSERT(nextItem != m_Suballocations.end());
    6695  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
    6696 
    6697  item->size += nextItem->size;
    6698  --m_FreeCount;
    6699  m_Suballocations.erase(nextItem);
    6700 }
    6701 
    6702 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
    6703 {
    6704  // Change this suballocation to be marked as free.
    6705  VmaSuballocation& suballoc = *suballocItem;
    6706  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6707  suballoc.hAllocation = VK_NULL_HANDLE;
    6708 
    6709  // Update totals.
    6710  ++m_FreeCount;
    6711  m_SumFreeSize += suballoc.size;
    6712 
    6713  // Merge with previous and/or next suballocation if it's also free.
    6714  bool mergeWithNext = false;
    6715  bool mergeWithPrev = false;
    6716 
    6717  VmaSuballocationList::iterator nextItem = suballocItem;
    6718  ++nextItem;
    6719  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
    6720  {
    6721  mergeWithNext = true;
    6722  }
    6723 
    6724  VmaSuballocationList::iterator prevItem = suballocItem;
    6725  if(suballocItem != m_Suballocations.begin())
    6726  {
    6727  --prevItem;
    6728  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6729  {
    6730  mergeWithPrev = true;
    6731  }
    6732  }
    6733 
    6734  if(mergeWithNext)
    6735  {
    6736  UnregisterFreeSuballocation(nextItem);
    6737  MergeFreeWithNext(suballocItem);
    6738  }
    6739 
    6740  if(mergeWithPrev)
    6741  {
    6742  UnregisterFreeSuballocation(prevItem);
    6743  MergeFreeWithNext(prevItem);
    6744  RegisterFreeSuballocation(prevItem);
    6745  return prevItem;
    6746  }
    6747  else
    6748  {
    6749  RegisterFreeSuballocation(suballocItem);
    6750  return suballocItem;
    6751  }
    6752 }
    6753 
    6754 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
    6755 {
    6756  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6757  VMA_ASSERT(item->size > 0);
    6758 
    6759  // You may want to enable this validation at the beginning or at the end of
    6760  // this function, depending on what do you want to check.
    6761  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6762 
    6763  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6764  {
    6765  if(m_FreeSuballocationsBySize.empty())
    6766  {
    6767  m_FreeSuballocationsBySize.push_back(item);
    6768  }
    6769  else
    6770  {
    6771  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
    6772  }
    6773  }
    6774 
    6775  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6776 }
    6777 
    6778 
    6779 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
    6780 {
    6781  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6782  VMA_ASSERT(item->size > 0);
    6783 
    6784  // You may want to enable this validation at the beginning or at the end of
    6785  // this function, depending on what do you want to check.
    6786  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6787 
    6788  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6789  {
    6790  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6791  m_FreeSuballocationsBySize.data(),
    6792  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
    6793  item,
    6794  VmaSuballocationItemSizeLess());
    6795  for(size_t index = it - m_FreeSuballocationsBySize.data();
    6796  index < m_FreeSuballocationsBySize.size();
    6797  ++index)
    6798  {
    6799  if(m_FreeSuballocationsBySize[index] == item)
    6800  {
    6801  VmaVectorRemove(m_FreeSuballocationsBySize, index);
    6802  return;
    6803  }
    6804  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
    6805  }
    6806  VMA_ASSERT(0 && "Not found.");
    6807  }
    6808 
    6809  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6810 }
    6811 
    6813 // class VmaDeviceMemoryBlock
    6814 
    6815 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
    6816  m_Metadata(hAllocator),
    6817  m_MemoryTypeIndex(UINT32_MAX),
    6818  m_Id(0),
    6819  m_hMemory(VK_NULL_HANDLE),
    6820  m_MapCount(0),
    6821  m_pMappedData(VMA_NULL)
    6822 {
    6823 }
    6824 
    6825 void VmaDeviceMemoryBlock::Init(
    6826  uint32_t newMemoryTypeIndex,
    6827  VkDeviceMemory newMemory,
    6828  VkDeviceSize newSize,
    6829  uint32_t id)
    6830 {
    6831  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    6832 
    6833  m_MemoryTypeIndex = newMemoryTypeIndex;
    6834  m_Id = id;
    6835  m_hMemory = newMemory;
    6836 
    6837  m_Metadata.Init(newSize);
    6838 }
    6839 
    6840 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
    6841 {
    6842  // This is the most important assert in the entire library.
    6843  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
    6844  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
    6845 
    6846  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
    6847  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
    6848  m_hMemory = VK_NULL_HANDLE;
    6849 }
    6850 
    6851 bool VmaDeviceMemoryBlock::Validate() const
    6852 {
    6853  if((m_hMemory == VK_NULL_HANDLE) ||
    6854  (m_Metadata.GetSize() == 0))
    6855  {
    6856  return false;
    6857  }
    6858 
    6859  return m_Metadata.Validate();
    6860 }
    6861 
    6862 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
    6863 {
    6864  void* pData = nullptr;
    6865  VkResult res = Map(hAllocator, 1, &pData);
    6866  if(res != VK_SUCCESS)
    6867  {
    6868  return res;
    6869  }
    6870 
    6871  res = m_Metadata.CheckCorruption(pData);
    6872 
    6873  Unmap(hAllocator, 1);
    6874 
    6875  return res;
    6876 }
    6877 
    6878 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
    6879 {
    6880  if(count == 0)
    6881  {
    6882  return VK_SUCCESS;
    6883  }
    6884 
    6885  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6886  if(m_MapCount != 0)
    6887  {
    6888  m_MapCount += count;
    6889  VMA_ASSERT(m_pMappedData != VMA_NULL);
    6890  if(ppData != VMA_NULL)
    6891  {
    6892  *ppData = m_pMappedData;
    6893  }
    6894  return VK_SUCCESS;
    6895  }
    6896  else
    6897  {
    6898  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    6899  hAllocator->m_hDevice,
    6900  m_hMemory,
    6901  0, // offset
    6902  VK_WHOLE_SIZE,
    6903  0, // flags
    6904  &m_pMappedData);
    6905  if(result == VK_SUCCESS)
    6906  {
    6907  if(ppData != VMA_NULL)
    6908  {
    6909  *ppData = m_pMappedData;
    6910  }
    6911  m_MapCount = count;
    6912  }
    6913  return result;
    6914  }
    6915 }
    6916 
    6917 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
    6918 {
    6919  if(count == 0)
    6920  {
    6921  return;
    6922  }
    6923 
    6924  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6925  if(m_MapCount >= count)
    6926  {
    6927  m_MapCount -= count;
    6928  if(m_MapCount == 0)
    6929  {
    6930  m_pMappedData = VMA_NULL;
    6931  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
    6932  }
    6933  }
    6934  else
    6935  {
    6936  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
    6937  }
    6938 }
    6939 
    6940 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6941 {
    6942  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    6943  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    6944 
    6945  void* pData;
    6946  VkResult res = Map(hAllocator, 1, &pData);
    6947  if(res != VK_SUCCESS)
    6948  {
    6949  return res;
    6950  }
    6951 
    6952  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
    6953  VmaWriteMagicValue(pData, allocOffset + allocSize);
    6954 
    6955  Unmap(hAllocator, 1);
    6956 
    6957  return VK_SUCCESS;
    6958 }
    6959 
    6960 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6961 {
    6962  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    6963  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    6964 
    6965  void* pData;
    6966  VkResult res = Map(hAllocator, 1, &pData);
    6967  if(res != VK_SUCCESS)
    6968  {
    6969  return res;
    6970  }
    6971 
    6972  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
    6973  {
    6974  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
    6975  }
    6976  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
    6977  {
    6978  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
    6979  }
    6980 
    6981  Unmap(hAllocator, 1);
    6982 
    6983  return VK_SUCCESS;
    6984 }
    6985 
    6986 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
    6987  const VmaAllocator hAllocator,
    6988  const VmaAllocation hAllocation,
    6989  VkBuffer hBuffer)
    6990 {
    6991  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    6992  hAllocation->GetBlock() == this);
    6993  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    6994  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6995  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
    6996  hAllocator->m_hDevice,
    6997  hBuffer,
    6998  m_hMemory,
    6999  hAllocation->GetOffset());
    7000 }
    7001 
    7002 VkResult VmaDeviceMemoryBlock::BindImageMemory(
    7003  const VmaAllocator hAllocator,
    7004  const VmaAllocation hAllocation,
    7005  VkImage hImage)
    7006 {
    7007  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    7008  hAllocation->GetBlock() == this);
    7009  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    7010  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    7011  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
    7012  hAllocator->m_hDevice,
    7013  hImage,
    7014  m_hMemory,
    7015  hAllocation->GetOffset());
    7016 }
    7017 
    7018 static void InitStatInfo(VmaStatInfo& outInfo)
    7019 {
    7020  memset(&outInfo, 0, sizeof(outInfo));
    7021  outInfo.allocationSizeMin = UINT64_MAX;
    7022  outInfo.unusedRangeSizeMin = UINT64_MAX;
    7023 }
    7024 
    7025 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
    7026 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
    7027 {
    7028  inoutInfo.blockCount += srcInfo.blockCount;
    7029  inoutInfo.allocationCount += srcInfo.allocationCount;
    7030  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
    7031  inoutInfo.usedBytes += srcInfo.usedBytes;
    7032  inoutInfo.unusedBytes += srcInfo.unusedBytes;
    7033  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
    7034  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
    7035  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
    7036  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
    7037 }
    7038 
    7039 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
    7040 {
    7041  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
    7042  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
    7043  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
    7044  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
    7045 }
    7046 
    7047 VmaPool_T::VmaPool_T(
    7048  VmaAllocator hAllocator,
    7049  const VmaPoolCreateInfo& createInfo) :
    7050  m_BlockVector(
    7051  hAllocator,
    7052  createInfo.memoryTypeIndex,
    7053  createInfo.blockSize,
    7054  createInfo.minBlockCount,
    7055  createInfo.maxBlockCount,
    7056  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
    7057  createInfo.frameInUseCount,
    7058  true), // isCustomPool
    7059  m_Id(0)
    7060 {
    7061 }
    7062 
    7063 VmaPool_T::~VmaPool_T()
    7064 {
    7065 }
    7066 
    7067 #if VMA_STATS_STRING_ENABLED
    7068 
    7069 #endif // #if VMA_STATS_STRING_ENABLED
    7070 
    7071 VmaBlockVector::VmaBlockVector(
    7072  VmaAllocator hAllocator,
    7073  uint32_t memoryTypeIndex,
    7074  VkDeviceSize preferredBlockSize,
    7075  size_t minBlockCount,
    7076  size_t maxBlockCount,
    7077  VkDeviceSize bufferImageGranularity,
    7078  uint32_t frameInUseCount,
    7079  bool isCustomPool) :
    7080  m_hAllocator(hAllocator),
    7081  m_MemoryTypeIndex(memoryTypeIndex),
    7082  m_PreferredBlockSize(preferredBlockSize),
    7083  m_MinBlockCount(minBlockCount),
    7084  m_MaxBlockCount(maxBlockCount),
    7085  m_BufferImageGranularity(bufferImageGranularity),
    7086  m_FrameInUseCount(frameInUseCount),
    7087  m_IsCustomPool(isCustomPool),
    7088  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
    7089  m_HasEmptyBlock(false),
    7090  m_pDefragmentator(VMA_NULL),
    7091  m_NextBlockId(0)
    7092 {
    7093 }
    7094 
    7095 VmaBlockVector::~VmaBlockVector()
    7096 {
    7097  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
    7098 
    7099  for(size_t i = m_Blocks.size(); i--; )
    7100  {
    7101  m_Blocks[i]->Destroy(m_hAllocator);
    7102  vma_delete(m_hAllocator, m_Blocks[i]);
    7103  }
    7104 }
    7105 
    7106 VkResult VmaBlockVector::CreateMinBlocks()
    7107 {
    7108  for(size_t i = 0; i < m_MinBlockCount; ++i)
    7109  {
    7110  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
    7111  if(res != VK_SUCCESS)
    7112  {
    7113  return res;
    7114  }
    7115  }
    7116  return VK_SUCCESS;
    7117 }
    7118 
    7119 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
    7120 {
    7121  pStats->size = 0;
    7122  pStats->unusedSize = 0;
    7123  pStats->allocationCount = 0;
    7124  pStats->unusedRangeCount = 0;
    7125  pStats->unusedRangeSizeMax = 0;
    7126 
    7127  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7128 
    7129  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7130  {
    7131  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7132  VMA_ASSERT(pBlock);
    7133  VMA_HEAVY_ASSERT(pBlock->Validate());
    7134  pBlock->m_Metadata.AddPoolStats(*pStats);
    7135  }
    7136 }
    7137 
    7138 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
    7139 {
    7140  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    7141  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
    7142  (VMA_DEBUG_MARGIN > 0) &&
    7143  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
    7144 }
    7145 
    7146 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
    7147 
    7148 VkResult VmaBlockVector::Allocate(
    7149  VmaPool hCurrentPool,
    7150  uint32_t currentFrameIndex,
    7151  VkDeviceSize size,
    7152  VkDeviceSize alignment,
    7153  const VmaAllocationCreateInfo& createInfo,
    7154  VmaSuballocationType suballocType,
    7155  VmaAllocation* pAllocation)
    7156 {
    7157  // Early reject: requested allocation size is larger that maximum block size for this block vector.
    7158  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
    7159  {
    7160  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7161  }
    7162 
    7163  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    7164  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
    7165 
    7166  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7167 
    7168  // 1. Search existing allocations. Try to allocate without making other allocations lost.
    7169  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7170  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7171  {
    7172  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7173  VMA_ASSERT(pCurrBlock);
    7174  VmaAllocationRequest currRequest = {};
    7175  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7176  currentFrameIndex,
    7177  m_FrameInUseCount,
    7178  m_BufferImageGranularity,
    7179  size,
    7180  alignment,
    7181  suballocType,
    7182  false, // canMakeOtherLost
    7183  &currRequest))
    7184  {
    7185  // Allocate from pCurrBlock.
    7186  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
    7187 
    7188  if(mapped)
    7189  {
    7190  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
    7191  if(res != VK_SUCCESS)
    7192  {
    7193  return res;
    7194  }
    7195  }
    7196 
    7197  // We no longer have an empty Allocation.
    7198  if(pCurrBlock->m_Metadata.IsEmpty())
    7199  {
    7200  m_HasEmptyBlock = false;
    7201  }
    7202 
    7203  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7204  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation);
    7205  (*pAllocation)->InitBlockAllocation(
    7206  hCurrentPool,
    7207  pCurrBlock,
    7208  currRequest.offset,
    7209  alignment,
    7210  size,
    7211  suballocType,
    7212  mapped,
    7213  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7214  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
    7215  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7216  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7217  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7218  {
    7219  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7220  }
    7221  if(IsCorruptionDetectionEnabled())
    7222  {
    7223  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
    7224  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7225  }
    7226  return VK_SUCCESS;
    7227  }
    7228  }
    7229 
    7230  const bool canCreateNewBlock =
    7231  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
    7232  (m_Blocks.size() < m_MaxBlockCount);
    7233 
    7234  // 2. Try to create new block.
    7235  if(canCreateNewBlock)
    7236  {
    7237  // Calculate optimal size for new block.
    7238  VkDeviceSize newBlockSize = m_PreferredBlockSize;
    7239  uint32_t newBlockSizeShift = 0;
    7240  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
    7241 
    7242  // Allocating blocks of other sizes is allowed only in default pools.
    7243  // In custom pools block size is fixed.
    7244  if(m_IsCustomPool == false)
    7245  {
    7246  // Allocate 1/8, 1/4, 1/2 as first blocks.
    7247  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
    7248  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
    7249  {
    7250  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7251  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
    7252  {
    7253  newBlockSize = smallerNewBlockSize;
    7254  ++newBlockSizeShift;
    7255  }
    7256  else
    7257  {
    7258  break;
    7259  }
    7260  }
    7261  }
    7262 
    7263  size_t newBlockIndex = 0;
    7264  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
    7265  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
    7266  if(m_IsCustomPool == false)
    7267  {
    7268  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
    7269  {
    7270  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7271  if(smallerNewBlockSize >= size)
    7272  {
    7273  newBlockSize = smallerNewBlockSize;
    7274  ++newBlockSizeShift;
    7275  res = CreateBlock(newBlockSize, &newBlockIndex);
    7276  }
    7277  else
    7278  {
    7279  break;
    7280  }
    7281  }
    7282  }
    7283 
    7284  if(res == VK_SUCCESS)
    7285  {
    7286  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
    7287  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size);
    7288 
    7289  if(mapped)
    7290  {
    7291  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
    7292  if(res != VK_SUCCESS)
    7293  {
    7294  return res;
    7295  }
    7296  }
    7297 
    7298  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
    7299  VmaAllocationRequest allocRequest;
    7300  if(pBlock->m_Metadata.CreateAllocationRequest(
    7301  currentFrameIndex,
    7302  m_FrameInUseCount,
    7303  m_BufferImageGranularity,
    7304  size,
    7305  alignment,
    7306  suballocType,
    7307  false, // canMakeOtherLost
    7308  &allocRequest))
    7309  {
    7310  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7311  pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation);
    7312  (*pAllocation)->InitBlockAllocation(
    7313  hCurrentPool,
    7314  pBlock,
    7315  allocRequest.offset,
    7316  alignment,
    7317  size,
    7318  suballocType,
    7319  mapped,
    7320  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7321  VMA_HEAVY_ASSERT(pBlock->Validate());
    7322  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
    7323  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7324  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7325  {
    7326  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7327  }
    7328  if(IsCorruptionDetectionEnabled())
    7329  {
    7330  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
    7331  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7332  }
    7333  return VK_SUCCESS;
    7334  }
    7335  else
    7336  {
    7337  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
    7338  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7339  }
    7340  }
    7341  }
    7342 
    7343  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
    7344 
    7345  // 3. Try to allocate from existing blocks with making other allocations lost.
    7346  if(canMakeOtherLost)
    7347  {
    7348  uint32_t tryIndex = 0;
    7349  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
    7350  {
    7351  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
    7352  VmaAllocationRequest bestRequest = {};
    7353  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
    7354 
    7355  // 1. Search existing allocations.
    7356  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7357  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7358  {
    7359  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7360  VMA_ASSERT(pCurrBlock);
    7361  VmaAllocationRequest currRequest = {};
    7362  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7363  currentFrameIndex,
    7364  m_FrameInUseCount,
    7365  m_BufferImageGranularity,
    7366  size,
    7367  alignment,
    7368  suballocType,
    7369  canMakeOtherLost,
    7370  &currRequest))
    7371  {
    7372  const VkDeviceSize currRequestCost = currRequest.CalcCost();
    7373  if(pBestRequestBlock == VMA_NULL ||
    7374  currRequestCost < bestRequestCost)
    7375  {
    7376  pBestRequestBlock = pCurrBlock;
    7377  bestRequest = currRequest;
    7378  bestRequestCost = currRequestCost;
    7379 
    7380  if(bestRequestCost == 0)
    7381  {
    7382  break;
    7383  }
    7384  }
    7385  }
    7386  }
    7387 
    7388  if(pBestRequestBlock != VMA_NULL)
    7389  {
    7390  if(mapped)
    7391  {
    7392  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
    7393  if(res != VK_SUCCESS)
    7394  {
    7395  return res;
    7396  }
    7397  }
    7398 
    7399  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
    7400  currentFrameIndex,
    7401  m_FrameInUseCount,
    7402  &bestRequest))
    7403  {
    7404  // We no longer have an empty Allocation.
    7405  if(pBestRequestBlock->m_Metadata.IsEmpty())
    7406  {
    7407  m_HasEmptyBlock = false;
    7408  }
    7409  // Allocate from this pBlock.
    7410  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7411  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation);
    7412  (*pAllocation)->InitBlockAllocation(
    7413  hCurrentPool,
    7414  pBestRequestBlock,
    7415  bestRequest.offset,
    7416  alignment,
    7417  size,
    7418  suballocType,
    7419  mapped,
    7420  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7421  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
    7422  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7423  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7424  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7425  {
    7426  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7427  }
    7428  if(IsCorruptionDetectionEnabled())
    7429  {
    7430  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
    7431  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7432  }
    7433  return VK_SUCCESS;
    7434  }
    7435  // else: Some allocations must have been touched while we are here. Next try.
    7436  }
    7437  else
    7438  {
    7439  // Could not find place in any of the blocks - break outer loop.
    7440  break;
    7441  }
    7442  }
    7443  /* Maximum number of tries exceeded - a very unlike event when many other
    7444  threads are simultaneously touching allocations making it impossible to make
    7445  lost at the same time as we try to allocate. */
    7446  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
    7447  {
    7448  return VK_ERROR_TOO_MANY_OBJECTS;
    7449  }
    7450  }
    7451 
    7452  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7453 }
    7454 
    7455 void VmaBlockVector::Free(
    7456  VmaAllocation hAllocation)
    7457 {
    7458  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
    7459 
    7460  // Scope for lock.
    7461  {
    7462  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7463 
    7464  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    7465 
    7466  if(IsCorruptionDetectionEnabled())
    7467  {
    7468  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
    7469  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
    7470  }
    7471 
    7472  if(hAllocation->IsPersistentMap())
    7473  {
    7474  pBlock->Unmap(m_hAllocator, 1);
    7475  }
    7476 
    7477  pBlock->m_Metadata.Free(hAllocation);
    7478  VMA_HEAVY_ASSERT(pBlock->Validate());
    7479 
    7480  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
    7481 
    7482  // pBlock became empty after this deallocation.
    7483  if(pBlock->m_Metadata.IsEmpty())
    7484  {
    7485  // Already has empty Allocation. We don't want to have two, so delete this one.
    7486  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
    7487  {
    7488  pBlockToDelete = pBlock;
    7489  Remove(pBlock);
    7490  }
    7491  // We now have first empty block.
    7492  else
    7493  {
    7494  m_HasEmptyBlock = true;
    7495  }
    7496  }
    7497  // pBlock didn't become empty, but we have another empty block - find and free that one.
    7498  // (This is optional, heuristics.)
    7499  else if(m_HasEmptyBlock)
    7500  {
    7501  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
    7502  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
    7503  {
    7504  pBlockToDelete = pLastBlock;
    7505  m_Blocks.pop_back();
    7506  m_HasEmptyBlock = false;
    7507  }
    7508  }
    7509 
    7510  IncrementallySortBlocks();
    7511  }
    7512 
    7513  // Destruction of a free Allocation. Deferred until this point, outside of mutex
    7514  // lock, for performance reason.
    7515  if(pBlockToDelete != VMA_NULL)
    7516  {
    7517  VMA_DEBUG_LOG(" Deleted empty allocation");
    7518  pBlockToDelete->Destroy(m_hAllocator);
    7519  vma_delete(m_hAllocator, pBlockToDelete);
    7520  }
    7521 }
    7522 
    7523 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
    7524 {
    7525  VkDeviceSize result = 0;
    7526  for(size_t i = m_Blocks.size(); i--; )
    7527  {
    7528  result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize());
    7529  if(result >= m_PreferredBlockSize)
    7530  {
    7531  break;
    7532  }
    7533  }
    7534  return result;
    7535 }
    7536 
    7537 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
    7538 {
    7539  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7540  {
    7541  if(m_Blocks[blockIndex] == pBlock)
    7542  {
    7543  VmaVectorRemove(m_Blocks, blockIndex);
    7544  return;
    7545  }
    7546  }
    7547  VMA_ASSERT(0);
    7548 }
    7549 
    7550 void VmaBlockVector::IncrementallySortBlocks()
    7551 {
    7552  // Bubble sort only until first swap.
    7553  for(size_t i = 1; i < m_Blocks.size(); ++i)
    7554  {
    7555  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
    7556  {
    7557  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
    7558  return;
    7559  }
    7560  }
    7561 }
    7562 
    7563 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
    7564 {
    7565  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    7566  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
    7567  allocInfo.allocationSize = blockSize;
    7568  VkDeviceMemory mem = VK_NULL_HANDLE;
    7569  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
    7570  if(res < 0)
    7571  {
    7572  return res;
    7573  }
    7574 
    7575  // New VkDeviceMemory successfully created.
    7576 
    7577  // Create new Allocation for it.
    7578  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
    7579  pBlock->Init(
    7580  m_MemoryTypeIndex,
    7581  mem,
    7582  allocInfo.allocationSize,
    7583  m_NextBlockId++);
    7584 
    7585  m_Blocks.push_back(pBlock);
    7586  if(pNewBlockIndex != VMA_NULL)
    7587  {
    7588  *pNewBlockIndex = m_Blocks.size() - 1;
    7589  }
    7590 
    7591  return VK_SUCCESS;
    7592 }
    7593 
    7594 #if VMA_STATS_STRING_ENABLED
    7595 
    7596 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
    7597 {
    7598  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7599 
    7600  json.BeginObject();
    7601 
    7602  if(m_IsCustomPool)
    7603  {
    7604  json.WriteString("MemoryTypeIndex");
    7605  json.WriteNumber(m_MemoryTypeIndex);
    7606 
    7607  json.WriteString("BlockSize");
    7608  json.WriteNumber(m_PreferredBlockSize);
    7609 
    7610  json.WriteString("BlockCount");
    7611  json.BeginObject(true);
    7612  if(m_MinBlockCount > 0)
    7613  {
    7614  json.WriteString("Min");
    7615  json.WriteNumber((uint64_t)m_MinBlockCount);
    7616  }
    7617  if(m_MaxBlockCount < SIZE_MAX)
    7618  {
    7619  json.WriteString("Max");
    7620  json.WriteNumber((uint64_t)m_MaxBlockCount);
    7621  }
    7622  json.WriteString("Cur");
    7623  json.WriteNumber((uint64_t)m_Blocks.size());
    7624  json.EndObject();
    7625 
    7626  if(m_FrameInUseCount > 0)
    7627  {
    7628  json.WriteString("FrameInUseCount");
    7629  json.WriteNumber(m_FrameInUseCount);
    7630  }
    7631  }
    7632  else
    7633  {
    7634  json.WriteString("PreferredBlockSize");
    7635  json.WriteNumber(m_PreferredBlockSize);
    7636  }
    7637 
    7638  json.WriteString("Blocks");
    7639  json.BeginObject();
    7640  for(size_t i = 0; i < m_Blocks.size(); ++i)
    7641  {
    7642  json.BeginString();
    7643  json.ContinueString(m_Blocks[i]->GetId());
    7644  json.EndString();
    7645 
    7646  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
    7647  }
    7648  json.EndObject();
    7649 
    7650  json.EndObject();
    7651 }
    7652 
    7653 #endif // #if VMA_STATS_STRING_ENABLED
    7654 
    7655 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
    7656  VmaAllocator hAllocator,
    7657  uint32_t currentFrameIndex)
    7658 {
    7659  if(m_pDefragmentator == VMA_NULL)
    7660  {
    7661  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
    7662  hAllocator,
    7663  this,
    7664  currentFrameIndex);
    7665  }
    7666 
    7667  return m_pDefragmentator;
    7668 }
    7669 
    7670 VkResult VmaBlockVector::Defragment(
    7671  VmaDefragmentationStats* pDefragmentationStats,
    7672  VkDeviceSize& maxBytesToMove,
    7673  uint32_t& maxAllocationsToMove)
    7674 {
    7675  if(m_pDefragmentator == VMA_NULL)
    7676  {
    7677  return VK_SUCCESS;
    7678  }
    7679 
    7680  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7681 
    7682  // Defragment.
    7683  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
    7684 
    7685  // Accumulate statistics.
    7686  if(pDefragmentationStats != VMA_NULL)
    7687  {
    7688  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
    7689  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
    7690  pDefragmentationStats->bytesMoved += bytesMoved;
    7691  pDefragmentationStats->allocationsMoved += allocationsMoved;
    7692  VMA_ASSERT(bytesMoved <= maxBytesToMove);
    7693  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
    7694  maxBytesToMove -= bytesMoved;
    7695  maxAllocationsToMove -= allocationsMoved;
    7696  }
    7697 
    7698  // Free empty blocks.
    7699  m_HasEmptyBlock = false;
    7700  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
    7701  {
    7702  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
    7703  if(pBlock->m_Metadata.IsEmpty())
    7704  {
    7705  if(m_Blocks.size() > m_MinBlockCount)
    7706  {
    7707  if(pDefragmentationStats != VMA_NULL)
    7708  {
    7709  ++pDefragmentationStats->deviceMemoryBlocksFreed;
    7710  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
    7711  }
    7712 
    7713  VmaVectorRemove(m_Blocks, blockIndex);
    7714  pBlock->Destroy(m_hAllocator);
    7715  vma_delete(m_hAllocator, pBlock);
    7716  }
    7717  else
    7718  {
    7719  m_HasEmptyBlock = true;
    7720  }
    7721  }
    7722  }
    7723 
    7724  return result;
    7725 }
    7726 
    7727 void VmaBlockVector::DestroyDefragmentator()
    7728 {
    7729  if(m_pDefragmentator != VMA_NULL)
    7730  {
    7731  vma_delete(m_hAllocator, m_pDefragmentator);
    7732  m_pDefragmentator = VMA_NULL;
    7733  }
    7734 }
    7735 
    7736 void VmaBlockVector::MakePoolAllocationsLost(
    7737  uint32_t currentFrameIndex,
    7738  size_t* pLostAllocationCount)
    7739 {
    7740  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7741  size_t lostAllocationCount = 0;
    7742  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7743  {
    7744  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7745  VMA_ASSERT(pBlock);
    7746  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
    7747  }
    7748  if(pLostAllocationCount != VMA_NULL)
    7749  {
    7750  *pLostAllocationCount = lostAllocationCount;
    7751  }
    7752 }
    7753 
    7754 VkResult VmaBlockVector::CheckCorruption()
    7755 {
    7756  if(!IsCorruptionDetectionEnabled())
    7757  {
    7758  return VK_ERROR_FEATURE_NOT_PRESENT;
    7759  }
    7760 
    7761  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7762  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7763  {
    7764  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7765  VMA_ASSERT(pBlock);
    7766  VkResult res = pBlock->CheckCorruption(m_hAllocator);
    7767  if(res != VK_SUCCESS)
    7768  {
    7769  return res;
    7770  }
    7771  }
    7772  return VK_SUCCESS;
    7773 }
    7774 
    7775 void VmaBlockVector::AddStats(VmaStats* pStats)
    7776 {
    7777  const uint32_t memTypeIndex = m_MemoryTypeIndex;
    7778  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
    7779 
    7780  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7781 
    7782  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7783  {
    7784  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7785  VMA_ASSERT(pBlock);
    7786  VMA_HEAVY_ASSERT(pBlock->Validate());
    7787  VmaStatInfo allocationStatInfo;
    7788  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
    7789  VmaAddStatInfo(pStats->total, allocationStatInfo);
    7790  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    7791  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    7792  }
    7793 }
    7794 
    7796 // VmaDefragmentator members definition
    7797 
    7798 VmaDefragmentator::VmaDefragmentator(
    7799  VmaAllocator hAllocator,
    7800  VmaBlockVector* pBlockVector,
    7801  uint32_t currentFrameIndex) :
    7802  m_hAllocator(hAllocator),
    7803  m_pBlockVector(pBlockVector),
    7804  m_CurrentFrameIndex(currentFrameIndex),
    7805  m_BytesMoved(0),
    7806  m_AllocationsMoved(0),
    7807  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
    7808  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
    7809 {
    7810 }
    7811 
    7812 VmaDefragmentator::~VmaDefragmentator()
    7813 {
    7814  for(size_t i = m_Blocks.size(); i--; )
    7815  {
    7816  vma_delete(m_hAllocator, m_Blocks[i]);
    7817  }
    7818 }
    7819 
    7820 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
    7821 {
    7822  AllocationInfo allocInfo;
    7823  allocInfo.m_hAllocation = hAlloc;
    7824  allocInfo.m_pChanged = pChanged;
    7825  m_Allocations.push_back(allocInfo);
    7826 }
    7827 
    7828 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
    7829 {
    7830  // It has already been mapped for defragmentation.
    7831  if(m_pMappedDataForDefragmentation)
    7832  {
    7833  *ppMappedData = m_pMappedDataForDefragmentation;
    7834  return VK_SUCCESS;
    7835  }
    7836 
    7837  // It is originally mapped.
    7838  if(m_pBlock->GetMappedData())
    7839  {
    7840  *ppMappedData = m_pBlock->GetMappedData();
    7841  return VK_SUCCESS;
    7842  }
    7843 
    7844  // Map on first usage.
    7845  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
    7846  *ppMappedData = m_pMappedDataForDefragmentation;
    7847  return res;
    7848 }
    7849 
    7850 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
    7851 {
    7852  if(m_pMappedDataForDefragmentation != VMA_NULL)
    7853  {
    7854  m_pBlock->Unmap(hAllocator, 1);
    7855  }
    7856 }
    7857 
    7858 VkResult VmaDefragmentator::DefragmentRound(
    7859  VkDeviceSize maxBytesToMove,
    7860  uint32_t maxAllocationsToMove)
    7861 {
    7862  if(m_Blocks.empty())
    7863  {
    7864  return VK_SUCCESS;
    7865  }
    7866 
    7867  size_t srcBlockIndex = m_Blocks.size() - 1;
    7868  size_t srcAllocIndex = SIZE_MAX;
    7869  for(;;)
    7870  {
    7871  // 1. Find next allocation to move.
    7872  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
    7873  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
    7874  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
    7875  {
    7876  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
    7877  {
    7878  // Finished: no more allocations to process.
    7879  if(srcBlockIndex == 0)
    7880  {
    7881  return VK_SUCCESS;
    7882  }
    7883  else
    7884  {
    7885  --srcBlockIndex;
    7886  srcAllocIndex = SIZE_MAX;
    7887  }
    7888  }
    7889  else
    7890  {
    7891  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
    7892  }
    7893  }
    7894 
    7895  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
    7896  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
    7897 
    7898  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
    7899  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
    7900  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
    7901  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
    7902 
    7903  // 2. Try to find new place for this allocation in preceding or current block.
    7904  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
    7905  {
    7906  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
    7907  VmaAllocationRequest dstAllocRequest;
    7908  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
    7909  m_CurrentFrameIndex,
    7910  m_pBlockVector->GetFrameInUseCount(),
    7911  m_pBlockVector->GetBufferImageGranularity(),
    7912  size,
    7913  alignment,
    7914  suballocType,
    7915  false, // canMakeOtherLost
    7916  &dstAllocRequest) &&
    7917  MoveMakesSense(
    7918  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
    7919  {
    7920  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
    7921 
    7922  // Reached limit on number of allocations or bytes to move.
    7923  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
    7924  (m_BytesMoved + size > maxBytesToMove))
    7925  {
    7926  return VK_INCOMPLETE;
    7927  }
    7928 
    7929  void* pDstMappedData = VMA_NULL;
    7930  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
    7931  if(res != VK_SUCCESS)
    7932  {
    7933  return res;
    7934  }
    7935 
    7936  void* pSrcMappedData = VMA_NULL;
    7937  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
    7938  if(res != VK_SUCCESS)
    7939  {
    7940  return res;
    7941  }
    7942 
    7943  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
    7944  memcpy(
    7945  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
    7946  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
    7947  static_cast<size_t>(size));
    7948 
    7949  if(VMA_DEBUG_MARGIN > 0)
    7950  {
    7951  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
    7952  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
    7953  }
    7954 
    7955  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
    7956  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
    7957 
    7958  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
    7959 
    7960  if(allocInfo.m_pChanged != VMA_NULL)
    7961  {
    7962  *allocInfo.m_pChanged = VK_TRUE;
    7963  }
    7964 
    7965  ++m_AllocationsMoved;
    7966  m_BytesMoved += size;
    7967 
    7968  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
    7969 
    7970  break;
    7971  }
    7972  }
    7973 
    7974  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
    7975 
    7976  if(srcAllocIndex > 0)
    7977  {
    7978  --srcAllocIndex;
    7979  }
    7980  else
    7981  {
    7982  if(srcBlockIndex > 0)
    7983  {
    7984  --srcBlockIndex;
    7985  srcAllocIndex = SIZE_MAX;
    7986  }
    7987  else
    7988  {
    7989  return VK_SUCCESS;
    7990  }
    7991  }
    7992  }
    7993 }
    7994 
    7995 VkResult VmaDefragmentator::Defragment(
    7996  VkDeviceSize maxBytesToMove,
    7997  uint32_t maxAllocationsToMove)
    7998 {
    7999  if(m_Allocations.empty())
    8000  {
    8001  return VK_SUCCESS;
    8002  }
    8003 
    8004  // Create block info for each block.
    8005  const size_t blockCount = m_pBlockVector->m_Blocks.size();
    8006  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8007  {
    8008  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
    8009  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
    8010  m_Blocks.push_back(pBlockInfo);
    8011  }
    8012 
    8013  // Sort them by m_pBlock pointer value.
    8014  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
    8015 
    8016  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
    8017  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
    8018  {
    8019  AllocationInfo& allocInfo = m_Allocations[blockIndex];
    8020  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
    8021  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    8022  {
    8023  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
    8024  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
    8025  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
    8026  {
    8027  (*it)->m_Allocations.push_back(allocInfo);
    8028  }
    8029  else
    8030  {
    8031  VMA_ASSERT(0);
    8032  }
    8033  }
    8034  }
    8035  m_Allocations.clear();
    8036 
    8037  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8038  {
    8039  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
    8040  pBlockInfo->CalcHasNonMovableAllocations();
    8041  pBlockInfo->SortAllocationsBySizeDescecnding();
    8042  }
    8043 
    8044  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
    8045  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
    8046 
    8047  // Execute defragmentation rounds (the main part).
    8048  VkResult result = VK_SUCCESS;
    8049  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
    8050  {
    8051  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
    8052  }
    8053 
    8054  // Unmap blocks that were mapped for defragmentation.
    8055  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8056  {
    8057  m_Blocks[blockIndex]->Unmap(m_hAllocator);
    8058  }
    8059 
    8060  return result;
    8061 }
    8062 
    8063 bool VmaDefragmentator::MoveMakesSense(
    8064  size_t dstBlockIndex, VkDeviceSize dstOffset,
    8065  size_t srcBlockIndex, VkDeviceSize srcOffset)
    8066 {
    8067  if(dstBlockIndex < srcBlockIndex)
    8068  {
    8069  return true;
    8070  }
    8071  if(dstBlockIndex > srcBlockIndex)
    8072  {
    8073  return false;
    8074  }
    8075  if(dstOffset < srcOffset)
    8076  {
    8077  return true;
    8078  }
    8079  return false;
    8080 }
    8081 
    8083 // VmaRecorder
    8084 
    8085 #if VMA_RECORDING_ENABLED
    8086 
    8087 VmaRecorder::VmaRecorder() :
    8088  m_UseMutex(true),
    8089  m_Flags(0),
    8090  m_File(VMA_NULL),
    8091  m_Freq(INT64_MAX),
    8092  m_StartCounter(INT64_MAX)
    8093 {
    8094 }
    8095 
    8096 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
    8097 {
    8098  m_UseMutex = useMutex;
    8099  m_Flags = settings.flags;
    8100 
    8101  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
    8102  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
    8103 
    8104  // Open file for writing.
    8105  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
    8106  if(err != 0)
    8107  {
    8108  return VK_ERROR_INITIALIZATION_FAILED;
    8109  }
    8110 
    8111  // Write header.
    8112  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
    8113  fprintf(m_File, "%s\n", "1,2");
    8114 
    8115  return VK_SUCCESS;
    8116 }
    8117 
    8118 VmaRecorder::~VmaRecorder()
    8119 {
    8120  if(m_File != VMA_NULL)
    8121  {
    8122  fclose(m_File);
    8123  }
    8124 }
    8125 
    8126 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
    8127 {
    8128  CallParams callParams;
    8129  GetBasicParams(callParams);
    8130 
    8131  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8132  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8133  Flush();
    8134 }
    8135 
    8136 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
    8137 {
    8138  CallParams callParams;
    8139  GetBasicParams(callParams);
    8140 
    8141  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8142  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8143  Flush();
    8144 }
    8145 
    8146 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
    8147 {
    8148  CallParams callParams;
    8149  GetBasicParams(callParams);
    8150 
    8151  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8152  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
    8153  createInfo.memoryTypeIndex,
    8154  createInfo.flags,
    8155  createInfo.blockSize,
    8156  createInfo.minBlockCount,
    8157  createInfo.maxBlockCount,
    8158  createInfo.frameInUseCount,
    8159  pool);
    8160  Flush();
    8161 }
    8162 
    8163 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
    8164 {
    8165  CallParams callParams;
    8166  GetBasicParams(callParams);
    8167 
    8168  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8169  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
    8170  pool);
    8171  Flush();
    8172 }
    8173 
    8174 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
    8175  const VkMemoryRequirements& vkMemReq,
    8176  const VmaAllocationCreateInfo& createInfo,
    8177  VmaAllocation allocation)
    8178 {
    8179  CallParams callParams;
    8180  GetBasicParams(callParams);
    8181 
    8182  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8183  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8184  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8185  vkMemReq.size,
    8186  vkMemReq.alignment,
    8187  vkMemReq.memoryTypeBits,
    8188  createInfo.flags,
    8189  createInfo.usage,
    8190  createInfo.requiredFlags,
    8191  createInfo.preferredFlags,
    8192  createInfo.memoryTypeBits,
    8193  createInfo.pool,
    8194  allocation,
    8195  userDataStr.GetString());
    8196  Flush();
    8197 }
    8198 
    8199 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    8200  const VkMemoryRequirements& vkMemReq,
    8201  bool requiresDedicatedAllocation,
    8202  bool prefersDedicatedAllocation,
    8203  const VmaAllocationCreateInfo& createInfo,
    8204  VmaAllocation allocation)
    8205 {
    8206  CallParams callParams;
    8207  GetBasicParams(callParams);
    8208 
    8209  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8210  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8211  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8212  vkMemReq.size,
    8213  vkMemReq.alignment,
    8214  vkMemReq.memoryTypeBits,
    8215  requiresDedicatedAllocation ? 1 : 0,
    8216  prefersDedicatedAllocation ? 1 : 0,
    8217  createInfo.flags,
    8218  createInfo.usage,
    8219  createInfo.requiredFlags,
    8220  createInfo.preferredFlags,
    8221  createInfo.memoryTypeBits,
    8222  createInfo.pool,
    8223  allocation,
    8224  userDataStr.GetString());
    8225  Flush();
    8226 }
    8227 
    8228 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
    8229  const VkMemoryRequirements& vkMemReq,
    8230  bool requiresDedicatedAllocation,
    8231  bool prefersDedicatedAllocation,
    8232  const VmaAllocationCreateInfo& createInfo,
    8233  VmaAllocation allocation)
    8234 {
    8235  CallParams callParams;
    8236  GetBasicParams(callParams);
    8237 
    8238  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8239  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8240  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8241  vkMemReq.size,
    8242  vkMemReq.alignment,
    8243  vkMemReq.memoryTypeBits,
    8244  requiresDedicatedAllocation ? 1 : 0,
    8245  prefersDedicatedAllocation ? 1 : 0,
    8246  createInfo.flags,
    8247  createInfo.usage,
    8248  createInfo.requiredFlags,
    8249  createInfo.preferredFlags,
    8250  createInfo.memoryTypeBits,
    8251  createInfo.pool,
    8252  allocation,
    8253  userDataStr.GetString());
    8254  Flush();
    8255 }
    8256 
    8257 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
    8258  VmaAllocation allocation)
    8259 {
    8260  CallParams callParams;
    8261  GetBasicParams(callParams);
    8262 
    8263  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8264  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8265  allocation);
    8266  Flush();
    8267 }
    8268 
    8269 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
    8270  VmaAllocation allocation,
    8271  const void* pUserData)
    8272 {
    8273  CallParams callParams;
    8274  GetBasicParams(callParams);
    8275 
    8276  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8277  UserDataString userDataStr(
    8278  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
    8279  pUserData);
    8280  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8281  allocation,
    8282  userDataStr.GetString());
    8283  Flush();
    8284 }
    8285 
    8286 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
    8287  VmaAllocation allocation)
    8288 {
    8289  CallParams callParams;
    8290  GetBasicParams(callParams);
    8291 
    8292  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8293  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
    8294  allocation);
    8295  Flush();
    8296 }
    8297 
    8298 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
    8299  VmaAllocation allocation)
    8300 {
    8301  CallParams callParams;
    8302  GetBasicParams(callParams);
    8303 
    8304  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8305  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8306  allocation);
    8307  Flush();
    8308 }
    8309 
    8310 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
    8311  VmaAllocation allocation)
    8312 {
    8313  CallParams callParams;
    8314  GetBasicParams(callParams);
    8315 
    8316  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8317  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8318  allocation);
    8319  Flush();
    8320 }
    8321 
    8322 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
    8323  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8324 {
    8325  CallParams callParams;
    8326  GetBasicParams(callParams);
    8327 
    8328  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8329  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8330  allocation,
    8331  offset,
    8332  size);
    8333  Flush();
    8334 }
    8335 
    8336 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
    8337  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8338 {
    8339  CallParams callParams;
    8340  GetBasicParams(callParams);
    8341 
    8342  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8343  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8344  allocation,
    8345  offset,
    8346  size);
    8347  Flush();
    8348 }
    8349 
    8350 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
    8351  const VkBufferCreateInfo& bufCreateInfo,
    8352  const VmaAllocationCreateInfo& allocCreateInfo,
    8353  VmaAllocation allocation)
    8354 {
    8355  CallParams callParams;
    8356  GetBasicParams(callParams);
    8357 
    8358  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8359  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8360  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8361  bufCreateInfo.flags,
    8362  bufCreateInfo.size,
    8363  bufCreateInfo.usage,
    8364  bufCreateInfo.sharingMode,
    8365  allocCreateInfo.flags,
    8366  allocCreateInfo.usage,
    8367  allocCreateInfo.requiredFlags,
    8368  allocCreateInfo.preferredFlags,
    8369  allocCreateInfo.memoryTypeBits,
    8370  allocCreateInfo.pool,
    8371  allocation,
    8372  userDataStr.GetString());
    8373  Flush();
    8374 }
    8375 
    8376 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
    8377  const VkImageCreateInfo& imageCreateInfo,
    8378  const VmaAllocationCreateInfo& allocCreateInfo,
    8379  VmaAllocation allocation)
    8380 {
    8381  CallParams callParams;
    8382  GetBasicParams(callParams);
    8383 
    8384  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8385  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8386  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8387  imageCreateInfo.flags,
    8388  imageCreateInfo.imageType,
    8389  imageCreateInfo.format,
    8390  imageCreateInfo.extent.width,
    8391  imageCreateInfo.extent.height,
    8392  imageCreateInfo.extent.depth,
    8393  imageCreateInfo.mipLevels,
    8394  imageCreateInfo.arrayLayers,
    8395  imageCreateInfo.samples,
    8396  imageCreateInfo.tiling,
    8397  imageCreateInfo.usage,
    8398  imageCreateInfo.sharingMode,
    8399  imageCreateInfo.initialLayout,
    8400  allocCreateInfo.flags,
    8401  allocCreateInfo.usage,
    8402  allocCreateInfo.requiredFlags,
    8403  allocCreateInfo.preferredFlags,
    8404  allocCreateInfo.memoryTypeBits,
    8405  allocCreateInfo.pool,
    8406  allocation,
    8407  userDataStr.GetString());
    8408  Flush();
    8409 }
    8410 
    8411 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
    8412  VmaAllocation allocation)
    8413 {
    8414  CallParams callParams;
    8415  GetBasicParams(callParams);
    8416 
    8417  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8418  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
    8419  allocation);
    8420  Flush();
    8421 }
    8422 
    8423 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
    8424  VmaAllocation allocation)
    8425 {
    8426  CallParams callParams;
    8427  GetBasicParams(callParams);
    8428 
    8429  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8430  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
    8431  allocation);
    8432  Flush();
    8433 }
    8434 
    8435 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
    8436 {
    8437  if(pUserData != VMA_NULL)
    8438  {
    8439  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
    8440  {
    8441  m_Str = (const char*)pUserData;
    8442  }
    8443  else
    8444  {
    8445  sprintf_s(m_PtrStr, "%p", pUserData);
    8446  m_Str = m_PtrStr;
    8447  }
    8448  }
    8449  else
    8450  {
    8451  m_Str = "";
    8452  }
    8453 }
    8454 
    8455 void VmaRecorder::GetBasicParams(CallParams& outParams)
    8456 {
    8457  outParams.threadId = GetCurrentThreadId();
    8458 
    8459  LARGE_INTEGER counter;
    8460  QueryPerformanceCounter(&counter);
    8461  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
    8462 }
    8463 
    8464 void VmaRecorder::Flush()
    8465 {
    8466  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
    8467  {
    8468  fflush(m_File);
    8469  }
    8470 }
    8471 
    8472 #endif // #if VMA_RECORDING_ENABLED
    8473 
    8475 // VmaAllocator_T
    8476 
    8477 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
    8478  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
    8479  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
    8480  m_hDevice(pCreateInfo->device),
    8481  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
    8482  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
    8483  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
    8484  m_PreferredLargeHeapBlockSize(0),
    8485  m_PhysicalDevice(pCreateInfo->physicalDevice),
    8486  m_CurrentFrameIndex(0),
    8487  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
    8488  m_NextPoolId(0)
    8489 #if VMA_RECORDING_ENABLED
    8490  ,m_pRecorder(VMA_NULL)
    8491 #endif
    8492 {
    8493  if(VMA_DEBUG_DETECT_CORRUPTION)
    8494  {
    8495  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
    8496  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
    8497  }
    8498 
    8499  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
    8500 
    8501 #if !(VMA_DEDICATED_ALLOCATION)
    8502  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
    8503  {
    8504  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
    8505  }
    8506 #endif
    8507 
    8508  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
    8509  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
    8510  memset(&m_MemProps, 0, sizeof(m_MemProps));
    8511 
    8512  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
    8513  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
    8514 
    8515  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    8516  {
    8517  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
    8518  }
    8519 
    8520  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
    8521  {
    8522  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
    8523  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
    8524  }
    8525 
    8526  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
    8527 
    8528  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
    8529  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
    8530 
    8531  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
    8532  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
    8533 
    8534  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
    8535  {
    8536  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
    8537  {
    8538  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
    8539  if(limit != VK_WHOLE_SIZE)
    8540  {
    8541  m_HeapSizeLimit[heapIndex] = limit;
    8542  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
    8543  {
    8544  m_MemProps.memoryHeaps[heapIndex].size = limit;
    8545  }
    8546  }
    8547  }
    8548  }
    8549 
    8550  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    8551  {
    8552  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
    8553 
    8554  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
    8555  this,
    8556  memTypeIndex,
    8557  preferredBlockSize,
    8558  0,
    8559  SIZE_MAX,
    8560  GetBufferImageGranularity(),
    8561  pCreateInfo->frameInUseCount,
    8562  false); // isCustomPool
    8563  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
    8564  // becase minBlockCount is 0.
    8565  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
    8566 
    8567  }
    8568 }
    8569 
    8570 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
    8571 {
    8572  VkResult res = VK_SUCCESS;
    8573 
    8574  if(pCreateInfo->pRecordSettings != VMA_NULL &&
    8575  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
    8576  {
    8577 #if VMA_RECORDING_ENABLED
    8578  m_pRecorder = vma_new(this, VmaRecorder)();
    8579  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
    8580  if(res != VK_SUCCESS)
    8581  {
    8582  return res;
    8583  }
    8584  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
    8585 #else
    8586  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
    8587  return VK_ERROR_FEATURE_NOT_PRESENT;
    8588 #endif
    8589  }
    8590 
    8591  return res;
    8592 }
    8593 
    8594 VmaAllocator_T::~VmaAllocator_T()
    8595 {
    8596 #if VMA_RECORDING_ENABLED
    8597  if(m_pRecorder != VMA_NULL)
    8598  {
    8599  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
    8600  vma_delete(this, m_pRecorder);
    8601  }
    8602 #endif
    8603 
    8604  VMA_ASSERT(m_Pools.empty());
    8605 
    8606  for(size_t i = GetMemoryTypeCount(); i--; )
    8607  {
    8608  vma_delete(this, m_pDedicatedAllocations[i]);
    8609  vma_delete(this, m_pBlockVectors[i]);
    8610  }
    8611 }
    8612 
    8613 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
    8614 {
    8615 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8616  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
    8617  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
    8618  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    8619  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
    8620  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
    8621  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
    8622  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
    8623  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
    8624  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
    8625  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
    8626  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
    8627  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
    8628  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
    8629  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
    8630  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
    8631  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
    8632 #if VMA_DEDICATED_ALLOCATION
    8633  if(m_UseKhrDedicatedAllocation)
    8634  {
    8635  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
    8636  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
    8637  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
    8638  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
    8639  }
    8640 #endif // #if VMA_DEDICATED_ALLOCATION
    8641 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8642 
    8643 #define VMA_COPY_IF_NOT_NULL(funcName) \
    8644  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
    8645 
    8646  if(pVulkanFunctions != VMA_NULL)
    8647  {
    8648  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
    8649  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
    8650  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
    8651  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
    8652  VMA_COPY_IF_NOT_NULL(vkMapMemory);
    8653  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
    8654  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
    8655  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
    8656  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
    8657  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
    8658  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
    8659  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
    8660  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
    8661  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
    8662  VMA_COPY_IF_NOT_NULL(vkCreateImage);
    8663  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
    8664 #if VMA_DEDICATED_ALLOCATION
    8665  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
    8666  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
    8667 #endif
    8668  }
    8669 
    8670 #undef VMA_COPY_IF_NOT_NULL
    8671 
    8672  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
    8673  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
    8674  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
    8675  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
    8676  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
    8677  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
    8678  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
    8679  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
    8680  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
    8681  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
    8682  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
    8683  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
    8684  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
    8685  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
    8686  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
    8687  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
    8688  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
    8689  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
    8690 #if VMA_DEDICATED_ALLOCATION
    8691  if(m_UseKhrDedicatedAllocation)
    8692  {
    8693  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
    8694  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
    8695  }
    8696 #endif
    8697 }
    8698 
    8699 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
    8700 {
    8701  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    8702  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
    8703  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
    8704  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
    8705 }
    8706 
    8707 VkResult VmaAllocator_T::AllocateMemoryOfType(
    8708  VkDeviceSize size,
    8709  VkDeviceSize alignment,
    8710  bool dedicatedAllocation,
    8711  VkBuffer dedicatedBuffer,
    8712  VkImage dedicatedImage,
    8713  const VmaAllocationCreateInfo& createInfo,
    8714  uint32_t memTypeIndex,
    8715  VmaSuballocationType suballocType,
    8716  VmaAllocation* pAllocation)
    8717 {
    8718  VMA_ASSERT(pAllocation != VMA_NULL);
    8719  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
    8720 
    8721  VmaAllocationCreateInfo finalCreateInfo = createInfo;
    8722 
    8723  // If memory type is not HOST_VISIBLE, disable MAPPED.
    8724  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8725  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    8726  {
    8727  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
    8728  }
    8729 
    8730  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
    8731  VMA_ASSERT(blockVector);
    8732 
    8733  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
    8734  bool preferDedicatedMemory =
    8735  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
    8736  dedicatedAllocation ||
    8737  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
    8738  size > preferredBlockSize / 2;
    8739 
    8740  if(preferDedicatedMemory &&
    8741  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
    8742  finalCreateInfo.pool == VK_NULL_HANDLE)
    8743  {
    8744  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
    8745  }
    8746 
    8747  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
    8748  {
    8749  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8750  {
    8751  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8752  }
    8753  else
    8754  {
    8755  return AllocateDedicatedMemory(
    8756  size,
    8757  suballocType,
    8758  memTypeIndex,
    8759  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8760  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8761  finalCreateInfo.pUserData,
    8762  dedicatedBuffer,
    8763  dedicatedImage,
    8764  pAllocation);
    8765  }
    8766  }
    8767  else
    8768  {
    8769  VkResult res = blockVector->Allocate(
    8770  VK_NULL_HANDLE, // hCurrentPool
    8771  m_CurrentFrameIndex.load(),
    8772  size,
    8773  alignment,
    8774  finalCreateInfo,
    8775  suballocType,
    8776  pAllocation);
    8777  if(res == VK_SUCCESS)
    8778  {
    8779  return res;
    8780  }
    8781 
    8782  // 5. Try dedicated memory.
    8783  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8784  {
    8785  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8786  }
    8787  else
    8788  {
    8789  res = AllocateDedicatedMemory(
    8790  size,
    8791  suballocType,
    8792  memTypeIndex,
    8793  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8794  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8795  finalCreateInfo.pUserData,
    8796  dedicatedBuffer,
    8797  dedicatedImage,
    8798  pAllocation);
    8799  if(res == VK_SUCCESS)
    8800  {
    8801  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
    8802  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
    8803  return VK_SUCCESS;
    8804  }
    8805  else
    8806  {
    8807  // Everything failed: Return error code.
    8808  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8809  return res;
    8810  }
    8811  }
    8812  }
    8813 }
    8814 
    8815 VkResult VmaAllocator_T::AllocateDedicatedMemory(
    8816  VkDeviceSize size,
    8817  VmaSuballocationType suballocType,
    8818  uint32_t memTypeIndex,
    8819  bool map,
    8820  bool isUserDataString,
    8821  void* pUserData,
    8822  VkBuffer dedicatedBuffer,
    8823  VkImage dedicatedImage,
    8824  VmaAllocation* pAllocation)
    8825 {
    8826  VMA_ASSERT(pAllocation);
    8827 
    8828  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    8829  allocInfo.memoryTypeIndex = memTypeIndex;
    8830  allocInfo.allocationSize = size;
    8831 
    8832 #if VMA_DEDICATED_ALLOCATION
    8833  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
    8834  if(m_UseKhrDedicatedAllocation)
    8835  {
    8836  if(dedicatedBuffer != VK_NULL_HANDLE)
    8837  {
    8838  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
    8839  dedicatedAllocInfo.buffer = dedicatedBuffer;
    8840  allocInfo.pNext = &dedicatedAllocInfo;
    8841  }
    8842  else if(dedicatedImage != VK_NULL_HANDLE)
    8843  {
    8844  dedicatedAllocInfo.image = dedicatedImage;
    8845  allocInfo.pNext = &dedicatedAllocInfo;
    8846  }
    8847  }
    8848 #endif // #if VMA_DEDICATED_ALLOCATION
    8849 
    8850  // Allocate VkDeviceMemory.
    8851  VkDeviceMemory hMemory = VK_NULL_HANDLE;
    8852  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
    8853  if(res < 0)
    8854  {
    8855  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8856  return res;
    8857  }
    8858 
    8859  void* pMappedData = VMA_NULL;
    8860  if(map)
    8861  {
    8862  res = (*m_VulkanFunctions.vkMapMemory)(
    8863  m_hDevice,
    8864  hMemory,
    8865  0,
    8866  VK_WHOLE_SIZE,
    8867  0,
    8868  &pMappedData);
    8869  if(res < 0)
    8870  {
    8871  VMA_DEBUG_LOG(" vkMapMemory FAILED");
    8872  FreeVulkanMemory(memTypeIndex, size, hMemory);
    8873  return res;
    8874  }
    8875  }
    8876 
    8877  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
    8878  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
    8879  (*pAllocation)->SetUserData(this, pUserData);
    8880  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    8881  {
    8882  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    8883  }
    8884 
    8885  // Register it in m_pDedicatedAllocations.
    8886  {
    8887  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    8888  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    8889  VMA_ASSERT(pDedicatedAllocations);
    8890  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
    8891  }
    8892 
    8893  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
    8894 
    8895  return VK_SUCCESS;
    8896 }
    8897 
    8898 void VmaAllocator_T::GetBufferMemoryRequirements(
    8899  VkBuffer hBuffer,
    8900  VkMemoryRequirements& memReq,
    8901  bool& requiresDedicatedAllocation,
    8902  bool& prefersDedicatedAllocation) const
    8903 {
    8904 #if VMA_DEDICATED_ALLOCATION
    8905  if(m_UseKhrDedicatedAllocation)
    8906  {
    8907  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
    8908  memReqInfo.buffer = hBuffer;
    8909 
    8910  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    8911 
    8912  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    8913  memReq2.pNext = &memDedicatedReq;
    8914 
    8915  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    8916 
    8917  memReq = memReq2.memoryRequirements;
    8918  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    8919  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    8920  }
    8921  else
    8922 #endif // #if VMA_DEDICATED_ALLOCATION
    8923  {
    8924  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
    8925  requiresDedicatedAllocation = false;
    8926  prefersDedicatedAllocation = false;
    8927  }
    8928 }
    8929 
    8930 void VmaAllocator_T::GetImageMemoryRequirements(
    8931  VkImage hImage,
    8932  VkMemoryRequirements& memReq,
    8933  bool& requiresDedicatedAllocation,
    8934  bool& prefersDedicatedAllocation) const
    8935 {
    8936 #if VMA_DEDICATED_ALLOCATION
    8937  if(m_UseKhrDedicatedAllocation)
    8938  {
    8939  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
    8940  memReqInfo.image = hImage;
    8941 
    8942  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    8943 
    8944  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    8945  memReq2.pNext = &memDedicatedReq;
    8946 
    8947  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    8948 
    8949  memReq = memReq2.memoryRequirements;
    8950  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    8951  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    8952  }
    8953  else
    8954 #endif // #if VMA_DEDICATED_ALLOCATION
    8955  {
    8956  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
    8957  requiresDedicatedAllocation = false;
    8958  prefersDedicatedAllocation = false;
    8959  }
    8960 }
    8961 
    8962 VkResult VmaAllocator_T::AllocateMemory(
    8963  const VkMemoryRequirements& vkMemReq,
    8964  bool requiresDedicatedAllocation,
    8965  bool prefersDedicatedAllocation,
    8966  VkBuffer dedicatedBuffer,
    8967  VkImage dedicatedImage,
    8968  const VmaAllocationCreateInfo& createInfo,
    8969  VmaSuballocationType suballocType,
    8970  VmaAllocation* pAllocation)
    8971 {
    8972  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
    8973  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8974  {
    8975  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
    8976  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8977  }
    8978  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8979  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
    8980  {
    8981  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
    8982  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8983  }
    8984  if(requiresDedicatedAllocation)
    8985  {
    8986  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8987  {
    8988  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
    8989  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8990  }
    8991  if(createInfo.pool != VK_NULL_HANDLE)
    8992  {
    8993  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
    8994  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8995  }
    8996  }
    8997  if((createInfo.pool != VK_NULL_HANDLE) &&
    8998  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
    8999  {
    9000  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
    9001  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9002  }
    9003 
    9004  if(createInfo.pool != VK_NULL_HANDLE)
    9005  {
    9006  const VkDeviceSize alignmentForPool = VMA_MAX(
    9007  vkMemReq.alignment,
    9008  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
    9009  return createInfo.pool->m_BlockVector.Allocate(
    9010  createInfo.pool,
    9011  m_CurrentFrameIndex.load(),
    9012  vkMemReq.size,
    9013  alignmentForPool,
    9014  createInfo,
    9015  suballocType,
    9016  pAllocation);
    9017  }
    9018  else
    9019  {
    9020  // Bit mask of memory Vulkan types acceptable for this allocation.
    9021  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
    9022  uint32_t memTypeIndex = UINT32_MAX;
    9023  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9024  if(res == VK_SUCCESS)
    9025  {
    9026  VkDeviceSize alignmentForMemType = VMA_MAX(
    9027  vkMemReq.alignment,
    9028  GetMemoryTypeMinAlignment(memTypeIndex));
    9029 
    9030  res = AllocateMemoryOfType(
    9031  vkMemReq.size,
    9032  alignmentForMemType,
    9033  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9034  dedicatedBuffer,
    9035  dedicatedImage,
    9036  createInfo,
    9037  memTypeIndex,
    9038  suballocType,
    9039  pAllocation);
    9040  // Succeeded on first try.
    9041  if(res == VK_SUCCESS)
    9042  {
    9043  return res;
    9044  }
    9045  // Allocation from this memory type failed. Try other compatible memory types.
    9046  else
    9047  {
    9048  for(;;)
    9049  {
    9050  // Remove old memTypeIndex from list of possibilities.
    9051  memoryTypeBits &= ~(1u << memTypeIndex);
    9052  // Find alternative memTypeIndex.
    9053  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9054  if(res == VK_SUCCESS)
    9055  {
    9056  alignmentForMemType = VMA_MAX(
    9057  vkMemReq.alignment,
    9058  GetMemoryTypeMinAlignment(memTypeIndex));
    9059 
    9060  res = AllocateMemoryOfType(
    9061  vkMemReq.size,
    9062  alignmentForMemType,
    9063  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9064  dedicatedBuffer,
    9065  dedicatedImage,
    9066  createInfo,
    9067  memTypeIndex,
    9068  suballocType,
    9069  pAllocation);
    9070  // Allocation from this alternative memory type succeeded.
    9071  if(res == VK_SUCCESS)
    9072  {
    9073  return res;
    9074  }
    9075  // else: Allocation from this memory type failed. Try next one - next loop iteration.
    9076  }
    9077  // No other matching memory type index could be found.
    9078  else
    9079  {
    9080  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
    9081  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9082  }
    9083  }
    9084  }
    9085  }
    9086  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
    9087  else
    9088  return res;
    9089  }
    9090 }
    9091 
    9092 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
    9093 {
    9094  VMA_ASSERT(allocation);
    9095 
    9096  if(allocation->CanBecomeLost() == false ||
    9097  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    9098  {
    9099  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    9100  {
    9101  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
    9102  }
    9103 
    9104  switch(allocation->GetType())
    9105  {
    9106  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9107  {
    9108  VmaBlockVector* pBlockVector = VMA_NULL;
    9109  VmaPool hPool = allocation->GetPool();
    9110  if(hPool != VK_NULL_HANDLE)
    9111  {
    9112  pBlockVector = &hPool->m_BlockVector;
    9113  }
    9114  else
    9115  {
    9116  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9117  pBlockVector = m_pBlockVectors[memTypeIndex];
    9118  }
    9119  pBlockVector->Free(allocation);
    9120  }
    9121  break;
    9122  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9123  FreeDedicatedMemory(allocation);
    9124  break;
    9125  default:
    9126  VMA_ASSERT(0);
    9127  }
    9128  }
    9129 
    9130  allocation->SetUserData(this, VMA_NULL);
    9131  vma_delete(this, allocation);
    9132 }
    9133 
    9134 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
    9135 {
    9136  // Initialize.
    9137  InitStatInfo(pStats->total);
    9138  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
    9139  InitStatInfo(pStats->memoryType[i]);
    9140  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    9141  InitStatInfo(pStats->memoryHeap[i]);
    9142 
    9143  // Process default pools.
    9144  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9145  {
    9146  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9147  VMA_ASSERT(pBlockVector);
    9148  pBlockVector->AddStats(pStats);
    9149  }
    9150 
    9151  // Process custom pools.
    9152  {
    9153  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9154  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9155  {
    9156  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
    9157  }
    9158  }
    9159 
    9160  // Process dedicated allocations.
    9161  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9162  {
    9163  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    9164  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9165  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9166  VMA_ASSERT(pDedicatedAllocVector);
    9167  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
    9168  {
    9169  VmaStatInfo allocationStatInfo;
    9170  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
    9171  VmaAddStatInfo(pStats->total, allocationStatInfo);
    9172  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    9173  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    9174  }
    9175  }
    9176 
    9177  // Postprocess.
    9178  VmaPostprocessCalcStatInfo(pStats->total);
    9179  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
    9180  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
    9181  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
    9182  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
    9183 }
    9184 
    9185 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
    9186 
    9187 VkResult VmaAllocator_T::Defragment(
    9188  VmaAllocation* pAllocations,
    9189  size_t allocationCount,
    9190  VkBool32* pAllocationsChanged,
    9191  const VmaDefragmentationInfo* pDefragmentationInfo,
    9192  VmaDefragmentationStats* pDefragmentationStats)
    9193 {
    9194  if(pAllocationsChanged != VMA_NULL)
    9195  {
    9196  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
    9197  }
    9198  if(pDefragmentationStats != VMA_NULL)
    9199  {
    9200  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
    9201  }
    9202 
    9203  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
    9204 
    9205  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
    9206 
    9207  const size_t poolCount = m_Pools.size();
    9208 
    9209  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
    9210  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
    9211  {
    9212  VmaAllocation hAlloc = pAllocations[allocIndex];
    9213  VMA_ASSERT(hAlloc);
    9214  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
    9215  // DedicatedAlloc cannot be defragmented.
    9216  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
    9217  // Only HOST_VISIBLE memory types can be defragmented.
    9218  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
    9219  // Lost allocation cannot be defragmented.
    9220  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
    9221  {
    9222  VmaBlockVector* pAllocBlockVector = VMA_NULL;
    9223 
    9224  const VmaPool hAllocPool = hAlloc->GetPool();
    9225  // This allocation belongs to custom pool.
    9226  if(hAllocPool != VK_NULL_HANDLE)
    9227  {
    9228  pAllocBlockVector = &hAllocPool->GetBlockVector();
    9229  }
    9230  // This allocation belongs to general pool.
    9231  else
    9232  {
    9233  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
    9234  }
    9235 
    9236  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
    9237 
    9238  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
    9239  &pAllocationsChanged[allocIndex] : VMA_NULL;
    9240  pDefragmentator->AddAllocation(hAlloc, pChanged);
    9241  }
    9242  }
    9243 
    9244  VkResult result = VK_SUCCESS;
    9245 
    9246  // ======== Main processing.
    9247 
    9248  VkDeviceSize maxBytesToMove = SIZE_MAX;
    9249  uint32_t maxAllocationsToMove = UINT32_MAX;
    9250  if(pDefragmentationInfo != VMA_NULL)
    9251  {
    9252  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
    9253  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
    9254  }
    9255 
    9256  // Process standard memory.
    9257  for(uint32_t memTypeIndex = 0;
    9258  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
    9259  ++memTypeIndex)
    9260  {
    9261  // Only HOST_VISIBLE memory types can be defragmented.
    9262  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9263  {
    9264  result = m_pBlockVectors[memTypeIndex]->Defragment(
    9265  pDefragmentationStats,
    9266  maxBytesToMove,
    9267  maxAllocationsToMove);
    9268  }
    9269  }
    9270 
    9271  // Process custom pools.
    9272  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
    9273  {
    9274  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
    9275  pDefragmentationStats,
    9276  maxBytesToMove,
    9277  maxAllocationsToMove);
    9278  }
    9279 
    9280  // ======== Destroy defragmentators.
    9281 
    9282  // Process custom pools.
    9283  for(size_t poolIndex = poolCount; poolIndex--; )
    9284  {
    9285  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
    9286  }
    9287 
    9288  // Process standard memory.
    9289  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
    9290  {
    9291  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9292  {
    9293  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
    9294  }
    9295  }
    9296 
    9297  return result;
    9298 }
    9299 
    9300 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
    9301 {
    9302  if(hAllocation->CanBecomeLost())
    9303  {
    9304  /*
    9305  Warning: This is a carefully designed algorithm.
    9306  Do not modify unless you really know what you're doing :)
    9307  */
    9308  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9309  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9310  for(;;)
    9311  {
    9312  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9313  {
    9314  pAllocationInfo->memoryType = UINT32_MAX;
    9315  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
    9316  pAllocationInfo->offset = 0;
    9317  pAllocationInfo->size = hAllocation->GetSize();
    9318  pAllocationInfo->pMappedData = VMA_NULL;
    9319  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9320  return;
    9321  }
    9322  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9323  {
    9324  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9325  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9326  pAllocationInfo->offset = hAllocation->GetOffset();
    9327  pAllocationInfo->size = hAllocation->GetSize();
    9328  pAllocationInfo->pMappedData = VMA_NULL;
    9329  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9330  return;
    9331  }
    9332  else // Last use time earlier than current time.
    9333  {
    9334  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9335  {
    9336  localLastUseFrameIndex = localCurrFrameIndex;
    9337  }
    9338  }
    9339  }
    9340  }
    9341  else
    9342  {
    9343 #if VMA_STATS_STRING_ENABLED
    9344  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9345  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9346  for(;;)
    9347  {
    9348  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9349  if(localLastUseFrameIndex == localCurrFrameIndex)
    9350  {
    9351  break;
    9352  }
    9353  else // Last use time earlier than current time.
    9354  {
    9355  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9356  {
    9357  localLastUseFrameIndex = localCurrFrameIndex;
    9358  }
    9359  }
    9360  }
    9361 #endif
    9362 
    9363  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9364  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9365  pAllocationInfo->offset = hAllocation->GetOffset();
    9366  pAllocationInfo->size = hAllocation->GetSize();
    9367  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
    9368  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9369  }
    9370 }
    9371 
    9372 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
    9373 {
    9374  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
    9375  if(hAllocation->CanBecomeLost())
    9376  {
    9377  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9378  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9379  for(;;)
    9380  {
    9381  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9382  {
    9383  return false;
    9384  }
    9385  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9386  {
    9387  return true;
    9388  }
    9389  else // Last use time earlier than current time.
    9390  {
    9391  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9392  {
    9393  localLastUseFrameIndex = localCurrFrameIndex;
    9394  }
    9395  }
    9396  }
    9397  }
    9398  else
    9399  {
    9400 #if VMA_STATS_STRING_ENABLED
    9401  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9402  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9403  for(;;)
    9404  {
    9405  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9406  if(localLastUseFrameIndex == localCurrFrameIndex)
    9407  {
    9408  break;
    9409  }
    9410  else // Last use time earlier than current time.
    9411  {
    9412  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9413  {
    9414  localLastUseFrameIndex = localCurrFrameIndex;
    9415  }
    9416  }
    9417  }
    9418 #endif
    9419 
    9420  return true;
    9421  }
    9422 }
    9423 
    9424 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
    9425 {
    9426  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
    9427 
    9428  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
    9429 
    9430  if(newCreateInfo.maxBlockCount == 0)
    9431  {
    9432  newCreateInfo.maxBlockCount = SIZE_MAX;
    9433  }
    9434  if(newCreateInfo.blockSize == 0)
    9435  {
    9436  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
    9437  }
    9438 
    9439  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
    9440 
    9441  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
    9442  if(res != VK_SUCCESS)
    9443  {
    9444  vma_delete(this, *pPool);
    9445  *pPool = VMA_NULL;
    9446  return res;
    9447  }
    9448 
    9449  // Add to m_Pools.
    9450  {
    9451  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9452  (*pPool)->SetId(m_NextPoolId++);
    9453  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
    9454  }
    9455 
    9456  return VK_SUCCESS;
    9457 }
    9458 
    9459 void VmaAllocator_T::DestroyPool(VmaPool pool)
    9460 {
    9461  // Remove from m_Pools.
    9462  {
    9463  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9464  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
    9465  VMA_ASSERT(success && "Pool not found in Allocator.");
    9466  }
    9467 
    9468  vma_delete(this, pool);
    9469 }
    9470 
    9471 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
    9472 {
    9473  pool->m_BlockVector.GetPoolStats(pPoolStats);
    9474 }
    9475 
    9476 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
    9477 {
    9478  m_CurrentFrameIndex.store(frameIndex);
    9479 }
    9480 
    9481 void VmaAllocator_T::MakePoolAllocationsLost(
    9482  VmaPool hPool,
    9483  size_t* pLostAllocationCount)
    9484 {
    9485  hPool->m_BlockVector.MakePoolAllocationsLost(
    9486  m_CurrentFrameIndex.load(),
    9487  pLostAllocationCount);
    9488 }
    9489 
    9490 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
    9491 {
    9492  return hPool->m_BlockVector.CheckCorruption();
    9493 }
    9494 
    9495 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
    9496 {
    9497  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
    9498 
    9499  // Process default pools.
    9500  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9501  {
    9502  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
    9503  {
    9504  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9505  VMA_ASSERT(pBlockVector);
    9506  VkResult localRes = pBlockVector->CheckCorruption();
    9507  switch(localRes)
    9508  {
    9509  case VK_ERROR_FEATURE_NOT_PRESENT:
    9510  break;
    9511  case VK_SUCCESS:
    9512  finalRes = VK_SUCCESS;
    9513  break;
    9514  default:
    9515  return localRes;
    9516  }
    9517  }
    9518  }
    9519 
    9520  // Process custom pools.
    9521  {
    9522  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9523  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9524  {
    9525  if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0)
    9526  {
    9527  VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption();
    9528  switch(localRes)
    9529  {
    9530  case VK_ERROR_FEATURE_NOT_PRESENT:
    9531  break;
    9532  case VK_SUCCESS:
    9533  finalRes = VK_SUCCESS;
    9534  break;
    9535  default:
    9536  return localRes;
    9537  }
    9538  }
    9539  }
    9540  }
    9541 
    9542  return finalRes;
    9543 }
    9544 
    9545 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
    9546 {
    9547  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
    9548  (*pAllocation)->InitLost();
    9549 }
    9550 
    9551 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
    9552 {
    9553  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
    9554 
    9555  VkResult res;
    9556  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9557  {
    9558  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9559  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
    9560  {
    9561  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9562  if(res == VK_SUCCESS)
    9563  {
    9564  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
    9565  }
    9566  }
    9567  else
    9568  {
    9569  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9570  }
    9571  }
    9572  else
    9573  {
    9574  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9575  }
    9576 
    9577  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
    9578  {
    9579  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
    9580  }
    9581 
    9582  return res;
    9583 }
    9584 
    9585 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
    9586 {
    9587  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
    9588  {
    9589  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
    9590  }
    9591 
    9592  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
    9593 
    9594  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
    9595  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9596  {
    9597  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9598  m_HeapSizeLimit[heapIndex] += size;
    9599  }
    9600 }
    9601 
    9602 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
    9603 {
    9604  if(hAllocation->CanBecomeLost())
    9605  {
    9606  return VK_ERROR_MEMORY_MAP_FAILED;
    9607  }
    9608 
    9609  switch(hAllocation->GetType())
    9610  {
    9611  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9612  {
    9613  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9614  char *pBytes = VMA_NULL;
    9615  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
    9616  if(res == VK_SUCCESS)
    9617  {
    9618  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
    9619  hAllocation->BlockAllocMap();
    9620  }
    9621  return res;
    9622  }
    9623  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9624  return hAllocation->DedicatedAllocMap(this, ppData);
    9625  default:
    9626  VMA_ASSERT(0);
    9627  return VK_ERROR_MEMORY_MAP_FAILED;
    9628  }
    9629 }
    9630 
    9631 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
    9632 {
    9633  switch(hAllocation->GetType())
    9634  {
    9635  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9636  {
    9637  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9638  hAllocation->BlockAllocUnmap();
    9639  pBlock->Unmap(this, 1);
    9640  }
    9641  break;
    9642  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9643  hAllocation->DedicatedAllocUnmap(this);
    9644  break;
    9645  default:
    9646  VMA_ASSERT(0);
    9647  }
    9648 }
    9649 
    9650 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
    9651 {
    9652  VkResult res = VK_SUCCESS;
    9653  switch(hAllocation->GetType())
    9654  {
    9655  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9656  res = GetVulkanFunctions().vkBindBufferMemory(
    9657  m_hDevice,
    9658  hBuffer,
    9659  hAllocation->GetMemory(),
    9660  0); //memoryOffset
    9661  break;
    9662  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9663  {
    9664  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9665  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
    9666  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
    9667  break;
    9668  }
    9669  default:
    9670  VMA_ASSERT(0);
    9671  }
    9672  return res;
    9673 }
    9674 
    9675 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
    9676 {
    9677  VkResult res = VK_SUCCESS;
    9678  switch(hAllocation->GetType())
    9679  {
    9680  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9681  res = GetVulkanFunctions().vkBindImageMemory(
    9682  m_hDevice,
    9683  hImage,
    9684  hAllocation->GetMemory(),
    9685  0); //memoryOffset
    9686  break;
    9687  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9688  {
    9689  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9690  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
    9691  res = pBlock->BindImageMemory(this, hAllocation, hImage);
    9692  break;
    9693  }
    9694  default:
    9695  VMA_ASSERT(0);
    9696  }
    9697  return res;
    9698 }
    9699 
    9700 void VmaAllocator_T::FlushOrInvalidateAllocation(
    9701  VmaAllocation hAllocation,
    9702  VkDeviceSize offset, VkDeviceSize size,
    9703  VMA_CACHE_OPERATION op)
    9704 {
    9705  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
    9706  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
    9707  {
    9708  const VkDeviceSize allocationSize = hAllocation->GetSize();
    9709  VMA_ASSERT(offset <= allocationSize);
    9710 
    9711  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
    9712 
    9713  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
    9714  memRange.memory = hAllocation->GetMemory();
    9715 
    9716  switch(hAllocation->GetType())
    9717  {
    9718  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9719  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9720  if(size == VK_WHOLE_SIZE)
    9721  {
    9722  memRange.size = allocationSize - memRange.offset;
    9723  }
    9724  else
    9725  {
    9726  VMA_ASSERT(offset + size <= allocationSize);
    9727  memRange.size = VMA_MIN(
    9728  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
    9729  allocationSize - memRange.offset);
    9730  }
    9731  break;
    9732 
    9733  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9734  {
    9735  // 1. Still within this allocation.
    9736  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9737  if(size == VK_WHOLE_SIZE)
    9738  {
    9739  size = allocationSize - offset;
    9740  }
    9741  else
    9742  {
    9743  VMA_ASSERT(offset + size <= allocationSize);
    9744  }
    9745  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
    9746 
    9747  // 2. Adjust to whole block.
    9748  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
    9749  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
    9750  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize();
    9751  memRange.offset += allocationOffset;
    9752  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
    9753 
    9754  break;
    9755  }
    9756 
    9757  default:
    9758  VMA_ASSERT(0);
    9759  }
    9760 
    9761  switch(op)
    9762  {
    9763  case VMA_CACHE_FLUSH:
    9764  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9765  break;
    9766  case VMA_CACHE_INVALIDATE:
    9767  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9768  break;
    9769  default:
    9770  VMA_ASSERT(0);
    9771  }
    9772  }
    9773  // else: Just ignore this call.
    9774 }
    9775 
    9776 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
    9777 {
    9778  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
    9779 
    9780  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9781  {
    9782  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9783  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    9784  VMA_ASSERT(pDedicatedAllocations);
    9785  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
    9786  VMA_ASSERT(success);
    9787  }
    9788 
    9789  VkDeviceMemory hMemory = allocation->GetMemory();
    9790 
    9791  if(allocation->GetMappedData() != VMA_NULL)
    9792  {
    9793  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
    9794  }
    9795 
    9796  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
    9797 
    9798  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
    9799 }
    9800 
    9801 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
    9802 {
    9803  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
    9804  !hAllocation->CanBecomeLost() &&
    9805  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9806  {
    9807  void* pData = VMA_NULL;
    9808  VkResult res = Map(hAllocation, &pData);
    9809  if(res == VK_SUCCESS)
    9810  {
    9811  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
    9812  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
    9813  Unmap(hAllocation);
    9814  }
    9815  else
    9816  {
    9817  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
    9818  }
    9819  }
    9820 }
    9821 
    9822 #if VMA_STATS_STRING_ENABLED
    9823 
    9824 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
    9825 {
    9826  bool dedicatedAllocationsStarted = false;
    9827  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9828  {
    9829  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9830  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9831  VMA_ASSERT(pDedicatedAllocVector);
    9832  if(pDedicatedAllocVector->empty() == false)
    9833  {
    9834  if(dedicatedAllocationsStarted == false)
    9835  {
    9836  dedicatedAllocationsStarted = true;
    9837  json.WriteString("DedicatedAllocations");
    9838  json.BeginObject();
    9839  }
    9840 
    9841  json.BeginString("Type ");
    9842  json.ContinueString(memTypeIndex);
    9843  json.EndString();
    9844 
    9845  json.BeginArray();
    9846 
    9847  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
    9848  {
    9849  json.BeginObject(true);
    9850  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
    9851  hAlloc->PrintParameters(json);
    9852  json.EndObject();
    9853  }
    9854 
    9855  json.EndArray();
    9856  }
    9857  }
    9858  if(dedicatedAllocationsStarted)
    9859  {
    9860  json.EndObject();
    9861  }
    9862 
    9863  {
    9864  bool allocationsStarted = false;
    9865  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9866  {
    9867  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
    9868  {
    9869  if(allocationsStarted == false)
    9870  {
    9871  allocationsStarted = true;
    9872  json.WriteString("DefaultPools");
    9873  json.BeginObject();
    9874  }
    9875 
    9876  json.BeginString("Type ");
    9877  json.ContinueString(memTypeIndex);
    9878  json.EndString();
    9879 
    9880  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
    9881  }
    9882  }
    9883  if(allocationsStarted)
    9884  {
    9885  json.EndObject();
    9886  }
    9887  }
    9888 
    9889  {
    9890  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9891  const size_t poolCount = m_Pools.size();
    9892  if(poolCount > 0)
    9893  {
    9894  json.WriteString("Pools");
    9895  json.BeginObject();
    9896  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
    9897  {
    9898  json.BeginString();
    9899  json.ContinueString(m_Pools[poolIndex]->GetId());
    9900  json.EndString();
    9901 
    9902  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
    9903  }
    9904  json.EndObject();
    9905  }
    9906  }
    9907 }
    9908 
    9909 #endif // #if VMA_STATS_STRING_ENABLED
    9910 
    9912 // Public interface
    9913 
    9914 VkResult vmaCreateAllocator(
    9915  const VmaAllocatorCreateInfo* pCreateInfo,
    9916  VmaAllocator* pAllocator)
    9917 {
    9918  VMA_ASSERT(pCreateInfo && pAllocator);
    9919  VMA_DEBUG_LOG("vmaCreateAllocator");
    9920  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
    9921  return (*pAllocator)->Init(pCreateInfo);
    9922 }
    9923 
    9924 void vmaDestroyAllocator(
    9925  VmaAllocator allocator)
    9926 {
    9927  if(allocator != VK_NULL_HANDLE)
    9928  {
    9929  VMA_DEBUG_LOG("vmaDestroyAllocator");
    9930  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
    9931  vma_delete(&allocationCallbacks, allocator);
    9932  }
    9933 }
    9934 
    9935 void vmaGetPhysicalDeviceProperties(
    9936  VmaAllocator allocator,
    9937  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
    9938 {
    9939  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
    9940  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
    9941 }
    9942 
    9943 void vmaGetMemoryProperties(
    9944  VmaAllocator allocator,
    9945  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
    9946 {
    9947  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
    9948  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
    9949 }
    9950 
    9951 void vmaGetMemoryTypeProperties(
    9952  VmaAllocator allocator,
    9953  uint32_t memoryTypeIndex,
    9954  VkMemoryPropertyFlags* pFlags)
    9955 {
    9956  VMA_ASSERT(allocator && pFlags);
    9957  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
    9958  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
    9959 }
    9960 
    9961 void vmaSetCurrentFrameIndex(
    9962  VmaAllocator allocator,
    9963  uint32_t frameIndex)
    9964 {
    9965  VMA_ASSERT(allocator);
    9966  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
    9967 
    9968  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9969 
    9970  allocator->SetCurrentFrameIndex(frameIndex);
    9971 }
    9972 
    9973 void vmaCalculateStats(
    9974  VmaAllocator allocator,
    9975  VmaStats* pStats)
    9976 {
    9977  VMA_ASSERT(allocator && pStats);
    9978  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9979  allocator->CalculateStats(pStats);
    9980 }
    9981 
    9982 #if VMA_STATS_STRING_ENABLED
    9983 
    9984 void vmaBuildStatsString(
    9985  VmaAllocator allocator,
    9986  char** ppStatsString,
    9987  VkBool32 detailedMap)
    9988 {
    9989  VMA_ASSERT(allocator && ppStatsString);
    9990  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    9991 
    9992  VmaStringBuilder sb(allocator);
    9993  {
    9994  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
    9995  json.BeginObject();
    9996 
    9997  VmaStats stats;
    9998  allocator->CalculateStats(&stats);
    9999 
    10000  json.WriteString("Total");
    10001  VmaPrintStatInfo(json, stats.total);
    10002 
    10003  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
    10004  {
    10005  json.BeginString("Heap ");
    10006  json.ContinueString(heapIndex);
    10007  json.EndString();
    10008  json.BeginObject();
    10009 
    10010  json.WriteString("Size");
    10011  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
    10012 
    10013  json.WriteString("Flags");
    10014  json.BeginArray(true);
    10015  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
    10016  {
    10017  json.WriteString("DEVICE_LOCAL");
    10018  }
    10019  json.EndArray();
    10020 
    10021  if(stats.memoryHeap[heapIndex].blockCount > 0)
    10022  {
    10023  json.WriteString("Stats");
    10024  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
    10025  }
    10026 
    10027  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
    10028  {
    10029  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
    10030  {
    10031  json.BeginString("Type ");
    10032  json.ContinueString(typeIndex);
    10033  json.EndString();
    10034 
    10035  json.BeginObject();
    10036 
    10037  json.WriteString("Flags");
    10038  json.BeginArray(true);
    10039  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
    10040  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
    10041  {
    10042  json.WriteString("DEVICE_LOCAL");
    10043  }
    10044  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    10045  {
    10046  json.WriteString("HOST_VISIBLE");
    10047  }
    10048  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
    10049  {
    10050  json.WriteString("HOST_COHERENT");
    10051  }
    10052  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
    10053  {
    10054  json.WriteString("HOST_CACHED");
    10055  }
    10056  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
    10057  {
    10058  json.WriteString("LAZILY_ALLOCATED");
    10059  }
    10060  json.EndArray();
    10061 
    10062  if(stats.memoryType[typeIndex].blockCount > 0)
    10063  {
    10064  json.WriteString("Stats");
    10065  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
    10066  }
    10067 
    10068  json.EndObject();
    10069  }
    10070  }
    10071 
    10072  json.EndObject();
    10073  }
    10074  if(detailedMap == VK_TRUE)
    10075  {
    10076  allocator->PrintDetailedMap(json);
    10077  }
    10078 
    10079  json.EndObject();
    10080  }
    10081 
    10082  const size_t len = sb.GetLength();
    10083  char* const pChars = vma_new_array(allocator, char, len + 1);
    10084  if(len > 0)
    10085  {
    10086  memcpy(pChars, sb.GetData(), len);
    10087  }
    10088  pChars[len] = '\0';
    10089  *ppStatsString = pChars;
    10090 }
    10091 
    10092 void vmaFreeStatsString(
    10093  VmaAllocator allocator,
    10094  char* pStatsString)
    10095 {
    10096  if(pStatsString != VMA_NULL)
    10097  {
    10098  VMA_ASSERT(allocator);
    10099  size_t len = strlen(pStatsString);
    10100  vma_delete_array(allocator, pStatsString, len + 1);
    10101  }
    10102 }
    10103 
    10104 #endif // #if VMA_STATS_STRING_ENABLED
    10105 
    10106 /*
    10107 This function is not protected by any mutex because it just reads immutable data.
    10108 */
    10109 VkResult vmaFindMemoryTypeIndex(
    10110  VmaAllocator allocator,
    10111  uint32_t memoryTypeBits,
    10112  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10113  uint32_t* pMemoryTypeIndex)
    10114 {
    10115  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10116  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10117  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10118 
    10119  if(pAllocationCreateInfo->memoryTypeBits != 0)
    10120  {
    10121  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
    10122  }
    10123 
    10124  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
    10125  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
    10126 
    10127  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    10128  if(mapped)
    10129  {
    10130  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10131  }
    10132 
    10133  // Convert usage to requiredFlags and preferredFlags.
    10134  switch(pAllocationCreateInfo->usage)
    10135  {
    10136  case VMA_MEMORY_USAGE_UNKNOWN:
    10137  break;
    10138  case VMA_MEMORY_USAGE_GPU_ONLY:
    10139  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10140  {
    10141  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10142  }
    10143  break;
    10144  case VMA_MEMORY_USAGE_CPU_ONLY:
    10145  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    10146  break;
    10147  case VMA_MEMORY_USAGE_CPU_TO_GPU:
    10148  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10149  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10150  {
    10151  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10152  }
    10153  break;
    10154  case VMA_MEMORY_USAGE_GPU_TO_CPU:
    10155  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10156  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
    10157  break;
    10158  default:
    10159  break;
    10160  }
    10161 
    10162  *pMemoryTypeIndex = UINT32_MAX;
    10163  uint32_t minCost = UINT32_MAX;
    10164  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
    10165  memTypeIndex < allocator->GetMemoryTypeCount();
    10166  ++memTypeIndex, memTypeBit <<= 1)
    10167  {
    10168  // This memory type is acceptable according to memoryTypeBits bitmask.
    10169  if((memTypeBit & memoryTypeBits) != 0)
    10170  {
    10171  const VkMemoryPropertyFlags currFlags =
    10172  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
    10173  // This memory type contains requiredFlags.
    10174  if((requiredFlags & ~currFlags) == 0)
    10175  {
    10176  // Calculate cost as number of bits from preferredFlags not present in this memory type.
    10177  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
    10178  // Remember memory type with lowest cost.
    10179  if(currCost < minCost)
    10180  {
    10181  *pMemoryTypeIndex = memTypeIndex;
    10182  if(currCost == 0)
    10183  {
    10184  return VK_SUCCESS;
    10185  }
    10186  minCost = currCost;
    10187  }
    10188  }
    10189  }
    10190  }
    10191  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
    10192 }
    10193 
    10194 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    10195  VmaAllocator allocator,
    10196  const VkBufferCreateInfo* pBufferCreateInfo,
    10197  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10198  uint32_t* pMemoryTypeIndex)
    10199 {
    10200  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10201  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
    10202  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10203  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10204 
    10205  const VkDevice hDev = allocator->m_hDevice;
    10206  VkBuffer hBuffer = VK_NULL_HANDLE;
    10207  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
    10208  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
    10209  if(res == VK_SUCCESS)
    10210  {
    10211  VkMemoryRequirements memReq = {};
    10212  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
    10213  hDev, hBuffer, &memReq);
    10214 
    10215  res = vmaFindMemoryTypeIndex(
    10216  allocator,
    10217  memReq.memoryTypeBits,
    10218  pAllocationCreateInfo,
    10219  pMemoryTypeIndex);
    10220 
    10221  allocator->GetVulkanFunctions().vkDestroyBuffer(
    10222  hDev, hBuffer, allocator->GetAllocationCallbacks());
    10223  }
    10224  return res;
    10225 }
    10226 
    10227 VkResult vmaFindMemoryTypeIndexForImageInfo(
    10228  VmaAllocator allocator,
    10229  const VkImageCreateInfo* pImageCreateInfo,
    10230  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10231  uint32_t* pMemoryTypeIndex)
    10232 {
    10233  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10234  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
    10235  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10236  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10237 
    10238  const VkDevice hDev = allocator->m_hDevice;
    10239  VkImage hImage = VK_NULL_HANDLE;
    10240  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
    10241  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
    10242  if(res == VK_SUCCESS)
    10243  {
    10244  VkMemoryRequirements memReq = {};
    10245  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
    10246  hDev, hImage, &memReq);
    10247 
    10248  res = vmaFindMemoryTypeIndex(
    10249  allocator,
    10250  memReq.memoryTypeBits,
    10251  pAllocationCreateInfo,
    10252  pMemoryTypeIndex);
    10253 
    10254  allocator->GetVulkanFunctions().vkDestroyImage(
    10255  hDev, hImage, allocator->GetAllocationCallbacks());
    10256  }
    10257  return res;
    10258 }
    10259 
    10260 VkResult vmaCreatePool(
    10261  VmaAllocator allocator,
    10262  const VmaPoolCreateInfo* pCreateInfo,
    10263  VmaPool* pPool)
    10264 {
    10265  VMA_ASSERT(allocator && pCreateInfo && pPool);
    10266 
    10267  VMA_DEBUG_LOG("vmaCreatePool");
    10268 
    10269  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10270 
    10271  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
    10272 
    10273 #if VMA_RECORDING_ENABLED
    10274  if(allocator->GetRecorder() != VMA_NULL)
    10275  {
    10276  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
    10277  }
    10278 #endif
    10279 
    10280  return res;
    10281 }
    10282 
    10283 void vmaDestroyPool(
    10284  VmaAllocator allocator,
    10285  VmaPool pool)
    10286 {
    10287  VMA_ASSERT(allocator);
    10288 
    10289  if(pool == VK_NULL_HANDLE)
    10290  {
    10291  return;
    10292  }
    10293 
    10294  VMA_DEBUG_LOG("vmaDestroyPool");
    10295 
    10296  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10297 
    10298 #if VMA_RECORDING_ENABLED
    10299  if(allocator->GetRecorder() != VMA_NULL)
    10300  {
    10301  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
    10302  }
    10303 #endif
    10304 
    10305  allocator->DestroyPool(pool);
    10306 }
    10307 
    10308 void vmaGetPoolStats(
    10309  VmaAllocator allocator,
    10310  VmaPool pool,
    10311  VmaPoolStats* pPoolStats)
    10312 {
    10313  VMA_ASSERT(allocator && pool && pPoolStats);
    10314 
    10315  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10316 
    10317  allocator->GetPoolStats(pool, pPoolStats);
    10318 }
    10319 
    10320 void vmaMakePoolAllocationsLost(
    10321  VmaAllocator allocator,
    10322  VmaPool pool,
    10323  size_t* pLostAllocationCount)
    10324 {
    10325  VMA_ASSERT(allocator && pool);
    10326 
    10327  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10328 
    10329  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
    10330 }
    10331 
    10332 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
    10333 {
    10334  VMA_ASSERT(allocator && pool);
    10335 
    10336  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10337 
    10338  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
    10339 
    10340  return allocator->CheckPoolCorruption(pool);
    10341 }
    10342 
    10343 VkResult vmaAllocateMemory(
    10344  VmaAllocator allocator,
    10345  const VkMemoryRequirements* pVkMemoryRequirements,
    10346  const VmaAllocationCreateInfo* pCreateInfo,
    10347  VmaAllocation* pAllocation,
    10348  VmaAllocationInfo* pAllocationInfo)
    10349 {
    10350  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
    10351 
    10352  VMA_DEBUG_LOG("vmaAllocateMemory");
    10353 
    10354  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10355 
    10356  VkResult result = allocator->AllocateMemory(
    10357  *pVkMemoryRequirements,
    10358  false, // requiresDedicatedAllocation
    10359  false, // prefersDedicatedAllocation
    10360  VK_NULL_HANDLE, // dedicatedBuffer
    10361  VK_NULL_HANDLE, // dedicatedImage
    10362  *pCreateInfo,
    10363  VMA_SUBALLOCATION_TYPE_UNKNOWN,
    10364  pAllocation);
    10365 
    10366 #if VMA_RECORDING_ENABLED
    10367  if(allocator->GetRecorder() != VMA_NULL)
    10368  {
    10369  allocator->GetRecorder()->RecordAllocateMemory(
    10370  allocator->GetCurrentFrameIndex(),
    10371  *pVkMemoryRequirements,
    10372  *pCreateInfo,
    10373  *pAllocation);
    10374  }
    10375 #endif
    10376 
    10377  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
    10378  {
    10379  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10380  }
    10381 
    10382  return result;
    10383 }
    10384 
    10385 VkResult vmaAllocateMemoryForBuffer(
    10386  VmaAllocator allocator,
    10387  VkBuffer buffer,
    10388  const VmaAllocationCreateInfo* pCreateInfo,
    10389  VmaAllocation* pAllocation,
    10390  VmaAllocationInfo* pAllocationInfo)
    10391 {
    10392  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10393 
    10394  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
    10395 
    10396  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10397 
    10398  VkMemoryRequirements vkMemReq = {};
    10399  bool requiresDedicatedAllocation = false;
    10400  bool prefersDedicatedAllocation = false;
    10401  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
    10402  requiresDedicatedAllocation,
    10403  prefersDedicatedAllocation);
    10404 
    10405  VkResult result = allocator->AllocateMemory(
    10406  vkMemReq,
    10407  requiresDedicatedAllocation,
    10408  prefersDedicatedAllocation,
    10409  buffer, // dedicatedBuffer
    10410  VK_NULL_HANDLE, // dedicatedImage
    10411  *pCreateInfo,
    10412  VMA_SUBALLOCATION_TYPE_BUFFER,
    10413  pAllocation);
    10414 
    10415 #if VMA_RECORDING_ENABLED
    10416  if(allocator->GetRecorder() != VMA_NULL)
    10417  {
    10418  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
    10419  allocator->GetCurrentFrameIndex(),
    10420  vkMemReq,
    10421  requiresDedicatedAllocation,
    10422  prefersDedicatedAllocation,
    10423  *pCreateInfo,
    10424  *pAllocation);
    10425  }
    10426 #endif
    10427 
    10428  if(pAllocationInfo && result == VK_SUCCESS)
    10429  {
    10430  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10431  }
    10432 
    10433  return result;
    10434 }
    10435 
    10436 VkResult vmaAllocateMemoryForImage(
    10437  VmaAllocator allocator,
    10438  VkImage image,
    10439  const VmaAllocationCreateInfo* pCreateInfo,
    10440  VmaAllocation* pAllocation,
    10441  VmaAllocationInfo* pAllocationInfo)
    10442 {
    10443  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10444 
    10445  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
    10446 
    10447  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10448 
    10449  VkMemoryRequirements vkMemReq = {};
    10450  bool requiresDedicatedAllocation = false;
    10451  bool prefersDedicatedAllocation = false;
    10452  allocator->GetImageMemoryRequirements(image, vkMemReq,
    10453  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10454 
    10455  VkResult result = allocator->AllocateMemory(
    10456  vkMemReq,
    10457  requiresDedicatedAllocation,
    10458  prefersDedicatedAllocation,
    10459  VK_NULL_HANDLE, // dedicatedBuffer
    10460  image, // dedicatedImage
    10461  *pCreateInfo,
    10462  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
    10463  pAllocation);
    10464 
    10465 #if VMA_RECORDING_ENABLED
    10466  if(allocator->GetRecorder() != VMA_NULL)
    10467  {
    10468  allocator->GetRecorder()->RecordAllocateMemoryForImage(
    10469  allocator->GetCurrentFrameIndex(),
    10470  vkMemReq,
    10471  requiresDedicatedAllocation,
    10472  prefersDedicatedAllocation,
    10473  *pCreateInfo,
    10474  *pAllocation);
    10475  }
    10476 #endif
    10477 
    10478  if(pAllocationInfo && result == VK_SUCCESS)
    10479  {
    10480  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10481  }
    10482 
    10483  return result;
    10484 }
    10485 
    10486 void vmaFreeMemory(
    10487  VmaAllocator allocator,
    10488  VmaAllocation allocation)
    10489 {
    10490  VMA_ASSERT(allocator);
    10491 
    10492  if(allocation == VK_NULL_HANDLE)
    10493  {
    10494  return;
    10495  }
    10496 
    10497  VMA_DEBUG_LOG("vmaFreeMemory");
    10498 
    10499  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10500 
    10501 #if VMA_RECORDING_ENABLED
    10502  if(allocator->GetRecorder() != VMA_NULL)
    10503  {
    10504  allocator->GetRecorder()->RecordFreeMemory(
    10505  allocator->GetCurrentFrameIndex(),
    10506  allocation);
    10507  }
    10508 #endif
    10509 
    10510  allocator->FreeMemory(allocation);
    10511 }
    10512 
    10513 void vmaGetAllocationInfo(
    10514  VmaAllocator allocator,
    10515  VmaAllocation allocation,
    10516  VmaAllocationInfo* pAllocationInfo)
    10517 {
    10518  VMA_ASSERT(allocator && allocation && pAllocationInfo);
    10519 
    10520  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10521 
    10522  allocator->GetAllocationInfo(allocation, pAllocationInfo);
    10523 }
    10524 
    10525 VkBool32 vmaTouchAllocation(
    10526  VmaAllocator allocator,
    10527  VmaAllocation allocation)
    10528 {
    10529  VMA_ASSERT(allocator && allocation);
    10530 
    10531  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10532 
    10533  return allocator->TouchAllocation(allocation);
    10534 }
    10535 
    10536 void vmaSetAllocationUserData(
    10537  VmaAllocator allocator,
    10538  VmaAllocation allocation,
    10539  void* pUserData)
    10540 {
    10541  VMA_ASSERT(allocator && allocation);
    10542 
    10543  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10544 
    10545  allocation->SetUserData(allocator, pUserData);
    10546 
    10547 #if VMA_RECORDING_ENABLED
    10548  if(allocator->GetRecorder() != VMA_NULL)
    10549  {
    10550  allocator->GetRecorder()->RecordSetAllocationUserData(
    10551  allocator->GetCurrentFrameIndex(),
    10552  allocation,
    10553  pUserData);
    10554  }
    10555 #endif
    10556 }
    10557 
    10558 void vmaCreateLostAllocation(
    10559  VmaAllocator allocator,
    10560  VmaAllocation* pAllocation)
    10561 {
    10562  VMA_ASSERT(allocator && pAllocation);
    10563 
    10564  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
    10565 
    10566  allocator->CreateLostAllocation(pAllocation);
    10567 
    10568 #if VMA_RECORDING_ENABLED
    10569  if(allocator->GetRecorder() != VMA_NULL)
    10570  {
    10571  allocator->GetRecorder()->RecordCreateLostAllocation(
    10572  allocator->GetCurrentFrameIndex(),
    10573  *pAllocation);
    10574  }
    10575 #endif
    10576 }
    10577 
    10578 VkResult vmaMapMemory(
    10579  VmaAllocator allocator,
    10580  VmaAllocation allocation,
    10581  void** ppData)
    10582 {
    10583  VMA_ASSERT(allocator && allocation && ppData);
    10584 
    10585  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10586 
    10587  VkResult res = allocator->Map(allocation, ppData);
    10588 
    10589 #if VMA_RECORDING_ENABLED
    10590  if(allocator->GetRecorder() != VMA_NULL)
    10591  {
    10592  allocator->GetRecorder()->RecordMapMemory(
    10593  allocator->GetCurrentFrameIndex(),
    10594  allocation);
    10595  }
    10596 #endif
    10597 
    10598  return res;
    10599 }
    10600 
    10601 void vmaUnmapMemory(
    10602  VmaAllocator allocator,
    10603  VmaAllocation allocation)
    10604 {
    10605  VMA_ASSERT(allocator && allocation);
    10606 
    10607  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10608 
    10609 #if VMA_RECORDING_ENABLED
    10610  if(allocator->GetRecorder() != VMA_NULL)
    10611  {
    10612  allocator->GetRecorder()->RecordUnmapMemory(
    10613  allocator->GetCurrentFrameIndex(),
    10614  allocation);
    10615  }
    10616 #endif
    10617 
    10618  allocator->Unmap(allocation);
    10619 }
    10620 
    10621 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10622 {
    10623  VMA_ASSERT(allocator && allocation);
    10624 
    10625  VMA_DEBUG_LOG("vmaFlushAllocation");
    10626 
    10627  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10628 
    10629  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
    10630 
    10631 #if VMA_RECORDING_ENABLED
    10632  if(allocator->GetRecorder() != VMA_NULL)
    10633  {
    10634  allocator->GetRecorder()->RecordFlushAllocation(
    10635  allocator->GetCurrentFrameIndex(),
    10636  allocation, offset, size);
    10637  }
    10638 #endif
    10639 }
    10640 
    10641 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10642 {
    10643  VMA_ASSERT(allocator && allocation);
    10644 
    10645  VMA_DEBUG_LOG("vmaInvalidateAllocation");
    10646 
    10647  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10648 
    10649  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
    10650 
    10651 #if VMA_RECORDING_ENABLED
    10652  if(allocator->GetRecorder() != VMA_NULL)
    10653  {
    10654  allocator->GetRecorder()->RecordInvalidateAllocation(
    10655  allocator->GetCurrentFrameIndex(),
    10656  allocation, offset, size);
    10657  }
    10658 #endif
    10659 }
    10660 
    10661 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
    10662 {
    10663  VMA_ASSERT(allocator);
    10664 
    10665  VMA_DEBUG_LOG("vmaCheckCorruption");
    10666 
    10667  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10668 
    10669  return allocator->CheckCorruption(memoryTypeBits);
    10670 }
    10671 
    10672 VkResult vmaDefragment(
    10673  VmaAllocator allocator,
    10674  VmaAllocation* pAllocations,
    10675  size_t allocationCount,
    10676  VkBool32* pAllocationsChanged,
    10677  const VmaDefragmentationInfo *pDefragmentationInfo,
    10678  VmaDefragmentationStats* pDefragmentationStats)
    10679 {
    10680  VMA_ASSERT(allocator && pAllocations);
    10681 
    10682  VMA_DEBUG_LOG("vmaDefragment");
    10683 
    10684  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10685 
    10686  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
    10687 }
    10688 
    10689 VkResult vmaBindBufferMemory(
    10690  VmaAllocator allocator,
    10691  VmaAllocation allocation,
    10692  VkBuffer buffer)
    10693 {
    10694  VMA_ASSERT(allocator && allocation && buffer);
    10695 
    10696  VMA_DEBUG_LOG("vmaBindBufferMemory");
    10697 
    10698  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10699 
    10700  return allocator->BindBufferMemory(allocation, buffer);
    10701 }
    10702 
    10703 VkResult vmaBindImageMemory(
    10704  VmaAllocator allocator,
    10705  VmaAllocation allocation,
    10706  VkImage image)
    10707 {
    10708  VMA_ASSERT(allocator && allocation && image);
    10709 
    10710  VMA_DEBUG_LOG("vmaBindImageMemory");
    10711 
    10712  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10713 
    10714  return allocator->BindImageMemory(allocation, image);
    10715 }
    10716 
    10717 VkResult vmaCreateBuffer(
    10718  VmaAllocator allocator,
    10719  const VkBufferCreateInfo* pBufferCreateInfo,
    10720  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10721  VkBuffer* pBuffer,
    10722  VmaAllocation* pAllocation,
    10723  VmaAllocationInfo* pAllocationInfo)
    10724 {
    10725  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
    10726 
    10727  VMA_DEBUG_LOG("vmaCreateBuffer");
    10728 
    10729  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10730 
    10731  *pBuffer = VK_NULL_HANDLE;
    10732  *pAllocation = VK_NULL_HANDLE;
    10733 
    10734  // 1. Create VkBuffer.
    10735  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
    10736  allocator->m_hDevice,
    10737  pBufferCreateInfo,
    10738  allocator->GetAllocationCallbacks(),
    10739  pBuffer);
    10740  if(res >= 0)
    10741  {
    10742  // 2. vkGetBufferMemoryRequirements.
    10743  VkMemoryRequirements vkMemReq = {};
    10744  bool requiresDedicatedAllocation = false;
    10745  bool prefersDedicatedAllocation = false;
    10746  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
    10747  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10748 
    10749  // Make sure alignment requirements for specific buffer usages reported
    10750  // in Physical Device Properties are included in alignment reported by memory requirements.
    10751  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
    10752  {
    10753  VMA_ASSERT(vkMemReq.alignment %
    10754  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
    10755  }
    10756  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
    10757  {
    10758  VMA_ASSERT(vkMemReq.alignment %
    10759  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
    10760  }
    10761  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
    10762  {
    10763  VMA_ASSERT(vkMemReq.alignment %
    10764  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
    10765  }
    10766 
    10767  // 3. Allocate memory using allocator.
    10768  res = allocator->AllocateMemory(
    10769  vkMemReq,
    10770  requiresDedicatedAllocation,
    10771  prefersDedicatedAllocation,
    10772  *pBuffer, // dedicatedBuffer
    10773  VK_NULL_HANDLE, // dedicatedImage
    10774  *pAllocationCreateInfo,
    10775  VMA_SUBALLOCATION_TYPE_BUFFER,
    10776  pAllocation);
    10777 
    10778 #if VMA_RECORDING_ENABLED
    10779  if(allocator->GetRecorder() != VMA_NULL)
    10780  {
    10781  allocator->GetRecorder()->RecordCreateBuffer(
    10782  allocator->GetCurrentFrameIndex(),
    10783  *pBufferCreateInfo,
    10784  *pAllocationCreateInfo,
    10785  *pAllocation);
    10786  }
    10787 #endif
    10788 
    10789  if(res >= 0)
    10790  {
    10791  // 3. Bind buffer with memory.
    10792  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
    10793  if(res >= 0)
    10794  {
    10795  // All steps succeeded.
    10796  #if VMA_STATS_STRING_ENABLED
    10797  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
    10798  #endif
    10799  if(pAllocationInfo != VMA_NULL)
    10800  {
    10801  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10802  }
    10803 
    10804  return VK_SUCCESS;
    10805  }
    10806  allocator->FreeMemory(*pAllocation);
    10807  *pAllocation = VK_NULL_HANDLE;
    10808  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10809  *pBuffer = VK_NULL_HANDLE;
    10810  return res;
    10811  }
    10812  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10813  *pBuffer = VK_NULL_HANDLE;
    10814  return res;
    10815  }
    10816  return res;
    10817 }
    10818 
    10819 void vmaDestroyBuffer(
    10820  VmaAllocator allocator,
    10821  VkBuffer buffer,
    10822  VmaAllocation allocation)
    10823 {
    10824  VMA_ASSERT(allocator);
    10825 
    10826  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    10827  {
    10828  return;
    10829  }
    10830 
    10831  VMA_DEBUG_LOG("vmaDestroyBuffer");
    10832 
    10833  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10834 
    10835 #if VMA_RECORDING_ENABLED
    10836  if(allocator->GetRecorder() != VMA_NULL)
    10837  {
    10838  allocator->GetRecorder()->RecordDestroyBuffer(
    10839  allocator->GetCurrentFrameIndex(),
    10840  allocation);
    10841  }
    10842 #endif
    10843 
    10844  if(buffer != VK_NULL_HANDLE)
    10845  {
    10846  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
    10847  }
    10848 
    10849  if(allocation != VK_NULL_HANDLE)
    10850  {
    10851  allocator->FreeMemory(allocation);
    10852  }
    10853 }
    10854 
    10855 VkResult vmaCreateImage(
    10856  VmaAllocator allocator,
    10857  const VkImageCreateInfo* pImageCreateInfo,
    10858  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10859  VkImage* pImage,
    10860  VmaAllocation* pAllocation,
    10861  VmaAllocationInfo* pAllocationInfo)
    10862 {
    10863  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
    10864 
    10865  VMA_DEBUG_LOG("vmaCreateImage");
    10866 
    10867  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10868 
    10869  *pImage = VK_NULL_HANDLE;
    10870  *pAllocation = VK_NULL_HANDLE;
    10871 
    10872  // 1. Create VkImage.
    10873  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
    10874  allocator->m_hDevice,
    10875  pImageCreateInfo,
    10876  allocator->GetAllocationCallbacks(),
    10877  pImage);
    10878  if(res >= 0)
    10879  {
    10880  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
    10881  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
    10882  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
    10883 
    10884  // 2. Allocate memory using allocator.
    10885  VkMemoryRequirements vkMemReq = {};
    10886  bool requiresDedicatedAllocation = false;
    10887  bool prefersDedicatedAllocation = false;
    10888  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
    10889  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10890 
    10891  res = allocator->AllocateMemory(
    10892  vkMemReq,
    10893  requiresDedicatedAllocation,
    10894  prefersDedicatedAllocation,
    10895  VK_NULL_HANDLE, // dedicatedBuffer
    10896  *pImage, // dedicatedImage
    10897  *pAllocationCreateInfo,
    10898  suballocType,
    10899  pAllocation);
    10900 
    10901 #if VMA_RECORDING_ENABLED
    10902  if(allocator->GetRecorder() != VMA_NULL)
    10903  {
    10904  allocator->GetRecorder()->RecordCreateImage(
    10905  allocator->GetCurrentFrameIndex(),
    10906  *pImageCreateInfo,
    10907  *pAllocationCreateInfo,
    10908  *pAllocation);
    10909  }
    10910 #endif
    10911 
    10912  if(res >= 0)
    10913  {
    10914  // 3. Bind image with memory.
    10915  res = allocator->BindImageMemory(*pAllocation, *pImage);
    10916  if(res >= 0)
    10917  {
    10918  // All steps succeeded.
    10919  #if VMA_STATS_STRING_ENABLED
    10920  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
    10921  #endif
    10922  if(pAllocationInfo != VMA_NULL)
    10923  {
    10924  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10925  }
    10926 
    10927  return VK_SUCCESS;
    10928  }
    10929  allocator->FreeMemory(*pAllocation);
    10930  *pAllocation = VK_NULL_HANDLE;
    10931  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    10932  *pImage = VK_NULL_HANDLE;
    10933  return res;
    10934  }
    10935  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    10936  *pImage = VK_NULL_HANDLE;
    10937  return res;
    10938  }
    10939  return res;
    10940 }
    10941 
    10942 void vmaDestroyImage(
    10943  VmaAllocator allocator,
    10944  VkImage image,
    10945  VmaAllocation allocation)
    10946 {
    10947  VMA_ASSERT(allocator);
    10948 
    10949  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    10950  {
    10951  return;
    10952  }
    10953 
    10954  VMA_DEBUG_LOG("vmaDestroyImage");
    10955 
    10956  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10957 
    10958 #if VMA_RECORDING_ENABLED
    10959  if(allocator->GetRecorder() != VMA_NULL)
    10960  {
    10961  allocator->GetRecorder()->RecordDestroyImage(
    10962  allocator->GetCurrentFrameIndex(),
    10963  allocation);
    10964  }
    10965 #endif
    10966 
    10967  if(image != VK_NULL_HANDLE)
    10968  {
    10969  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
    10970  }
    10971  if(allocation != VK_NULL_HANDLE)
    10972  {
    10973  allocator->FreeMemory(allocation);
    10974  }
    10975 }
    10976 
    10977 #endif // #ifdef VMA_IMPLEMENTATION
    VmaVulkanFunctions::vkGetPhysicalDeviceProperties
    PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
    Definition: vk_mem_alloc.h:1298
    -
    VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
    Set this flag if the allocation should have its own memory block.
    Definition: vk_mem_alloc.h:1611
    +Go to the documentation of this file.
    1 //
    2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
    3 //
    4 // Permission is hereby granted, free of charge, to any person obtaining a copy
    5 // of this software and associated documentation files (the "Software"), to deal
    6 // in the Software without restriction, including without limitation the rights
    7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    8 // copies of the Software, and to permit persons to whom the Software is
    9 // furnished to do so, subject to the following conditions:
    10 //
    11 // The above copyright notice and this permission notice shall be included in
    12 // all copies or substantial portions of the Software.
    13 //
    14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    20 // THE SOFTWARE.
    21 //
    22 
    23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
    24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
    25 
    26 #ifdef __cplusplus
    27 extern "C" {
    28 #endif
    29 
    1258 #include <vulkan/vulkan.h>
    1259 
    1260 #if !defined(VMA_DEDICATED_ALLOCATION)
    1261  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
    1262  #define VMA_DEDICATED_ALLOCATION 1
    1263  #else
    1264  #define VMA_DEDICATED_ALLOCATION 0
    1265  #endif
    1266 #endif
    1267 
    1277 VK_DEFINE_HANDLE(VmaAllocator)
    1278 
    1279 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
    1281  VmaAllocator allocator,
    1282  uint32_t memoryType,
    1283  VkDeviceMemory memory,
    1284  VkDeviceSize size);
    1286 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
    1287  VmaAllocator allocator,
    1288  uint32_t memoryType,
    1289  VkDeviceMemory memory,
    1290  VkDeviceSize size);
    1291 
    1299 typedef struct VmaDeviceMemoryCallbacks {
    1301  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
    1303  PFN_vmaFreeDeviceMemoryFunction pfnFree;
    1304 } VmaDeviceMemoryCallbacks;
    1305 
    1307 typedef enum VmaAllocatorCreateFlagBits {
    1312  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
    1334  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
    1335 
    1336  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1337 } VmaAllocatorCreateFlagBits;
    1338 typedef VkFlags VmaAllocatorCreateFlags;
    1339 
    1344 typedef struct VmaVulkanFunctions {
    1345  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
    1346  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
    1347  PFN_vkAllocateMemory vkAllocateMemory;
    1348  PFN_vkFreeMemory vkFreeMemory;
    1349  PFN_vkMapMemory vkMapMemory;
    1350  PFN_vkUnmapMemory vkUnmapMemory;
    1351  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
    1352  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
    1353  PFN_vkBindBufferMemory vkBindBufferMemory;
    1354  PFN_vkBindImageMemory vkBindImageMemory;
    1355  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
    1356  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
    1357  PFN_vkCreateBuffer vkCreateBuffer;
    1358  PFN_vkDestroyBuffer vkDestroyBuffer;
    1359  PFN_vkCreateImage vkCreateImage;
    1360  PFN_vkDestroyImage vkDestroyImage;
    1361 #if VMA_DEDICATED_ALLOCATION
    1362  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
    1363  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
    1364 #endif
    1365 } VmaVulkanFunctions;
    1366 
    1368 typedef enum VmaRecordFlagBits {
    1374  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
    1375 
    1376  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1377 } VmaRecordFlagBits;
    1378 typedef VkFlags VmaRecordFlags;
    1379 
    1380 /*
    1381 Define this macro to 0/1 to disable/enable support for recording functionality,
    1382 available through VmaAllocatorCreateInfo::pRecordSettings.
    1383 */
    1384 #ifndef VMA_RECORDING_ENABLED
    1385  #ifdef _WIN32
    1386  #define VMA_RECORDING_ENABLED 1
    1387  #else
    1388  #define VMA_RECORDING_ENABLED 0
    1389  #endif
    1390 #endif
    1391 
    1393 typedef struct VmaRecordSettings
    1394 {
    1396  VmaRecordFlags flags;
    1404  const char* pFilePath;
    1405 } VmaRecordSettings;
    1406 
    1408 typedef struct VmaAllocatorCreateInfo
    1409 {
    1411  VmaAllocatorCreateFlags flags;
    1413 
    1414  VkPhysicalDevice physicalDevice;
    1416 
    1417  VkDevice device;
    1419 
    1420  VkDeviceSize preferredLargeHeapBlockSize;
    1422 
    1423  const VkAllocationCallbacks* pAllocationCallbacks;
    1425 
    1426  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
    1440  uint32_t frameInUseCount;
    1464  const VkDeviceSize* pHeapSizeLimit;
    1476  const VmaVulkanFunctions* pVulkanFunctions;
    1483  const VmaRecordSettings* pRecordSettings;
    1484 } VmaAllocatorCreateInfo;
    1485 
    1487 VkResult vmaCreateAllocator(
    1488  const VmaAllocatorCreateInfo* pCreateInfo,
    1489  VmaAllocator* pAllocator);
    1490 
    1492 void vmaDestroyAllocator(
    1493  VmaAllocator allocator);
    1494 
    1499 void vmaGetPhysicalDeviceProperties(
    1500  VmaAllocator allocator,
    1501  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
    1502 
    1507 void vmaGetMemoryProperties(
    1508  VmaAllocator allocator,
    1509  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
    1510 
    1517 void vmaGetMemoryTypeProperties(
    1518  VmaAllocator allocator,
    1519  uint32_t memoryTypeIndex,
    1520  VkMemoryPropertyFlags* pFlags);
    1521 
    1530 void vmaSetCurrentFrameIndex(
    1531  VmaAllocator allocator,
    1532  uint32_t frameIndex);
    1533 
    1536 typedef struct VmaStatInfo
    1537 {
    1539  uint32_t blockCount;
    1541  uint32_t allocationCount;
    1543  uint32_t unusedRangeCount;
    1545  VkDeviceSize usedBytes;
    1547  VkDeviceSize unusedBytes;
    1548  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
    1549  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
    1550 } VmaStatInfo;
    1551 
    1553 typedef struct VmaStats
    1554 {
    1555  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
    1556  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
    1557  VmaStatInfo total;
    1558 } VmaStats;
    1559 
    1561 void vmaCalculateStats(
    1562  VmaAllocator allocator,
    1563  VmaStats* pStats);
    1564 
    1565 #define VMA_STATS_STRING_ENABLED 1
    1566 
    1567 #if VMA_STATS_STRING_ENABLED
    1568 
    1570 
    1572 void vmaBuildStatsString(
    1573  VmaAllocator allocator,
    1574  char** ppStatsString,
    1575  VkBool32 detailedMap);
    1576 
    1577 void vmaFreeStatsString(
    1578  VmaAllocator allocator,
    1579  char* pStatsString);
    1580 
    1581 #endif // #if VMA_STATS_STRING_ENABLED
    1582 
    1591 VK_DEFINE_HANDLE(VmaPool)
    1592 
    1593 typedef enum VmaMemoryUsage
    1594 {
    1598  VMA_MEMORY_USAGE_UNKNOWN = 0,
    1615  VMA_MEMORY_USAGE_GPU_ONLY = 1,
    1625  VMA_MEMORY_USAGE_CPU_ONLY = 2,
    1632  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
    1641  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
    1642  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
    1643 } VmaMemoryUsage;
    1644 
    1646 typedef enum VmaAllocationCreateFlagBits {
    1658  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
    1659 
    1669  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
    1682  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
    1695  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
    1702  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
    1708  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
    1709 
    1710  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1711 } VmaAllocationCreateFlagBits;
    1712 typedef VkFlags VmaAllocationCreateFlags;
    1713 
    1714 typedef struct VmaAllocationCreateInfo
    1715 {
    1717  VmaAllocationCreateFlags flags;
    1723  VmaMemoryUsage usage;
    1728  VkMemoryPropertyFlags requiredFlags;
    1733  VkMemoryPropertyFlags preferredFlags;
    1741  uint32_t memoryTypeBits;
    1747  VmaPool pool;
    1754  void* pUserData;
    1755 } VmaAllocationCreateInfo;
    1756 
    1773 VkResult vmaFindMemoryTypeIndex(
    1774  VmaAllocator allocator,
    1775  uint32_t memoryTypeBits,
    1776  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1777  uint32_t* pMemoryTypeIndex);
    1778 
    1791 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    1792  VmaAllocator allocator,
    1793  const VkBufferCreateInfo* pBufferCreateInfo,
    1794  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1795  uint32_t* pMemoryTypeIndex);
    1796 
    1809 VkResult vmaFindMemoryTypeIndexForImageInfo(
    1810  VmaAllocator allocator,
    1811  const VkImageCreateInfo* pImageCreateInfo,
    1812  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1813  uint32_t* pMemoryTypeIndex);
    1814 
    1816 typedef enum VmaPoolCreateFlagBits {
    1834  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
    1835 
    1836  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1837 } VmaPoolCreateFlagBits;
    1838 typedef VkFlags VmaPoolCreateFlags;
    1839 
    1842 typedef struct VmaPoolCreateInfo {
    1845  uint32_t memoryTypeIndex;
    1848  VmaPoolCreateFlags flags;
    1853  VkDeviceSize blockSize;
    1858  size_t minBlockCount;
    1866  size_t maxBlockCount;
    1880  uint32_t frameInUseCount;
    1881 } VmaPoolCreateInfo;
    1882 
    1885 typedef struct VmaPoolStats {
    1888  VkDeviceSize size;
    1891  VkDeviceSize unusedSize;
    1894  size_t allocationCount;
    1897  size_t unusedRangeCount;
    1904  VkDeviceSize unusedRangeSizeMax;
    1905 } VmaPoolStats;
    1906 
    1913 VkResult vmaCreatePool(
    1914  VmaAllocator allocator,
    1915  const VmaPoolCreateInfo* pCreateInfo,
    1916  VmaPool* pPool);
    1917 
    1920 void vmaDestroyPool(
    1921  VmaAllocator allocator,
    1922  VmaPool pool);
    1923 
    1930 void vmaGetPoolStats(
    1931  VmaAllocator allocator,
    1932  VmaPool pool,
    1933  VmaPoolStats* pPoolStats);
    1934 
    1941 void vmaMakePoolAllocationsLost(
    1942  VmaAllocator allocator,
    1943  VmaPool pool,
    1944  size_t* pLostAllocationCount);
    1945 
    1960 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
    1961 
    1986 VK_DEFINE_HANDLE(VmaAllocation)
    1987 
    1988 
    1990 typedef struct VmaAllocationInfo {
    1995  uint32_t memoryType;
    2004  VkDeviceMemory deviceMemory;
    2009  VkDeviceSize offset;
    2014  VkDeviceSize size;
    2023  void* pMappedData;
    2028  void* pUserData;
    2029 } VmaAllocationInfo;
    2030 
    2041 VkResult vmaAllocateMemory(
    2042  VmaAllocator allocator,
    2043  const VkMemoryRequirements* pVkMemoryRequirements,
    2044  const VmaAllocationCreateInfo* pCreateInfo,
    2045  VmaAllocation* pAllocation,
    2046  VmaAllocationInfo* pAllocationInfo);
    2047 
    2054 VkResult vmaAllocateMemoryForBuffer(
    2055  VmaAllocator allocator,
    2056  VkBuffer buffer,
    2057  const VmaAllocationCreateInfo* pCreateInfo,
    2058  VmaAllocation* pAllocation,
    2059  VmaAllocationInfo* pAllocationInfo);
    2060 
    2062 VkResult vmaAllocateMemoryForImage(
    2063  VmaAllocator allocator,
    2064  VkImage image,
    2065  const VmaAllocationCreateInfo* pCreateInfo,
    2066  VmaAllocation* pAllocation,
    2067  VmaAllocationInfo* pAllocationInfo);
    2068 
    2070 void vmaFreeMemory(
    2071  VmaAllocator allocator,
    2072  VmaAllocation allocation);
    2073 
    2090 void vmaGetAllocationInfo(
    2091  VmaAllocator allocator,
    2092  VmaAllocation allocation,
    2093  VmaAllocationInfo* pAllocationInfo);
    2094 
    2109 VkBool32 vmaTouchAllocation(
    2110  VmaAllocator allocator,
    2111  VmaAllocation allocation);
    2112 
    2126 void vmaSetAllocationUserData(
    2127  VmaAllocator allocator,
    2128  VmaAllocation allocation,
    2129  void* pUserData);
    2130 
    2141 void vmaCreateLostAllocation(
    2142  VmaAllocator allocator,
    2143  VmaAllocation* pAllocation);
    2144 
    2179 VkResult vmaMapMemory(
    2180  VmaAllocator allocator,
    2181  VmaAllocation allocation,
    2182  void** ppData);
    2183 
    2188 void vmaUnmapMemory(
    2189  VmaAllocator allocator,
    2190  VmaAllocation allocation);
    2191 
    2204 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2205 
    2218 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2219 
    2236 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
    2237 
    2239 typedef struct VmaDefragmentationInfo {
    2244  VkDeviceSize maxBytesToMove;
    2249  uint32_t maxAllocationsToMove;
    2250 } VmaDefragmentationInfo;
    2251 
    2253 typedef struct VmaDefragmentationStats {
    2255  VkDeviceSize bytesMoved;
    2257  VkDeviceSize bytesFreed;
    2259  uint32_t allocationsMoved;
    2261  uint32_t deviceMemoryBlocksFreed;
    2262 } VmaDefragmentationStats;
    2263 
    2346 VkResult vmaDefragment(
    2347  VmaAllocator allocator,
    2348  VmaAllocation* pAllocations,
    2349  size_t allocationCount,
    2350  VkBool32* pAllocationsChanged,
    2351  const VmaDefragmentationInfo *pDefragmentationInfo,
    2352  VmaDefragmentationStats* pDefragmentationStats);
    2353 
    2366 VkResult vmaBindBufferMemory(
    2367  VmaAllocator allocator,
    2368  VmaAllocation allocation,
    2369  VkBuffer buffer);
    2370 
    2383 VkResult vmaBindImageMemory(
    2384  VmaAllocator allocator,
    2385  VmaAllocation allocation,
    2386  VkImage image);
    2387 
    2414 VkResult vmaCreateBuffer(
    2415  VmaAllocator allocator,
    2416  const VkBufferCreateInfo* pBufferCreateInfo,
    2417  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2418  VkBuffer* pBuffer,
    2419  VmaAllocation* pAllocation,
    2420  VmaAllocationInfo* pAllocationInfo);
    2421 
    2433 void vmaDestroyBuffer(
    2434  VmaAllocator allocator,
    2435  VkBuffer buffer,
    2436  VmaAllocation allocation);
    2437 
    2439 VkResult vmaCreateImage(
    2440  VmaAllocator allocator,
    2441  const VkImageCreateInfo* pImageCreateInfo,
    2442  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2443  VkImage* pImage,
    2444  VmaAllocation* pAllocation,
    2445  VmaAllocationInfo* pAllocationInfo);
    2446 
    2458 void vmaDestroyImage(
    2459  VmaAllocator allocator,
    2460  VkImage image,
    2461  VmaAllocation allocation);
    2462 
    2463 #ifdef __cplusplus
    2464 }
    2465 #endif
    2466 
    2467 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
    2468 
    2469 // For Visual Studio IntelliSense.
    2470 #if defined(__cplusplus) && defined(__INTELLISENSE__)
    2471 #define VMA_IMPLEMENTATION
    2472 #endif
    2473 
    2474 #ifdef VMA_IMPLEMENTATION
    2475 #undef VMA_IMPLEMENTATION
    2476 
    2477 #include <cstdint>
    2478 #include <cstdlib>
    2479 #include <cstring>
    2480 
    2481 /*******************************************************************************
    2482 CONFIGURATION SECTION
    2483 
    2484 Define some of these macros before each #include of this header or change them
    2485 here if you need other then default behavior depending on your environment.
    2486 */
    2487 
    2488 /*
    2489 Define this macro to 1 to make the library fetch pointers to Vulkan functions
    2490 internally, like:
    2491 
    2492  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    2493 
    2494 Define to 0 if you are going to provide you own pointers to Vulkan functions via
    2495 VmaAllocatorCreateInfo::pVulkanFunctions.
    2496 */
    2497 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
    2498 #define VMA_STATIC_VULKAN_FUNCTIONS 1
    2499 #endif
    2500 
    2501 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
    2502 //#define VMA_USE_STL_CONTAINERS 1
    2503 
    2504 /* Set this macro to 1 to make the library including and using STL containers:
    2505 std::pair, std::vector, std::list, std::unordered_map.
    2506 
    2507 Set it to 0 or undefined to make the library using its own implementation of
    2508 the containers.
    2509 */
    2510 #if VMA_USE_STL_CONTAINERS
    2511  #define VMA_USE_STL_VECTOR 1
    2512  #define VMA_USE_STL_UNORDERED_MAP 1
    2513  #define VMA_USE_STL_LIST 1
    2514 #endif
    2515 
    2516 #if VMA_USE_STL_VECTOR
    2517  #include <vector>
    2518 #endif
    2519 
    2520 #if VMA_USE_STL_UNORDERED_MAP
    2521  #include <unordered_map>
    2522 #endif
    2523 
    2524 #if VMA_USE_STL_LIST
    2525  #include <list>
    2526 #endif
    2527 
    2528 /*
    2529 Following headers are used in this CONFIGURATION section only, so feel free to
    2530 remove them if not needed.
    2531 */
    2532 #include <cassert> // for assert
    2533 #include <algorithm> // for min, max
    2534 #include <mutex> // for std::mutex
    2535 #include <atomic> // for std::atomic
    2536 
    2537 #ifndef VMA_NULL
    2538  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
    2539  #define VMA_NULL nullptr
    2540 #endif
    2541 
    2542 #if defined(__APPLE__) || defined(__ANDROID__)
    2543 #include <cstdlib>
    2544 void *aligned_alloc(size_t alignment, size_t size)
    2545 {
    2546  // alignment must be >= sizeof(void*)
    2547  if(alignment < sizeof(void*))
    2548  {
    2549  alignment = sizeof(void*);
    2550  }
    2551 
    2552  void *pointer;
    2553  if(posix_memalign(&pointer, alignment, size) == 0)
    2554  return pointer;
    2555  return VMA_NULL;
    2556 }
    2557 #endif
    2558 
    2559 // If your compiler is not compatible with C++11 and definition of
    2560 // aligned_alloc() function is missing, uncommeting following line may help:
    2561 
    2562 //#include <malloc.h>
    2563 
    2564 // Normal assert to check for programmer's errors, especially in Debug configuration.
    2565 #ifndef VMA_ASSERT
    2566  #ifdef _DEBUG
    2567  #define VMA_ASSERT(expr) assert(expr)
    2568  #else
    2569  #define VMA_ASSERT(expr)
    2570  #endif
    2571 #endif
    2572 
    2573 // Assert that will be called very often, like inside data structures e.g. operator[].
    2574 // Making it non-empty can make program slow.
    2575 #ifndef VMA_HEAVY_ASSERT
    2576  #ifdef _DEBUG
    2577  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
    2578  #else
    2579  #define VMA_HEAVY_ASSERT(expr)
    2580  #endif
    2581 #endif
    2582 
    2583 #ifndef VMA_ALIGN_OF
    2584  #define VMA_ALIGN_OF(type) (__alignof(type))
    2585 #endif
    2586 
    2587 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
    2588  #if defined(_WIN32)
    2589  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
    2590  #else
    2591  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
    2592  #endif
    2593 #endif
    2594 
    2595 #ifndef VMA_SYSTEM_FREE
    2596  #if defined(_WIN32)
    2597  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
    2598  #else
    2599  #define VMA_SYSTEM_FREE(ptr) free(ptr)
    2600  #endif
    2601 #endif
    2602 
    2603 #ifndef VMA_MIN
    2604  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
    2605 #endif
    2606 
    2607 #ifndef VMA_MAX
    2608  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
    2609 #endif
    2610 
    2611 #ifndef VMA_SWAP
    2612  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
    2613 #endif
    2614 
    2615 #ifndef VMA_SORT
    2616  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
    2617 #endif
    2618 
    2619 #ifndef VMA_DEBUG_LOG
    2620  #define VMA_DEBUG_LOG(format, ...)
    2621  /*
    2622  #define VMA_DEBUG_LOG(format, ...) do { \
    2623  printf(format, __VA_ARGS__); \
    2624  printf("\n"); \
    2625  } while(false)
    2626  */
    2627 #endif
    2628 
    2629 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
    2630 #if VMA_STATS_STRING_ENABLED
    2631  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
    2632  {
    2633  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
    2634  }
    2635  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
    2636  {
    2637  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
    2638  }
    2639  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
    2640  {
    2641  snprintf(outStr, strLen, "%p", ptr);
    2642  }
    2643 #endif
    2644 
    2645 #ifndef VMA_MUTEX
    2646  class VmaMutex
    2647  {
    2648  public:
    2649  VmaMutex() { }
    2650  ~VmaMutex() { }
    2651  void Lock() { m_Mutex.lock(); }
    2652  void Unlock() { m_Mutex.unlock(); }
    2653  private:
    2654  std::mutex m_Mutex;
    2655  };
    2656  #define VMA_MUTEX VmaMutex
    2657 #endif
    2658 
    2659 /*
    2660 If providing your own implementation, you need to implement a subset of std::atomic:
    2661 
    2662 - Constructor(uint32_t desired)
    2663 - uint32_t load() const
    2664 - void store(uint32_t desired)
    2665 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
    2666 */
    2667 #ifndef VMA_ATOMIC_UINT32
    2668  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
    2669 #endif
    2670 
    2671 #ifndef VMA_BEST_FIT
    2672 
    2684  #define VMA_BEST_FIT (1)
    2685 #endif
    2686 
    2687 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
    2688 
    2692  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
    2693 #endif
    2694 
    2695 #ifndef VMA_DEBUG_ALIGNMENT
    2696 
    2700  #define VMA_DEBUG_ALIGNMENT (1)
    2701 #endif
    2702 
    2703 #ifndef VMA_DEBUG_MARGIN
    2704 
    2708  #define VMA_DEBUG_MARGIN (0)
    2709 #endif
    2710 
    2711 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
    2712 
    2716  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
    2717 #endif
    2718 
    2719 #ifndef VMA_DEBUG_DETECT_CORRUPTION
    2720 
    2725  #define VMA_DEBUG_DETECT_CORRUPTION (0)
    2726 #endif
    2727 
    2728 #ifndef VMA_DEBUG_GLOBAL_MUTEX
    2729 
    2733  #define VMA_DEBUG_GLOBAL_MUTEX (0)
    2734 #endif
    2735 
    2736 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
    2737 
    2741  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
    2742 #endif
    2743 
    2744 #ifndef VMA_SMALL_HEAP_MAX_SIZE
    2745  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
    2747 #endif
    2748 
    2749 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
    2750  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
    2752 #endif
    2753 
    2754 #ifndef VMA_CLASS_NO_COPY
    2755  #define VMA_CLASS_NO_COPY(className) \
    2756  private: \
    2757  className(const className&) = delete; \
    2758  className& operator=(const className&) = delete;
    2759 #endif
    2760 
    2761 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
    2762 
    2763 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
    2764 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
    2765 
    2766 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
    2767 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
    2768 
    2769 /*******************************************************************************
    2770 END OF CONFIGURATION
    2771 */
    2772 
    2773 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
    2774  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
    2775 
    2776 // Returns number of bits set to 1 in (v).
    2777 static inline uint32_t VmaCountBitsSet(uint32_t v)
    2778 {
    2779  uint32_t c = v - ((v >> 1) & 0x55555555);
    2780  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
    2781  c = ((c >> 4) + c) & 0x0F0F0F0F;
    2782  c = ((c >> 8) + c) & 0x00FF00FF;
    2783  c = ((c >> 16) + c) & 0x0000FFFF;
    2784  return c;
    2785 }
    2786 
    2787 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
    2788 // Use types like uint32_t, uint64_t as T.
    2789 template <typename T>
    2790 static inline T VmaAlignUp(T val, T align)
    2791 {
    2792  return (val + align - 1) / align * align;
    2793 }
    2794 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
    2795 // Use types like uint32_t, uint64_t as T.
    2796 template <typename T>
    2797 static inline T VmaAlignDown(T val, T align)
    2798 {
    2799  return val / align * align;
    2800 }
    2801 
    2802 // Division with mathematical rounding to nearest number.
    2803 template <typename T>
    2804 inline T VmaRoundDiv(T x, T y)
    2805 {
    2806  return (x + (y / (T)2)) / y;
    2807 }
    2808 
    2809 static inline bool VmaStrIsEmpty(const char* pStr)
    2810 {
    2811  return pStr == VMA_NULL || *pStr == '\0';
    2812 }
    2813 
    2814 #ifndef VMA_SORT
    2815 
    2816 template<typename Iterator, typename Compare>
    2817 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
    2818 {
    2819  Iterator centerValue = end; --centerValue;
    2820  Iterator insertIndex = beg;
    2821  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
    2822  {
    2823  if(cmp(*memTypeIndex, *centerValue))
    2824  {
    2825  if(insertIndex != memTypeIndex)
    2826  {
    2827  VMA_SWAP(*memTypeIndex, *insertIndex);
    2828  }
    2829  ++insertIndex;
    2830  }
    2831  }
    2832  if(insertIndex != centerValue)
    2833  {
    2834  VMA_SWAP(*insertIndex, *centerValue);
    2835  }
    2836  return insertIndex;
    2837 }
    2838 
    2839 template<typename Iterator, typename Compare>
    2840 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
    2841 {
    2842  if(beg < end)
    2843  {
    2844  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
    2845  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
    2846  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
    2847  }
    2848 }
    2849 
    2850 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
    2851 
    2852 #endif // #ifndef VMA_SORT
    2853 
    2854 /*
    2855 Returns true if two memory blocks occupy overlapping pages.
    2856 ResourceA must be in less memory offset than ResourceB.
    2857 
    2858 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
    2859 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
    2860 */
    2861 static inline bool VmaBlocksOnSamePage(
    2862  VkDeviceSize resourceAOffset,
    2863  VkDeviceSize resourceASize,
    2864  VkDeviceSize resourceBOffset,
    2865  VkDeviceSize pageSize)
    2866 {
    2867  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
    2868  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
    2869  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
    2870  VkDeviceSize resourceBStart = resourceBOffset;
    2871  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
    2872  return resourceAEndPage == resourceBStartPage;
    2873 }
    2874 
    2875 enum VmaSuballocationType
    2876 {
    2877  VMA_SUBALLOCATION_TYPE_FREE = 0,
    2878  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
    2879  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
    2880  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
    2881  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
    2882  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
    2883  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
    2884 };
    2885 
    2886 /*
    2887 Returns true if given suballocation types could conflict and must respect
    2888 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
    2889 or linear image and another one is optimal image. If type is unknown, behave
    2890 conservatively.
    2891 */
    2892 static inline bool VmaIsBufferImageGranularityConflict(
    2893  VmaSuballocationType suballocType1,
    2894  VmaSuballocationType suballocType2)
    2895 {
    2896  if(suballocType1 > suballocType2)
    2897  {
    2898  VMA_SWAP(suballocType1, suballocType2);
    2899  }
    2900 
    2901  switch(suballocType1)
    2902  {
    2903  case VMA_SUBALLOCATION_TYPE_FREE:
    2904  return false;
    2905  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
    2906  return true;
    2907  case VMA_SUBALLOCATION_TYPE_BUFFER:
    2908  return
    2909  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2910  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2911  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
    2912  return
    2913  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2914  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
    2915  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2916  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
    2917  return
    2918  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2919  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
    2920  return false;
    2921  default:
    2922  VMA_ASSERT(0);
    2923  return true;
    2924  }
    2925 }
    2926 
    2927 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
    2928 {
    2929  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
    2930  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2931  for(size_t i = 0; i < numberCount; ++i, ++pDst)
    2932  {
    2933  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
    2934  }
    2935 }
    2936 
    2937 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
    2938 {
    2939  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
    2940  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2941  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
    2942  {
    2943  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
    2944  {
    2945  return false;
    2946  }
    2947  }
    2948  return true;
    2949 }
    2950 
    2951 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
    2952 struct VmaMutexLock
    2953 {
    2954  VMA_CLASS_NO_COPY(VmaMutexLock)
    2955 public:
    2956  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
    2957  m_pMutex(useMutex ? &mutex : VMA_NULL)
    2958  {
    2959  if(m_pMutex)
    2960  {
    2961  m_pMutex->Lock();
    2962  }
    2963  }
    2964 
    2965  ~VmaMutexLock()
    2966  {
    2967  if(m_pMutex)
    2968  {
    2969  m_pMutex->Unlock();
    2970  }
    2971  }
    2972 
    2973 private:
    2974  VMA_MUTEX* m_pMutex;
    2975 };
    2976 
    2977 #if VMA_DEBUG_GLOBAL_MUTEX
    2978  static VMA_MUTEX gDebugGlobalMutex;
    2979  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
    2980 #else
    2981  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
    2982 #endif
    2983 
    2984 // Minimum size of a free suballocation to register it in the free suballocation collection.
    2985 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
    2986 
    2987 /*
    2988 Performs binary search and returns iterator to first element that is greater or
    2989 equal to (key), according to comparison (cmp).
    2990 
    2991 Cmp should return true if first argument is less than second argument.
    2992 
    2993 Returned value is the found element, if present in the collection or place where
    2994 new element with value (key) should be inserted.
    2995 */
    2996 template <typename IterT, typename KeyT, typename CmpT>
    2997 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
    2998 {
    2999  size_t down = 0, up = (end - beg);
    3000  while(down < up)
    3001  {
    3002  const size_t mid = (down + up) / 2;
    3003  if(cmp(*(beg+mid), key))
    3004  {
    3005  down = mid + 1;
    3006  }
    3007  else
    3008  {
    3009  up = mid;
    3010  }
    3011  }
    3012  return beg + down;
    3013 }
    3014 
    3016 // Memory allocation
    3017 
    3018 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
    3019 {
    3020  if((pAllocationCallbacks != VMA_NULL) &&
    3021  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
    3022  {
    3023  return (*pAllocationCallbacks->pfnAllocation)(
    3024  pAllocationCallbacks->pUserData,
    3025  size,
    3026  alignment,
    3027  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    3028  }
    3029  else
    3030  {
    3031  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
    3032  }
    3033 }
    3034 
    3035 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
    3036 {
    3037  if((pAllocationCallbacks != VMA_NULL) &&
    3038  (pAllocationCallbacks->pfnFree != VMA_NULL))
    3039  {
    3040  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
    3041  }
    3042  else
    3043  {
    3044  VMA_SYSTEM_FREE(ptr);
    3045  }
    3046 }
    3047 
    3048 template<typename T>
    3049 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
    3050 {
    3051  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
    3052 }
    3053 
    3054 template<typename T>
    3055 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
    3056 {
    3057  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
    3058 }
    3059 
    3060 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
    3061 
    3062 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
    3063 
    3064 template<typename T>
    3065 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
    3066 {
    3067  ptr->~T();
    3068  VmaFree(pAllocationCallbacks, ptr);
    3069 }
    3070 
    3071 template<typename T>
    3072 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
    3073 {
    3074  if(ptr != VMA_NULL)
    3075  {
    3076  for(size_t i = count; i--; )
    3077  {
    3078  ptr[i].~T();
    3079  }
    3080  VmaFree(pAllocationCallbacks, ptr);
    3081  }
    3082 }
    3083 
    3084 // STL-compatible allocator.
    3085 template<typename T>
    3086 class VmaStlAllocator
    3087 {
    3088 public:
    3089  const VkAllocationCallbacks* const m_pCallbacks;
    3090  typedef T value_type;
    3091 
    3092  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
    3093  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
    3094 
    3095  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
    3096  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
    3097 
    3098  template<typename U>
    3099  bool operator==(const VmaStlAllocator<U>& rhs) const
    3100  {
    3101  return m_pCallbacks == rhs.m_pCallbacks;
    3102  }
    3103  template<typename U>
    3104  bool operator!=(const VmaStlAllocator<U>& rhs) const
    3105  {
    3106  return m_pCallbacks != rhs.m_pCallbacks;
    3107  }
    3108 
    3109  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
    3110 };
    3111 
    3112 #if VMA_USE_STL_VECTOR
    3113 
    3114 #define VmaVector std::vector
    3115 
    3116 template<typename T, typename allocatorT>
    3117 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
    3118 {
    3119  vec.insert(vec.begin() + index, item);
    3120 }
    3121 
    3122 template<typename T, typename allocatorT>
    3123 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
    3124 {
    3125  vec.erase(vec.begin() + index);
    3126 }
    3127 
    3128 #else // #if VMA_USE_STL_VECTOR
    3129 
    3130 /* Class with interface compatible with subset of std::vector.
    3131 T must be POD because constructors and destructors are not called and memcpy is
    3132 used for these objects. */
    3133 template<typename T, typename AllocatorT>
    3134 class VmaVector
    3135 {
    3136 public:
    3137  typedef T value_type;
    3138 
    3139  VmaVector(const AllocatorT& allocator) :
    3140  m_Allocator(allocator),
    3141  m_pArray(VMA_NULL),
    3142  m_Count(0),
    3143  m_Capacity(0)
    3144  {
    3145  }
    3146 
    3147  VmaVector(size_t count, const AllocatorT& allocator) :
    3148  m_Allocator(allocator),
    3149  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
    3150  m_Count(count),
    3151  m_Capacity(count)
    3152  {
    3153  }
    3154 
    3155  VmaVector(const VmaVector<T, AllocatorT>& src) :
    3156  m_Allocator(src.m_Allocator),
    3157  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
    3158  m_Count(src.m_Count),
    3159  m_Capacity(src.m_Count)
    3160  {
    3161  if(m_Count != 0)
    3162  {
    3163  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
    3164  }
    3165  }
    3166 
    3167  ~VmaVector()
    3168  {
    3169  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3170  }
    3171 
    3172  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
    3173  {
    3174  if(&rhs != this)
    3175  {
    3176  resize(rhs.m_Count);
    3177  if(m_Count != 0)
    3178  {
    3179  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
    3180  }
    3181  }
    3182  return *this;
    3183  }
    3184 
    3185  bool empty() const { return m_Count == 0; }
    3186  size_t size() const { return m_Count; }
    3187  T* data() { return m_pArray; }
    3188  const T* data() const { return m_pArray; }
    3189 
    3190  T& operator[](size_t index)
    3191  {
    3192  VMA_HEAVY_ASSERT(index < m_Count);
    3193  return m_pArray[index];
    3194  }
    3195  const T& operator[](size_t index) const
    3196  {
    3197  VMA_HEAVY_ASSERT(index < m_Count);
    3198  return m_pArray[index];
    3199  }
    3200 
    3201  T& front()
    3202  {
    3203  VMA_HEAVY_ASSERT(m_Count > 0);
    3204  return m_pArray[0];
    3205  }
    3206  const T& front() const
    3207  {
    3208  VMA_HEAVY_ASSERT(m_Count > 0);
    3209  return m_pArray[0];
    3210  }
    3211  T& back()
    3212  {
    3213  VMA_HEAVY_ASSERT(m_Count > 0);
    3214  return m_pArray[m_Count - 1];
    3215  }
    3216  const T& back() const
    3217  {
    3218  VMA_HEAVY_ASSERT(m_Count > 0);
    3219  return m_pArray[m_Count - 1];
    3220  }
    3221 
    3222  void reserve(size_t newCapacity, bool freeMemory = false)
    3223  {
    3224  newCapacity = VMA_MAX(newCapacity, m_Count);
    3225 
    3226  if((newCapacity < m_Capacity) && !freeMemory)
    3227  {
    3228  newCapacity = m_Capacity;
    3229  }
    3230 
    3231  if(newCapacity != m_Capacity)
    3232  {
    3233  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
    3234  if(m_Count != 0)
    3235  {
    3236  memcpy(newArray, m_pArray, m_Count * sizeof(T));
    3237  }
    3238  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3239  m_Capacity = newCapacity;
    3240  m_pArray = newArray;
    3241  }
    3242  }
    3243 
    3244  void resize(size_t newCount, bool freeMemory = false)
    3245  {
    3246  size_t newCapacity = m_Capacity;
    3247  if(newCount > m_Capacity)
    3248  {
    3249  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
    3250  }
    3251  else if(freeMemory)
    3252  {
    3253  newCapacity = newCount;
    3254  }
    3255 
    3256  if(newCapacity != m_Capacity)
    3257  {
    3258  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
    3259  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
    3260  if(elementsToCopy != 0)
    3261  {
    3262  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
    3263  }
    3264  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3265  m_Capacity = newCapacity;
    3266  m_pArray = newArray;
    3267  }
    3268 
    3269  m_Count = newCount;
    3270  }
    3271 
    3272  void clear(bool freeMemory = false)
    3273  {
    3274  resize(0, freeMemory);
    3275  }
    3276 
    3277  void insert(size_t index, const T& src)
    3278  {
    3279  VMA_HEAVY_ASSERT(index <= m_Count);
    3280  const size_t oldCount = size();
    3281  resize(oldCount + 1);
    3282  if(index < oldCount)
    3283  {
    3284  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
    3285  }
    3286  m_pArray[index] = src;
    3287  }
    3288 
    3289  void remove(size_t index)
    3290  {
    3291  VMA_HEAVY_ASSERT(index < m_Count);
    3292  const size_t oldCount = size();
    3293  if(index < oldCount - 1)
    3294  {
    3295  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
    3296  }
    3297  resize(oldCount - 1);
    3298  }
    3299 
    3300  void push_back(const T& src)
    3301  {
    3302  const size_t newIndex = size();
    3303  resize(newIndex + 1);
    3304  m_pArray[newIndex] = src;
    3305  }
    3306 
    3307  void pop_back()
    3308  {
    3309  VMA_HEAVY_ASSERT(m_Count > 0);
    3310  resize(size() - 1);
    3311  }
    3312 
    3313  void push_front(const T& src)
    3314  {
    3315  insert(0, src);
    3316  }
    3317 
    3318  void pop_front()
    3319  {
    3320  VMA_HEAVY_ASSERT(m_Count > 0);
    3321  remove(0);
    3322  }
    3323 
    3324  typedef T* iterator;
    3325 
    3326  iterator begin() { return m_pArray; }
    3327  iterator end() { return m_pArray + m_Count; }
    3328 
    3329 private:
    3330  AllocatorT m_Allocator;
    3331  T* m_pArray;
    3332  size_t m_Count;
    3333  size_t m_Capacity;
    3334 };
    3335 
    3336 template<typename T, typename allocatorT>
    3337 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
    3338 {
    3339  vec.insert(index, item);
    3340 }
    3341 
    3342 template<typename T, typename allocatorT>
    3343 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
    3344 {
    3345  vec.remove(index);
    3346 }
    3347 
    3348 #endif // #if VMA_USE_STL_VECTOR
    3349 
    3350 template<typename CmpLess, typename VectorT>
    3351 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
    3352 {
    3353  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    3354  vector.data(),
    3355  vector.data() + vector.size(),
    3356  value,
    3357  CmpLess()) - vector.data();
    3358  VmaVectorInsert(vector, indexToInsert, value);
    3359  return indexToInsert;
    3360 }
    3361 
    3362 template<typename CmpLess, typename VectorT>
    3363 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
    3364 {
    3365  CmpLess comparator;
    3366  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3367  vector.begin(),
    3368  vector.end(),
    3369  value,
    3370  comparator);
    3371  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
    3372  {
    3373  size_t indexToRemove = it - vector.begin();
    3374  VmaVectorRemove(vector, indexToRemove);
    3375  return true;
    3376  }
    3377  return false;
    3378 }
    3379 
    3380 template<typename CmpLess, typename VectorT>
    3381 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
    3382 {
    3383  CmpLess comparator;
    3384  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3385  vector.data(),
    3386  vector.data() + vector.size(),
    3387  value,
    3388  comparator);
    3389  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
    3390  {
    3391  return it - vector.begin();
    3392  }
    3393  else
    3394  {
    3395  return vector.size();
    3396  }
    3397 }
    3398 
    3400 // class VmaPoolAllocator
    3401 
    3402 /*
    3403 Allocator for objects of type T using a list of arrays (pools) to speed up
    3404 allocation. Number of elements that can be allocated is not bounded because
    3405 allocator can create multiple blocks.
    3406 */
    3407 template<typename T>
    3408 class VmaPoolAllocator
    3409 {
    3410  VMA_CLASS_NO_COPY(VmaPoolAllocator)
    3411 public:
    3412  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
    3413  ~VmaPoolAllocator();
    3414  void Clear();
    3415  T* Alloc();
    3416  void Free(T* ptr);
    3417 
    3418 private:
    3419  union Item
    3420  {
    3421  uint32_t NextFreeIndex;
    3422  T Value;
    3423  };
    3424 
    3425  struct ItemBlock
    3426  {
    3427  Item* pItems;
    3428  uint32_t FirstFreeIndex;
    3429  };
    3430 
    3431  const VkAllocationCallbacks* m_pAllocationCallbacks;
    3432  size_t m_ItemsPerBlock;
    3433  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
    3434 
    3435  ItemBlock& CreateNewBlock();
    3436 };
    3437 
    3438 template<typename T>
    3439 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
    3440  m_pAllocationCallbacks(pAllocationCallbacks),
    3441  m_ItemsPerBlock(itemsPerBlock),
    3442  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
    3443 {
    3444  VMA_ASSERT(itemsPerBlock > 0);
    3445 }
    3446 
    3447 template<typename T>
    3448 VmaPoolAllocator<T>::~VmaPoolAllocator()
    3449 {
    3450  Clear();
    3451 }
    3452 
    3453 template<typename T>
    3454 void VmaPoolAllocator<T>::Clear()
    3455 {
    3456  for(size_t i = m_ItemBlocks.size(); i--; )
    3457  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
    3458  m_ItemBlocks.clear();
    3459 }
    3460 
    3461 template<typename T>
    3462 T* VmaPoolAllocator<T>::Alloc()
    3463 {
    3464  for(size_t i = m_ItemBlocks.size(); i--; )
    3465  {
    3466  ItemBlock& block = m_ItemBlocks[i];
    3467  // This block has some free items: Use first one.
    3468  if(block.FirstFreeIndex != UINT32_MAX)
    3469  {
    3470  Item* const pItem = &block.pItems[block.FirstFreeIndex];
    3471  block.FirstFreeIndex = pItem->NextFreeIndex;
    3472  return &pItem->Value;
    3473  }
    3474  }
    3475 
    3476  // No block has free item: Create new one and use it.
    3477  ItemBlock& newBlock = CreateNewBlock();
    3478  Item* const pItem = &newBlock.pItems[0];
    3479  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
    3480  return &pItem->Value;
    3481 }
    3482 
    3483 template<typename T>
    3484 void VmaPoolAllocator<T>::Free(T* ptr)
    3485 {
    3486  // Search all memory blocks to find ptr.
    3487  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
    3488  {
    3489  ItemBlock& block = m_ItemBlocks[i];
    3490 
    3491  // Casting to union.
    3492  Item* pItemPtr;
    3493  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
    3494 
    3495  // Check if pItemPtr is in address range of this block.
    3496  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
    3497  {
    3498  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
    3499  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
    3500  block.FirstFreeIndex = index;
    3501  return;
    3502  }
    3503  }
    3504  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
    3505 }
    3506 
    3507 template<typename T>
    3508 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
    3509 {
    3510  ItemBlock newBlock = {
    3511  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
    3512 
    3513  m_ItemBlocks.push_back(newBlock);
    3514 
    3515  // Setup singly-linked list of all free items in this block.
    3516  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
    3517  newBlock.pItems[i].NextFreeIndex = i + 1;
    3518  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
    3519  return m_ItemBlocks.back();
    3520 }
    3521 
    3523 // class VmaRawList, VmaList
    3524 
    3525 #if VMA_USE_STL_LIST
    3526 
    3527 #define VmaList std::list
    3528 
    3529 #else // #if VMA_USE_STL_LIST
    3530 
    3531 template<typename T>
    3532 struct VmaListItem
    3533 {
    3534  VmaListItem* pPrev;
    3535  VmaListItem* pNext;
    3536  T Value;
    3537 };
    3538 
    3539 // Doubly linked list.
    3540 template<typename T>
    3541 class VmaRawList
    3542 {
    3543  VMA_CLASS_NO_COPY(VmaRawList)
    3544 public:
    3545  typedef VmaListItem<T> ItemType;
    3546 
    3547  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
    3548  ~VmaRawList();
    3549  void Clear();
    3550 
    3551  size_t GetCount() const { return m_Count; }
    3552  bool IsEmpty() const { return m_Count == 0; }
    3553 
    3554  ItemType* Front() { return m_pFront; }
    3555  const ItemType* Front() const { return m_pFront; }
    3556  ItemType* Back() { return m_pBack; }
    3557  const ItemType* Back() const { return m_pBack; }
    3558 
    3559  ItemType* PushBack();
    3560  ItemType* PushFront();
    3561  ItemType* PushBack(const T& value);
    3562  ItemType* PushFront(const T& value);
    3563  void PopBack();
    3564  void PopFront();
    3565 
    3566  // Item can be null - it means PushBack.
    3567  ItemType* InsertBefore(ItemType* pItem);
    3568  // Item can be null - it means PushFront.
    3569  ItemType* InsertAfter(ItemType* pItem);
    3570 
    3571  ItemType* InsertBefore(ItemType* pItem, const T& value);
    3572  ItemType* InsertAfter(ItemType* pItem, const T& value);
    3573 
    3574  void Remove(ItemType* pItem);
    3575 
    3576 private:
    3577  const VkAllocationCallbacks* const m_pAllocationCallbacks;
    3578  VmaPoolAllocator<ItemType> m_ItemAllocator;
    3579  ItemType* m_pFront;
    3580  ItemType* m_pBack;
    3581  size_t m_Count;
    3582 };
    3583 
    3584 template<typename T>
    3585 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
    3586  m_pAllocationCallbacks(pAllocationCallbacks),
    3587  m_ItemAllocator(pAllocationCallbacks, 128),
    3588  m_pFront(VMA_NULL),
    3589  m_pBack(VMA_NULL),
    3590  m_Count(0)
    3591 {
    3592 }
    3593 
    3594 template<typename T>
    3595 VmaRawList<T>::~VmaRawList()
    3596 {
    3597  // Intentionally not calling Clear, because that would be unnecessary
    3598  // computations to return all items to m_ItemAllocator as free.
    3599 }
    3600 
    3601 template<typename T>
    3602 void VmaRawList<T>::Clear()
    3603 {
    3604  if(IsEmpty() == false)
    3605  {
    3606  ItemType* pItem = m_pBack;
    3607  while(pItem != VMA_NULL)
    3608  {
    3609  ItemType* const pPrevItem = pItem->pPrev;
    3610  m_ItemAllocator.Free(pItem);
    3611  pItem = pPrevItem;
    3612  }
    3613  m_pFront = VMA_NULL;
    3614  m_pBack = VMA_NULL;
    3615  m_Count = 0;
    3616  }
    3617 }
    3618 
    3619 template<typename T>
    3620 VmaListItem<T>* VmaRawList<T>::PushBack()
    3621 {
    3622  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3623  pNewItem->pNext = VMA_NULL;
    3624  if(IsEmpty())
    3625  {
    3626  pNewItem->pPrev = VMA_NULL;
    3627  m_pFront = pNewItem;
    3628  m_pBack = pNewItem;
    3629  m_Count = 1;
    3630  }
    3631  else
    3632  {
    3633  pNewItem->pPrev = m_pBack;
    3634  m_pBack->pNext = pNewItem;
    3635  m_pBack = pNewItem;
    3636  ++m_Count;
    3637  }
    3638  return pNewItem;
    3639 }
    3640 
    3641 template<typename T>
    3642 VmaListItem<T>* VmaRawList<T>::PushFront()
    3643 {
    3644  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3645  pNewItem->pPrev = VMA_NULL;
    3646  if(IsEmpty())
    3647  {
    3648  pNewItem->pNext = VMA_NULL;
    3649  m_pFront = pNewItem;
    3650  m_pBack = pNewItem;
    3651  m_Count = 1;
    3652  }
    3653  else
    3654  {
    3655  pNewItem->pNext = m_pFront;
    3656  m_pFront->pPrev = pNewItem;
    3657  m_pFront = pNewItem;
    3658  ++m_Count;
    3659  }
    3660  return pNewItem;
    3661 }
    3662 
    3663 template<typename T>
    3664 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
    3665 {
    3666  ItemType* const pNewItem = PushBack();
    3667  pNewItem->Value = value;
    3668  return pNewItem;
    3669 }
    3670 
    3671 template<typename T>
    3672 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
    3673 {
    3674  ItemType* const pNewItem = PushFront();
    3675  pNewItem->Value = value;
    3676  return pNewItem;
    3677 }
    3678 
    3679 template<typename T>
    3680 void VmaRawList<T>::PopBack()
    3681 {
    3682  VMA_HEAVY_ASSERT(m_Count > 0);
    3683  ItemType* const pBackItem = m_pBack;
    3684  ItemType* const pPrevItem = pBackItem->pPrev;
    3685  if(pPrevItem != VMA_NULL)
    3686  {
    3687  pPrevItem->pNext = VMA_NULL;
    3688  }
    3689  m_pBack = pPrevItem;
    3690  m_ItemAllocator.Free(pBackItem);
    3691  --m_Count;
    3692 }
    3693 
    3694 template<typename T>
    3695 void VmaRawList<T>::PopFront()
    3696 {
    3697  VMA_HEAVY_ASSERT(m_Count > 0);
    3698  ItemType* const pFrontItem = m_pFront;
    3699  ItemType* const pNextItem = pFrontItem->pNext;
    3700  if(pNextItem != VMA_NULL)
    3701  {
    3702  pNextItem->pPrev = VMA_NULL;
    3703  }
    3704  m_pFront = pNextItem;
    3705  m_ItemAllocator.Free(pFrontItem);
    3706  --m_Count;
    3707 }
    3708 
    3709 template<typename T>
    3710 void VmaRawList<T>::Remove(ItemType* pItem)
    3711 {
    3712  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
    3713  VMA_HEAVY_ASSERT(m_Count > 0);
    3714 
    3715  if(pItem->pPrev != VMA_NULL)
    3716  {
    3717  pItem->pPrev->pNext = pItem->pNext;
    3718  }
    3719  else
    3720  {
    3721  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3722  m_pFront = pItem->pNext;
    3723  }
    3724 
    3725  if(pItem->pNext != VMA_NULL)
    3726  {
    3727  pItem->pNext->pPrev = pItem->pPrev;
    3728  }
    3729  else
    3730  {
    3731  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3732  m_pBack = pItem->pPrev;
    3733  }
    3734 
    3735  m_ItemAllocator.Free(pItem);
    3736  --m_Count;
    3737 }
    3738 
    3739 template<typename T>
    3740 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
    3741 {
    3742  if(pItem != VMA_NULL)
    3743  {
    3744  ItemType* const prevItem = pItem->pPrev;
    3745  ItemType* const newItem = m_ItemAllocator.Alloc();
    3746  newItem->pPrev = prevItem;
    3747  newItem->pNext = pItem;
    3748  pItem->pPrev = newItem;
    3749  if(prevItem != VMA_NULL)
    3750  {
    3751  prevItem->pNext = newItem;
    3752  }
    3753  else
    3754  {
    3755  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3756  m_pFront = newItem;
    3757  }
    3758  ++m_Count;
    3759  return newItem;
    3760  }
    3761  else
    3762  return PushBack();
    3763 }
    3764 
    3765 template<typename T>
    3766 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
    3767 {
    3768  if(pItem != VMA_NULL)
    3769  {
    3770  ItemType* const nextItem = pItem->pNext;
    3771  ItemType* const newItem = m_ItemAllocator.Alloc();
    3772  newItem->pNext = nextItem;
    3773  newItem->pPrev = pItem;
    3774  pItem->pNext = newItem;
    3775  if(nextItem != VMA_NULL)
    3776  {
    3777  nextItem->pPrev = newItem;
    3778  }
    3779  else
    3780  {
    3781  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3782  m_pBack = newItem;
    3783  }
    3784  ++m_Count;
    3785  return newItem;
    3786  }
    3787  else
    3788  return PushFront();
    3789 }
    3790 
    3791 template<typename T>
    3792 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
    3793 {
    3794  ItemType* const newItem = InsertBefore(pItem);
    3795  newItem->Value = value;
    3796  return newItem;
    3797 }
    3798 
    3799 template<typename T>
    3800 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
    3801 {
    3802  ItemType* const newItem = InsertAfter(pItem);
    3803  newItem->Value = value;
    3804  return newItem;
    3805 }
    3806 
    3807 template<typename T, typename AllocatorT>
    3808 class VmaList
    3809 {
    3810  VMA_CLASS_NO_COPY(VmaList)
    3811 public:
    3812  class iterator
    3813  {
    3814  public:
    3815  iterator() :
    3816  m_pList(VMA_NULL),
    3817  m_pItem(VMA_NULL)
    3818  {
    3819  }
    3820 
    3821  T& operator*() const
    3822  {
    3823  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3824  return m_pItem->Value;
    3825  }
    3826  T* operator->() const
    3827  {
    3828  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3829  return &m_pItem->Value;
    3830  }
    3831 
    3832  iterator& operator++()
    3833  {
    3834  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3835  m_pItem = m_pItem->pNext;
    3836  return *this;
    3837  }
    3838  iterator& operator--()
    3839  {
    3840  if(m_pItem != VMA_NULL)
    3841  {
    3842  m_pItem = m_pItem->pPrev;
    3843  }
    3844  else
    3845  {
    3846  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3847  m_pItem = m_pList->Back();
    3848  }
    3849  return *this;
    3850  }
    3851 
    3852  iterator operator++(int)
    3853  {
    3854  iterator result = *this;
    3855  ++*this;
    3856  return result;
    3857  }
    3858  iterator operator--(int)
    3859  {
    3860  iterator result = *this;
    3861  --*this;
    3862  return result;
    3863  }
    3864 
    3865  bool operator==(const iterator& rhs) const
    3866  {
    3867  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3868  return m_pItem == rhs.m_pItem;
    3869  }
    3870  bool operator!=(const iterator& rhs) const
    3871  {
    3872  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3873  return m_pItem != rhs.m_pItem;
    3874  }
    3875 
    3876  private:
    3877  VmaRawList<T>* m_pList;
    3878  VmaListItem<T>* m_pItem;
    3879 
    3880  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
    3881  m_pList(pList),
    3882  m_pItem(pItem)
    3883  {
    3884  }
    3885 
    3886  friend class VmaList<T, AllocatorT>;
    3887  };
    3888 
    3889  class const_iterator
    3890  {
    3891  public:
    3892  const_iterator() :
    3893  m_pList(VMA_NULL),
    3894  m_pItem(VMA_NULL)
    3895  {
    3896  }
    3897 
    3898  const_iterator(const iterator& src) :
    3899  m_pList(src.m_pList),
    3900  m_pItem(src.m_pItem)
    3901  {
    3902  }
    3903 
    3904  const T& operator*() const
    3905  {
    3906  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3907  return m_pItem->Value;
    3908  }
    3909  const T* operator->() const
    3910  {
    3911  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3912  return &m_pItem->Value;
    3913  }
    3914 
    3915  const_iterator& operator++()
    3916  {
    3917  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3918  m_pItem = m_pItem->pNext;
    3919  return *this;
    3920  }
    3921  const_iterator& operator--()
    3922  {
    3923  if(m_pItem != VMA_NULL)
    3924  {
    3925  m_pItem = m_pItem->pPrev;
    3926  }
    3927  else
    3928  {
    3929  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3930  m_pItem = m_pList->Back();
    3931  }
    3932  return *this;
    3933  }
    3934 
    3935  const_iterator operator++(int)
    3936  {
    3937  const_iterator result = *this;
    3938  ++*this;
    3939  return result;
    3940  }
    3941  const_iterator operator--(int)
    3942  {
    3943  const_iterator result = *this;
    3944  --*this;
    3945  return result;
    3946  }
    3947 
    3948  bool operator==(const const_iterator& rhs) const
    3949  {
    3950  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3951  return m_pItem == rhs.m_pItem;
    3952  }
    3953  bool operator!=(const const_iterator& rhs) const
    3954  {
    3955  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3956  return m_pItem != rhs.m_pItem;
    3957  }
    3958 
    3959  private:
    3960  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
    3961  m_pList(pList),
    3962  m_pItem(pItem)
    3963  {
    3964  }
    3965 
    3966  const VmaRawList<T>* m_pList;
    3967  const VmaListItem<T>* m_pItem;
    3968 
    3969  friend class VmaList<T, AllocatorT>;
    3970  };
    3971 
    3972  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
    3973 
    3974  bool empty() const { return m_RawList.IsEmpty(); }
    3975  size_t size() const { return m_RawList.GetCount(); }
    3976 
    3977  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
    3978  iterator end() { return iterator(&m_RawList, VMA_NULL); }
    3979 
    3980  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
    3981  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
    3982 
    3983  void clear() { m_RawList.Clear(); }
    3984  void push_back(const T& value) { m_RawList.PushBack(value); }
    3985  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
    3986  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
    3987 
    3988 private:
    3989  VmaRawList<T> m_RawList;
    3990 };
    3991 
    3992 #endif // #if VMA_USE_STL_LIST
    3993 
    3995 // class VmaMap
    3996 
    3997 // Unused in this version.
    3998 #if 0
    3999 
    4000 #if VMA_USE_STL_UNORDERED_MAP
    4001 
    4002 #define VmaPair std::pair
    4003 
    4004 #define VMA_MAP_TYPE(KeyT, ValueT) \
    4005  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
    4006 
    4007 #else // #if VMA_USE_STL_UNORDERED_MAP
    4008 
    4009 template<typename T1, typename T2>
    4010 struct VmaPair
    4011 {
    4012  T1 first;
    4013  T2 second;
    4014 
    4015  VmaPair() : first(), second() { }
    4016  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
    4017 };
    4018 
    4019 /* Class compatible with subset of interface of std::unordered_map.
    4020 KeyT, ValueT must be POD because they will be stored in VmaVector.
    4021 */
    4022 template<typename KeyT, typename ValueT>
    4023 class VmaMap
    4024 {
    4025 public:
    4026  typedef VmaPair<KeyT, ValueT> PairType;
    4027  typedef PairType* iterator;
    4028 
    4029  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
    4030 
    4031  iterator begin() { return m_Vector.begin(); }
    4032  iterator end() { return m_Vector.end(); }
    4033 
    4034  void insert(const PairType& pair);
    4035  iterator find(const KeyT& key);
    4036  void erase(iterator it);
    4037 
    4038 private:
    4039  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
    4040 };
    4041 
    4042 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
    4043 
    4044 template<typename FirstT, typename SecondT>
    4045 struct VmaPairFirstLess
    4046 {
    4047  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
    4048  {
    4049  return lhs.first < rhs.first;
    4050  }
    4051  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
    4052  {
    4053  return lhs.first < rhsFirst;
    4054  }
    4055 };
    4056 
    4057 template<typename KeyT, typename ValueT>
    4058 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
    4059 {
    4060  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    4061  m_Vector.data(),
    4062  m_Vector.data() + m_Vector.size(),
    4063  pair,
    4064  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
    4065  VmaVectorInsert(m_Vector, indexToInsert, pair);
    4066 }
    4067 
    4068 template<typename KeyT, typename ValueT>
    4069 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
    4070 {
    4071  PairType* it = VmaBinaryFindFirstNotLess(
    4072  m_Vector.data(),
    4073  m_Vector.data() + m_Vector.size(),
    4074  key,
    4075  VmaPairFirstLess<KeyT, ValueT>());
    4076  if((it != m_Vector.end()) && (it->first == key))
    4077  {
    4078  return it;
    4079  }
    4080  else
    4081  {
    4082  return m_Vector.end();
    4083  }
    4084 }
    4085 
    4086 template<typename KeyT, typename ValueT>
    4087 void VmaMap<KeyT, ValueT>::erase(iterator it)
    4088 {
    4089  VmaVectorRemove(m_Vector, it - m_Vector.begin());
    4090 }
    4091 
    4092 #endif // #if VMA_USE_STL_UNORDERED_MAP
    4093 
    4094 #endif // #if 0
    4095 
    4097 
    4098 class VmaDeviceMemoryBlock;
    4099 
    4100 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
    4101 
    4102 struct VmaAllocation_T
    4103 {
    4104  VMA_CLASS_NO_COPY(VmaAllocation_T)
    4105 private:
    4106  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
    4107 
    4108  enum FLAGS
    4109  {
    4110  FLAG_USER_DATA_STRING = 0x01,
    4111  };
    4112 
    4113 public:
    4114  enum ALLOCATION_TYPE
    4115  {
    4116  ALLOCATION_TYPE_NONE,
    4117  ALLOCATION_TYPE_BLOCK,
    4118  ALLOCATION_TYPE_DEDICATED,
    4119  };
    4120 
    4121  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
    4122  m_Alignment(1),
    4123  m_Size(0),
    4124  m_pUserData(VMA_NULL),
    4125  m_LastUseFrameIndex(currentFrameIndex),
    4126  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
    4127  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
    4128  m_MapCount(0),
    4129  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
    4130  {
    4131 #if VMA_STATS_STRING_ENABLED
    4132  m_CreationFrameIndex = currentFrameIndex;
    4133  m_BufferImageUsage = 0;
    4134 #endif
    4135  }
    4136 
    4137  ~VmaAllocation_T()
    4138  {
    4139  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
    4140 
    4141  // Check if owned string was freed.
    4142  VMA_ASSERT(m_pUserData == VMA_NULL);
    4143  }
    4144 
    4145  void InitBlockAllocation(
    4146  VmaPool hPool,
    4147  VmaDeviceMemoryBlock* block,
    4148  VkDeviceSize offset,
    4149  VkDeviceSize alignment,
    4150  VkDeviceSize size,
    4151  VmaSuballocationType suballocationType,
    4152  bool mapped,
    4153  bool canBecomeLost)
    4154  {
    4155  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4156  VMA_ASSERT(block != VMA_NULL);
    4157  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4158  m_Alignment = alignment;
    4159  m_Size = size;
    4160  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4161  m_SuballocationType = (uint8_t)suballocationType;
    4162  m_BlockAllocation.m_hPool = hPool;
    4163  m_BlockAllocation.m_Block = block;
    4164  m_BlockAllocation.m_Offset = offset;
    4165  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
    4166  }
    4167 
    4168  void InitLost()
    4169  {
    4170  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4171  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
    4172  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4173  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
    4174  m_BlockAllocation.m_Block = VMA_NULL;
    4175  m_BlockAllocation.m_Offset = 0;
    4176  m_BlockAllocation.m_CanBecomeLost = true;
    4177  }
    4178 
    4179  void ChangeBlockAllocation(
    4180  VmaAllocator hAllocator,
    4181  VmaDeviceMemoryBlock* block,
    4182  VkDeviceSize offset);
    4183 
    4184  // pMappedData not null means allocation is created with MAPPED flag.
    4185  void InitDedicatedAllocation(
    4186  uint32_t memoryTypeIndex,
    4187  VkDeviceMemory hMemory,
    4188  VmaSuballocationType suballocationType,
    4189  void* pMappedData,
    4190  VkDeviceSize size)
    4191  {
    4192  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4193  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
    4194  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
    4195  m_Alignment = 0;
    4196  m_Size = size;
    4197  m_SuballocationType = (uint8_t)suballocationType;
    4198  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4199  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
    4200  m_DedicatedAllocation.m_hMemory = hMemory;
    4201  m_DedicatedAllocation.m_pMappedData = pMappedData;
    4202  }
    4203 
    4204  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
    4205  VkDeviceSize GetAlignment() const { return m_Alignment; }
    4206  VkDeviceSize GetSize() const { return m_Size; }
    4207  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
    4208  void* GetUserData() const { return m_pUserData; }
    4209  void SetUserData(VmaAllocator hAllocator, void* pUserData);
    4210  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
    4211 
    4212  VmaDeviceMemoryBlock* GetBlock() const
    4213  {
    4214  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    4215  return m_BlockAllocation.m_Block;
    4216  }
    4217  VkDeviceSize GetOffset() const;
    4218  VkDeviceMemory GetMemory() const;
    4219  uint32_t GetMemoryTypeIndex() const;
    4220  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
    4221  void* GetMappedData() const;
    4222  bool CanBecomeLost() const;
    4223  VmaPool GetPool() const;
    4224 
    4225  uint32_t GetLastUseFrameIndex() const
    4226  {
    4227  return m_LastUseFrameIndex.load();
    4228  }
    4229  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
    4230  {
    4231  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
    4232  }
    4233  /*
    4234  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
    4235  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
    4236  - Else, returns false.
    4237 
    4238  If hAllocation is already lost, assert - you should not call it then.
    4239  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
    4240  */
    4241  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4242 
    4243  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
    4244  {
    4245  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
    4246  outInfo.blockCount = 1;
    4247  outInfo.allocationCount = 1;
    4248  outInfo.unusedRangeCount = 0;
    4249  outInfo.usedBytes = m_Size;
    4250  outInfo.unusedBytes = 0;
    4251  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
    4252  outInfo.unusedRangeSizeMin = UINT64_MAX;
    4253  outInfo.unusedRangeSizeMax = 0;
    4254  }
    4255 
    4256  void BlockAllocMap();
    4257  void BlockAllocUnmap();
    4258  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
    4259  void DedicatedAllocUnmap(VmaAllocator hAllocator);
    4260 
    4261 #if VMA_STATS_STRING_ENABLED
    4262  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
    4263  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
    4264 
    4265  void InitBufferImageUsage(uint32_t bufferImageUsage)
    4266  {
    4267  VMA_ASSERT(m_BufferImageUsage == 0);
    4268  m_BufferImageUsage = bufferImageUsage;
    4269  }
    4270 
    4271  void PrintParameters(class VmaJsonWriter& json) const;
    4272 #endif
    4273 
    4274 private:
    4275  VkDeviceSize m_Alignment;
    4276  VkDeviceSize m_Size;
    4277  void* m_pUserData;
    4278  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
    4279  uint8_t m_Type; // ALLOCATION_TYPE
    4280  uint8_t m_SuballocationType; // VmaSuballocationType
    4281  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
    4282  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
    4283  uint8_t m_MapCount;
    4284  uint8_t m_Flags; // enum FLAGS
    4285 
    4286  // Allocation out of VmaDeviceMemoryBlock.
    4287  struct BlockAllocation
    4288  {
    4289  VmaPool m_hPool; // Null if belongs to general memory.
    4290  VmaDeviceMemoryBlock* m_Block;
    4291  VkDeviceSize m_Offset;
    4292  bool m_CanBecomeLost;
    4293  };
    4294 
    4295  // Allocation for an object that has its own private VkDeviceMemory.
    4296  struct DedicatedAllocation
    4297  {
    4298  uint32_t m_MemoryTypeIndex;
    4299  VkDeviceMemory m_hMemory;
    4300  void* m_pMappedData; // Not null means memory is mapped.
    4301  };
    4302 
    4303  union
    4304  {
    4305  // Allocation out of VmaDeviceMemoryBlock.
    4306  BlockAllocation m_BlockAllocation;
    4307  // Allocation for an object that has its own private VkDeviceMemory.
    4308  DedicatedAllocation m_DedicatedAllocation;
    4309  };
    4310 
    4311 #if VMA_STATS_STRING_ENABLED
    4312  uint32_t m_CreationFrameIndex;
    4313  uint32_t m_BufferImageUsage; // 0 if unknown.
    4314 #endif
    4315 
    4316  void FreeUserDataString(VmaAllocator hAllocator);
    4317 };
    4318 
    4319 /*
    4320 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
    4321 allocated memory block or free.
    4322 */
    4323 struct VmaSuballocation
    4324 {
    4325  VkDeviceSize offset;
    4326  VkDeviceSize size;
    4327  VmaAllocation hAllocation;
    4328  VmaSuballocationType type;
    4329 };
    4330 
    4331 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
    4332 
    4333 // Cost of one additional allocation lost, as equivalent in bytes.
    4334 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
    4335 
    4336 /*
    4337 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
    4338 
    4339 If canMakeOtherLost was false:
    4340 - item points to a FREE suballocation.
    4341 - itemsToMakeLostCount is 0.
    4342 
    4343 If canMakeOtherLost was true:
    4344 - item points to first of sequence of suballocations, which are either FREE,
    4345  or point to VmaAllocations that can become lost.
    4346 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
    4347  the requested allocation to succeed.
    4348 */
    4349 struct VmaAllocationRequest
    4350 {
    4351  VkDeviceSize offset;
    4352  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
    4353  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
    4354  VmaSuballocationList::iterator item;
    4355  size_t itemsToMakeLostCount;
    4356 
    4357  VkDeviceSize CalcCost() const
    4358  {
    4359  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
    4360  }
    4361 };
    4362 
    4363 /*
    4364 Data structure used for bookkeeping of allocations and unused ranges of memory
    4365 in a single VkDeviceMemory block.
    4366 */
    4367 class VmaBlockMetadata
    4368 {
    4369  VMA_CLASS_NO_COPY(VmaBlockMetadata)
    4370 public:
    4371  VmaBlockMetadata(VmaAllocator hAllocator);
    4372  ~VmaBlockMetadata();
    4373  void Init(VkDeviceSize size);
    4374 
    4375  // Validates all data structures inside this object. If not valid, returns false.
    4376  bool Validate() const;
    4377  VkDeviceSize GetSize() const { return m_Size; }
    4378  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
    4379  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
    4380  VkDeviceSize GetUnusedRangeSizeMax() const;
    4381  // Returns true if this block is empty - contains only single free suballocation.
    4382  bool IsEmpty() const;
    4383 
    4384  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
    4385  void AddPoolStats(VmaPoolStats& inoutStats) const;
    4386 
    4387 #if VMA_STATS_STRING_ENABLED
    4388  void PrintDetailedMap(class VmaJsonWriter& json) const;
    4389 #endif
    4390 
    4391  // Tries to find a place for suballocation with given parameters inside this block.
    4392  // If succeeded, fills pAllocationRequest and returns true.
    4393  // If failed, returns false.
    4394  bool CreateAllocationRequest(
    4395  uint32_t currentFrameIndex,
    4396  uint32_t frameInUseCount,
    4397  VkDeviceSize bufferImageGranularity,
    4398  VkDeviceSize allocSize,
    4399  VkDeviceSize allocAlignment,
    4400  VmaSuballocationType allocType,
    4401  bool canMakeOtherLost,
    4402  VmaAllocationRequest* pAllocationRequest);
    4403 
    4404  bool MakeRequestedAllocationsLost(
    4405  uint32_t currentFrameIndex,
    4406  uint32_t frameInUseCount,
    4407  VmaAllocationRequest* pAllocationRequest);
    4408 
    4409  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4410 
    4411  VkResult CheckCorruption(const void* pBlockData);
    4412 
    4413  // Makes actual allocation based on request. Request must already be checked and valid.
    4414  void Alloc(
    4415  const VmaAllocationRequest& request,
    4416  VmaSuballocationType type,
    4417  VkDeviceSize allocSize,
    4418  VmaAllocation hAllocation);
    4419 
    4420  // Frees suballocation assigned to given memory region.
    4421  void Free(const VmaAllocation allocation);
    4422  void FreeAtOffset(VkDeviceSize offset);
    4423 
    4424 private:
    4425  VkDeviceSize m_Size;
    4426  uint32_t m_FreeCount;
    4427  VkDeviceSize m_SumFreeSize;
    4428  VmaSuballocationList m_Suballocations;
    4429  // Suballocations that are free and have size greater than certain threshold.
    4430  // Sorted by size, ascending.
    4431  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
    4432 
    4433  bool ValidateFreeSuballocationList() const;
    4434 
    4435  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
    4436  // If yes, fills pOffset and returns true. If no, returns false.
    4437  bool CheckAllocation(
    4438  uint32_t currentFrameIndex,
    4439  uint32_t frameInUseCount,
    4440  VkDeviceSize bufferImageGranularity,
    4441  VkDeviceSize allocSize,
    4442  VkDeviceSize allocAlignment,
    4443  VmaSuballocationType allocType,
    4444  VmaSuballocationList::const_iterator suballocItem,
    4445  bool canMakeOtherLost,
    4446  VkDeviceSize* pOffset,
    4447  size_t* itemsToMakeLostCount,
    4448  VkDeviceSize* pSumFreeSize,
    4449  VkDeviceSize* pSumItemSize) const;
    4450  // Given free suballocation, it merges it with following one, which must also be free.
    4451  void MergeFreeWithNext(VmaSuballocationList::iterator item);
    4452  // Releases given suballocation, making it free.
    4453  // Merges it with adjacent free suballocations if applicable.
    4454  // Returns iterator to new free suballocation at this place.
    4455  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
    4456  // Given free suballocation, it inserts it into sorted list of
    4457  // m_FreeSuballocationsBySize if it's suitable.
    4458  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
    4459  // Given free suballocation, it removes it from sorted list of
    4460  // m_FreeSuballocationsBySize if it's suitable.
    4461  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
    4462 };
    4463 
    4464 /*
    4465 Represents a single block of device memory (`VkDeviceMemory`) with all the
    4466 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
    4467 
    4468 Thread-safety: This class must be externally synchronized.
    4469 */
    4470 class VmaDeviceMemoryBlock
    4471 {
    4472  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
    4473 public:
    4474  VmaBlockMetadata m_Metadata;
    4475 
    4476  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
    4477 
    4478  ~VmaDeviceMemoryBlock()
    4479  {
    4480  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
    4481  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    4482  }
    4483 
    4484  // Always call after construction.
    4485  void Init(
    4486  uint32_t newMemoryTypeIndex,
    4487  VkDeviceMemory newMemory,
    4488  VkDeviceSize newSize,
    4489  uint32_t id);
    4490  // Always call before destruction.
    4491  void Destroy(VmaAllocator allocator);
    4492 
    4493  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
    4494  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4495  uint32_t GetId() const { return m_Id; }
    4496  void* GetMappedData() const { return m_pMappedData; }
    4497 
    4498  // Validates all data structures inside this object. If not valid, returns false.
    4499  bool Validate() const;
    4500 
    4501  VkResult CheckCorruption(VmaAllocator hAllocator);
    4502 
    4503  // ppData can be null.
    4504  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
    4505  void Unmap(VmaAllocator hAllocator, uint32_t count);
    4506 
    4507  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4508  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4509 
    4510  VkResult BindBufferMemory(
    4511  const VmaAllocator hAllocator,
    4512  const VmaAllocation hAllocation,
    4513  VkBuffer hBuffer);
    4514  VkResult BindImageMemory(
    4515  const VmaAllocator hAllocator,
    4516  const VmaAllocation hAllocation,
    4517  VkImage hImage);
    4518 
    4519 private:
    4520  uint32_t m_MemoryTypeIndex;
    4521  uint32_t m_Id;
    4522  VkDeviceMemory m_hMemory;
    4523 
    4524  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
    4525  // Also protects m_MapCount, m_pMappedData.
    4526  VMA_MUTEX m_Mutex;
    4527  uint32_t m_MapCount;
    4528  void* m_pMappedData;
    4529 };
    4530 
    4531 struct VmaPointerLess
    4532 {
    4533  bool operator()(const void* lhs, const void* rhs) const
    4534  {
    4535  return lhs < rhs;
    4536  }
    4537 };
    4538 
    4539 class VmaDefragmentator;
    4540 
    4541 /*
    4542 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
    4543 Vulkan memory type.
    4544 
    4545 Synchronized internally with a mutex.
    4546 */
    4547 struct VmaBlockVector
    4548 {
    4549  VMA_CLASS_NO_COPY(VmaBlockVector)
    4550 public:
    4551  VmaBlockVector(
    4552  VmaAllocator hAllocator,
    4553  uint32_t memoryTypeIndex,
    4554  VkDeviceSize preferredBlockSize,
    4555  size_t minBlockCount,
    4556  size_t maxBlockCount,
    4557  VkDeviceSize bufferImageGranularity,
    4558  uint32_t frameInUseCount,
    4559  bool isCustomPool);
    4560  ~VmaBlockVector();
    4561 
    4562  VkResult CreateMinBlocks();
    4563 
    4564  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4565  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
    4566  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
    4567  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
    4568 
    4569  void GetPoolStats(VmaPoolStats* pStats);
    4570 
    4571  bool IsEmpty() const { return m_Blocks.empty(); }
    4572  bool IsCorruptionDetectionEnabled() const;
    4573 
    4574  VkResult Allocate(
    4575  VmaPool hCurrentPool,
    4576  uint32_t currentFrameIndex,
    4577  VkDeviceSize size,
    4578  VkDeviceSize alignment,
    4579  const VmaAllocationCreateInfo& createInfo,
    4580  VmaSuballocationType suballocType,
    4581  VmaAllocation* pAllocation);
    4582 
    4583  void Free(
    4584  VmaAllocation hAllocation);
    4585 
    4586  // Adds statistics of this BlockVector to pStats.
    4587  void AddStats(VmaStats* pStats);
    4588 
    4589 #if VMA_STATS_STRING_ENABLED
    4590  void PrintDetailedMap(class VmaJsonWriter& json);
    4591 #endif
    4592 
    4593  void MakePoolAllocationsLost(
    4594  uint32_t currentFrameIndex,
    4595  size_t* pLostAllocationCount);
    4596  VkResult CheckCorruption();
    4597 
    4598  VmaDefragmentator* EnsureDefragmentator(
    4599  VmaAllocator hAllocator,
    4600  uint32_t currentFrameIndex);
    4601 
    4602  VkResult Defragment(
    4603  VmaDefragmentationStats* pDefragmentationStats,
    4604  VkDeviceSize& maxBytesToMove,
    4605  uint32_t& maxAllocationsToMove);
    4606 
    4607  void DestroyDefragmentator();
    4608 
    4609 private:
    4610  friend class VmaDefragmentator;
    4611 
    4612  const VmaAllocator m_hAllocator;
    4613  const uint32_t m_MemoryTypeIndex;
    4614  const VkDeviceSize m_PreferredBlockSize;
    4615  const size_t m_MinBlockCount;
    4616  const size_t m_MaxBlockCount;
    4617  const VkDeviceSize m_BufferImageGranularity;
    4618  const uint32_t m_FrameInUseCount;
    4619  const bool m_IsCustomPool;
    4620  VMA_MUTEX m_Mutex;
    4621  // Incrementally sorted by sumFreeSize, ascending.
    4622  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
    4623  /* There can be at most one allocation that is completely empty - a
    4624  hysteresis to avoid pessimistic case of alternating creation and destruction
    4625  of a VkDeviceMemory. */
    4626  bool m_HasEmptyBlock;
    4627  VmaDefragmentator* m_pDefragmentator;
    4628  uint32_t m_NextBlockId;
    4629 
    4630  VkDeviceSize CalcMaxBlockSize() const;
    4631 
    4632  // Finds and removes given block from vector.
    4633  void Remove(VmaDeviceMemoryBlock* pBlock);
    4634 
    4635  // Performs single step in sorting m_Blocks. They may not be fully sorted
    4636  // after this call.
    4637  void IncrementallySortBlocks();
    4638 
    4639  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
    4640 };
    4641 
    4642 struct VmaPool_T
    4643 {
    4644  VMA_CLASS_NO_COPY(VmaPool_T)
    4645 public:
    4646  VmaBlockVector m_BlockVector;
    4647 
    4648  VmaPool_T(
    4649  VmaAllocator hAllocator,
    4650  const VmaPoolCreateInfo& createInfo);
    4651  ~VmaPool_T();
    4652 
    4653  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
    4654  uint32_t GetId() const { return m_Id; }
    4655  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
    4656 
    4657 #if VMA_STATS_STRING_ENABLED
    4658  //void PrintDetailedMap(class VmaStringBuilder& sb);
    4659 #endif
    4660 
    4661 private:
    4662  uint32_t m_Id;
    4663 };
    4664 
    4665 class VmaDefragmentator
    4666 {
    4667  VMA_CLASS_NO_COPY(VmaDefragmentator)
    4668 private:
    4669  const VmaAllocator m_hAllocator;
    4670  VmaBlockVector* const m_pBlockVector;
    4671  uint32_t m_CurrentFrameIndex;
    4672  VkDeviceSize m_BytesMoved;
    4673  uint32_t m_AllocationsMoved;
    4674 
    4675  struct AllocationInfo
    4676  {
    4677  VmaAllocation m_hAllocation;
    4678  VkBool32* m_pChanged;
    4679 
    4680  AllocationInfo() :
    4681  m_hAllocation(VK_NULL_HANDLE),
    4682  m_pChanged(VMA_NULL)
    4683  {
    4684  }
    4685  };
    4686 
    4687  struct AllocationInfoSizeGreater
    4688  {
    4689  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
    4690  {
    4691  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
    4692  }
    4693  };
    4694 
    4695  // Used between AddAllocation and Defragment.
    4696  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4697 
    4698  struct BlockInfo
    4699  {
    4700  VmaDeviceMemoryBlock* m_pBlock;
    4701  bool m_HasNonMovableAllocations;
    4702  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4703 
    4704  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
    4705  m_pBlock(VMA_NULL),
    4706  m_HasNonMovableAllocations(true),
    4707  m_Allocations(pAllocationCallbacks),
    4708  m_pMappedDataForDefragmentation(VMA_NULL)
    4709  {
    4710  }
    4711 
    4712  void CalcHasNonMovableAllocations()
    4713  {
    4714  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
    4715  const size_t defragmentAllocCount = m_Allocations.size();
    4716  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
    4717  }
    4718 
    4719  void SortAllocationsBySizeDescecnding()
    4720  {
    4721  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
    4722  }
    4723 
    4724  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
    4725  void Unmap(VmaAllocator hAllocator);
    4726 
    4727  private:
    4728  // Not null if mapped for defragmentation only, not originally mapped.
    4729  void* m_pMappedDataForDefragmentation;
    4730  };
    4731 
    4732  struct BlockPointerLess
    4733  {
    4734  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
    4735  {
    4736  return pLhsBlockInfo->m_pBlock < pRhsBlock;
    4737  }
    4738  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4739  {
    4740  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
    4741  }
    4742  };
    4743 
    4744  // 1. Blocks with some non-movable allocations go first.
    4745  // 2. Blocks with smaller sumFreeSize go first.
    4746  struct BlockInfoCompareMoveDestination
    4747  {
    4748  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4749  {
    4750  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
    4751  {
    4752  return true;
    4753  }
    4754  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
    4755  {
    4756  return false;
    4757  }
    4758  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
    4759  {
    4760  return true;
    4761  }
    4762  return false;
    4763  }
    4764  };
    4765 
    4766  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
    4767  BlockInfoVector m_Blocks;
    4768 
    4769  VkResult DefragmentRound(
    4770  VkDeviceSize maxBytesToMove,
    4771  uint32_t maxAllocationsToMove);
    4772 
    4773  static bool MoveMakesSense(
    4774  size_t dstBlockIndex, VkDeviceSize dstOffset,
    4775  size_t srcBlockIndex, VkDeviceSize srcOffset);
    4776 
    4777 public:
    4778  VmaDefragmentator(
    4779  VmaAllocator hAllocator,
    4780  VmaBlockVector* pBlockVector,
    4781  uint32_t currentFrameIndex);
    4782 
    4783  ~VmaDefragmentator();
    4784 
    4785  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
    4786  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
    4787 
    4788  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
    4789 
    4790  VkResult Defragment(
    4791  VkDeviceSize maxBytesToMove,
    4792  uint32_t maxAllocationsToMove);
    4793 };
    4794 
    4795 #if VMA_RECORDING_ENABLED
    4796 
    4797 class VmaRecorder
    4798 {
    4799 public:
    4800  VmaRecorder();
    4801  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
    4802  ~VmaRecorder();
    4803 
    4804  void RecordCreateAllocator(uint32_t frameIndex);
    4805  void RecordDestroyAllocator(uint32_t frameIndex);
    4806  void RecordCreatePool(uint32_t frameIndex,
    4807  const VmaPoolCreateInfo& createInfo,
    4808  VmaPool pool);
    4809  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
    4810  void RecordAllocateMemory(uint32_t frameIndex,
    4811  const VkMemoryRequirements& vkMemReq,
    4812  const VmaAllocationCreateInfo& createInfo,
    4813  VmaAllocation allocation);
    4814  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    4815  const VkMemoryRequirements& vkMemReq,
    4816  bool requiresDedicatedAllocation,
    4817  bool prefersDedicatedAllocation,
    4818  const VmaAllocationCreateInfo& createInfo,
    4819  VmaAllocation allocation);
    4820  void RecordAllocateMemoryForImage(uint32_t frameIndex,
    4821  const VkMemoryRequirements& vkMemReq,
    4822  bool requiresDedicatedAllocation,
    4823  bool prefersDedicatedAllocation,
    4824  const VmaAllocationCreateInfo& createInfo,
    4825  VmaAllocation allocation);
    4826  void RecordFreeMemory(uint32_t frameIndex,
    4827  VmaAllocation allocation);
    4828  void RecordSetAllocationUserData(uint32_t frameIndex,
    4829  VmaAllocation allocation,
    4830  const void* pUserData);
    4831  void RecordCreateLostAllocation(uint32_t frameIndex,
    4832  VmaAllocation allocation);
    4833  void RecordMapMemory(uint32_t frameIndex,
    4834  VmaAllocation allocation);
    4835  void RecordUnmapMemory(uint32_t frameIndex,
    4836  VmaAllocation allocation);
    4837  void RecordFlushAllocation(uint32_t frameIndex,
    4838  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4839  void RecordInvalidateAllocation(uint32_t frameIndex,
    4840  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4841  void RecordCreateBuffer(uint32_t frameIndex,
    4842  const VkBufferCreateInfo& bufCreateInfo,
    4843  const VmaAllocationCreateInfo& allocCreateInfo,
    4844  VmaAllocation allocation);
    4845  void RecordCreateImage(uint32_t frameIndex,
    4846  const VkImageCreateInfo& imageCreateInfo,
    4847  const VmaAllocationCreateInfo& allocCreateInfo,
    4848  VmaAllocation allocation);
    4849  void RecordDestroyBuffer(uint32_t frameIndex,
    4850  VmaAllocation allocation);
    4851  void RecordDestroyImage(uint32_t frameIndex,
    4852  VmaAllocation allocation);
    4853  void RecordTouchAllocation(uint32_t frameIndex,
    4854  VmaAllocation allocation);
    4855  void RecordGetAllocationInfo(uint32_t frameIndex,
    4856  VmaAllocation allocation);
    4857 
    4858 private:
    4859  struct CallParams
    4860  {
    4861  uint32_t threadId;
    4862  double time;
    4863  };
    4864 
    4865  class UserDataString
    4866  {
    4867  public:
    4868  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
    4869  const char* GetString() const { return m_Str; }
    4870 
    4871  private:
    4872  char m_PtrStr[17];
    4873  const char* m_Str;
    4874  };
    4875 
    4876  bool m_UseMutex;
    4877  VmaRecordFlags m_Flags;
    4878  FILE* m_File;
    4879  VMA_MUTEX m_FileMutex;
    4880  int64_t m_Freq;
    4881  int64_t m_StartCounter;
    4882 
    4883  void GetBasicParams(CallParams& outParams);
    4884  void Flush();
    4885 };
    4886 
    4887 #endif // #if VMA_RECORDING_ENABLED
    4888 
    4889 // Main allocator object.
    4890 struct VmaAllocator_T
    4891 {
    4892  VMA_CLASS_NO_COPY(VmaAllocator_T)
    4893 public:
    4894  bool m_UseMutex;
    4895  bool m_UseKhrDedicatedAllocation;
    4896  VkDevice m_hDevice;
    4897  bool m_AllocationCallbacksSpecified;
    4898  VkAllocationCallbacks m_AllocationCallbacks;
    4899  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
    4900 
    4901  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
    4902  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
    4903  VMA_MUTEX m_HeapSizeLimitMutex;
    4904 
    4905  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
    4906  VkPhysicalDeviceMemoryProperties m_MemProps;
    4907 
    4908  // Default pools.
    4909  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
    4910 
    4911  // Each vector is sorted by memory (handle value).
    4912  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
    4913  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
    4914  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
    4915 
    4916  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
    4917  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
    4918  ~VmaAllocator_T();
    4919 
    4920  const VkAllocationCallbacks* GetAllocationCallbacks() const
    4921  {
    4922  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
    4923  }
    4924  const VmaVulkanFunctions& GetVulkanFunctions() const
    4925  {
    4926  return m_VulkanFunctions;
    4927  }
    4928 
    4929  VkDeviceSize GetBufferImageGranularity() const
    4930  {
    4931  return VMA_MAX(
    4932  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
    4933  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
    4934  }
    4935 
    4936  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
    4937  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
    4938 
    4939  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
    4940  {
    4941  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
    4942  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
    4943  }
    4944  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
    4945  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
    4946  {
    4947  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
    4948  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    4949  }
    4950  // Minimum alignment for all allocations in specific memory type.
    4951  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
    4952  {
    4953  return IsMemoryTypeNonCoherent(memTypeIndex) ?
    4954  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
    4955  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
    4956  }
    4957 
    4958  bool IsIntegratedGpu() const
    4959  {
    4960  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
    4961  }
    4962 
    4963 #if VMA_RECORDING_ENABLED
    4964  VmaRecorder* GetRecorder() const { return m_pRecorder; }
    4965 #endif
    4966 
    4967  void GetBufferMemoryRequirements(
    4968  VkBuffer hBuffer,
    4969  VkMemoryRequirements& memReq,
    4970  bool& requiresDedicatedAllocation,
    4971  bool& prefersDedicatedAllocation) const;
    4972  void GetImageMemoryRequirements(
    4973  VkImage hImage,
    4974  VkMemoryRequirements& memReq,
    4975  bool& requiresDedicatedAllocation,
    4976  bool& prefersDedicatedAllocation) const;
    4977 
    4978  // Main allocation function.
    4979  VkResult AllocateMemory(
    4980  const VkMemoryRequirements& vkMemReq,
    4981  bool requiresDedicatedAllocation,
    4982  bool prefersDedicatedAllocation,
    4983  VkBuffer dedicatedBuffer,
    4984  VkImage dedicatedImage,
    4985  const VmaAllocationCreateInfo& createInfo,
    4986  VmaSuballocationType suballocType,
    4987  VmaAllocation* pAllocation);
    4988 
    4989  // Main deallocation function.
    4990  void FreeMemory(const VmaAllocation allocation);
    4991 
    4992  void CalculateStats(VmaStats* pStats);
    4993 
    4994 #if VMA_STATS_STRING_ENABLED
    4995  void PrintDetailedMap(class VmaJsonWriter& json);
    4996 #endif
    4997 
    4998  VkResult Defragment(
    4999  VmaAllocation* pAllocations,
    5000  size_t allocationCount,
    5001  VkBool32* pAllocationsChanged,
    5002  const VmaDefragmentationInfo* pDefragmentationInfo,
    5003  VmaDefragmentationStats* pDefragmentationStats);
    5004 
    5005  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
    5006  bool TouchAllocation(VmaAllocation hAllocation);
    5007 
    5008  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
    5009  void DestroyPool(VmaPool pool);
    5010  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
    5011 
    5012  void SetCurrentFrameIndex(uint32_t frameIndex);
    5013  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
    5014 
    5015  void MakePoolAllocationsLost(
    5016  VmaPool hPool,
    5017  size_t* pLostAllocationCount);
    5018  VkResult CheckPoolCorruption(VmaPool hPool);
    5019  VkResult CheckCorruption(uint32_t memoryTypeBits);
    5020 
    5021  void CreateLostAllocation(VmaAllocation* pAllocation);
    5022 
    5023  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
    5024  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
    5025 
    5026  VkResult Map(VmaAllocation hAllocation, void** ppData);
    5027  void Unmap(VmaAllocation hAllocation);
    5028 
    5029  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
    5030  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
    5031 
    5032  void FlushOrInvalidateAllocation(
    5033  VmaAllocation hAllocation,
    5034  VkDeviceSize offset, VkDeviceSize size,
    5035  VMA_CACHE_OPERATION op);
    5036 
    5037  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
    5038 
    5039 private:
    5040  VkDeviceSize m_PreferredLargeHeapBlockSize;
    5041 
    5042  VkPhysicalDevice m_PhysicalDevice;
    5043  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
    5044 
    5045  VMA_MUTEX m_PoolsMutex;
    5046  // Protected by m_PoolsMutex. Sorted by pointer value.
    5047  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
    5048  uint32_t m_NextPoolId;
    5049 
    5050  VmaVulkanFunctions m_VulkanFunctions;
    5051 
    5052 #if VMA_RECORDING_ENABLED
    5053  VmaRecorder* m_pRecorder;
    5054 #endif
    5055 
    5056  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
    5057 
    5058  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
    5059 
    5060  VkResult AllocateMemoryOfType(
    5061  VkDeviceSize size,
    5062  VkDeviceSize alignment,
    5063  bool dedicatedAllocation,
    5064  VkBuffer dedicatedBuffer,
    5065  VkImage dedicatedImage,
    5066  const VmaAllocationCreateInfo& createInfo,
    5067  uint32_t memTypeIndex,
    5068  VmaSuballocationType suballocType,
    5069  VmaAllocation* pAllocation);
    5070 
    5071  // Allocates and registers new VkDeviceMemory specifically for single allocation.
    5072  VkResult AllocateDedicatedMemory(
    5073  VkDeviceSize size,
    5074  VmaSuballocationType suballocType,
    5075  uint32_t memTypeIndex,
    5076  bool map,
    5077  bool isUserDataString,
    5078  void* pUserData,
    5079  VkBuffer dedicatedBuffer,
    5080  VkImage dedicatedImage,
    5081  VmaAllocation* pAllocation);
    5082 
    5083  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
    5084  void FreeDedicatedMemory(VmaAllocation allocation);
    5085 };
    5086 
    5088 // Memory allocation #2 after VmaAllocator_T definition
    5089 
    5090 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
    5091 {
    5092  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
    5093 }
    5094 
    5095 static void VmaFree(VmaAllocator hAllocator, void* ptr)
    5096 {
    5097  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
    5098 }
    5099 
    5100 template<typename T>
    5101 static T* VmaAllocate(VmaAllocator hAllocator)
    5102 {
    5103  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
    5104 }
    5105 
    5106 template<typename T>
    5107 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
    5108 {
    5109  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
    5110 }
    5111 
    5112 template<typename T>
    5113 static void vma_delete(VmaAllocator hAllocator, T* ptr)
    5114 {
    5115  if(ptr != VMA_NULL)
    5116  {
    5117  ptr->~T();
    5118  VmaFree(hAllocator, ptr);
    5119  }
    5120 }
    5121 
    5122 template<typename T>
    5123 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
    5124 {
    5125  if(ptr != VMA_NULL)
    5126  {
    5127  for(size_t i = count; i--; )
    5128  ptr[i].~T();
    5129  VmaFree(hAllocator, ptr);
    5130  }
    5131 }
    5132 
    5134 // VmaStringBuilder
    5135 
    5136 #if VMA_STATS_STRING_ENABLED
    5137 
    5138 class VmaStringBuilder
    5139 {
    5140 public:
    5141  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
    5142  size_t GetLength() const { return m_Data.size(); }
    5143  const char* GetData() const { return m_Data.data(); }
    5144 
    5145  void Add(char ch) { m_Data.push_back(ch); }
    5146  void Add(const char* pStr);
    5147  void AddNewLine() { Add('\n'); }
    5148  void AddNumber(uint32_t num);
    5149  void AddNumber(uint64_t num);
    5150  void AddPointer(const void* ptr);
    5151 
    5152 private:
    5153  VmaVector< char, VmaStlAllocator<char> > m_Data;
    5154 };
    5155 
    5156 void VmaStringBuilder::Add(const char* pStr)
    5157 {
    5158  const size_t strLen = strlen(pStr);
    5159  if(strLen > 0)
    5160  {
    5161  const size_t oldCount = m_Data.size();
    5162  m_Data.resize(oldCount + strLen);
    5163  memcpy(m_Data.data() + oldCount, pStr, strLen);
    5164  }
    5165 }
    5166 
    5167 void VmaStringBuilder::AddNumber(uint32_t num)
    5168 {
    5169  char buf[11];
    5170  VmaUint32ToStr(buf, sizeof(buf), num);
    5171  Add(buf);
    5172 }
    5173 
    5174 void VmaStringBuilder::AddNumber(uint64_t num)
    5175 {
    5176  char buf[21];
    5177  VmaUint64ToStr(buf, sizeof(buf), num);
    5178  Add(buf);
    5179 }
    5180 
    5181 void VmaStringBuilder::AddPointer(const void* ptr)
    5182 {
    5183  char buf[21];
    5184  VmaPtrToStr(buf, sizeof(buf), ptr);
    5185  Add(buf);
    5186 }
    5187 
    5188 #endif // #if VMA_STATS_STRING_ENABLED
    5189 
    5191 // VmaJsonWriter
    5192 
    5193 #if VMA_STATS_STRING_ENABLED
    5194 
    5195 class VmaJsonWriter
    5196 {
    5197  VMA_CLASS_NO_COPY(VmaJsonWriter)
    5198 public:
    5199  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
    5200  ~VmaJsonWriter();
    5201 
    5202  void BeginObject(bool singleLine = false);
    5203  void EndObject();
    5204 
    5205  void BeginArray(bool singleLine = false);
    5206  void EndArray();
    5207 
    5208  void WriteString(const char* pStr);
    5209  void BeginString(const char* pStr = VMA_NULL);
    5210  void ContinueString(const char* pStr);
    5211  void ContinueString(uint32_t n);
    5212  void ContinueString(uint64_t n);
    5213  void ContinueString_Pointer(const void* ptr);
    5214  void EndString(const char* pStr = VMA_NULL);
    5215 
    5216  void WriteNumber(uint32_t n);
    5217  void WriteNumber(uint64_t n);
    5218  void WriteBool(bool b);
    5219  void WriteNull();
    5220 
    5221 private:
    5222  static const char* const INDENT;
    5223 
    5224  enum COLLECTION_TYPE
    5225  {
    5226  COLLECTION_TYPE_OBJECT,
    5227  COLLECTION_TYPE_ARRAY,
    5228  };
    5229  struct StackItem
    5230  {
    5231  COLLECTION_TYPE type;
    5232  uint32_t valueCount;
    5233  bool singleLineMode;
    5234  };
    5235 
    5236  VmaStringBuilder& m_SB;
    5237  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
    5238  bool m_InsideString;
    5239 
    5240  void BeginValue(bool isString);
    5241  void WriteIndent(bool oneLess = false);
    5242 };
    5243 
    5244 const char* const VmaJsonWriter::INDENT = " ";
    5245 
    5246 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
    5247  m_SB(sb),
    5248  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
    5249  m_InsideString(false)
    5250 {
    5251 }
    5252 
    5253 VmaJsonWriter::~VmaJsonWriter()
    5254 {
    5255  VMA_ASSERT(!m_InsideString);
    5256  VMA_ASSERT(m_Stack.empty());
    5257 }
    5258 
    5259 void VmaJsonWriter::BeginObject(bool singleLine)
    5260 {
    5261  VMA_ASSERT(!m_InsideString);
    5262 
    5263  BeginValue(false);
    5264  m_SB.Add('{');
    5265 
    5266  StackItem item;
    5267  item.type = COLLECTION_TYPE_OBJECT;
    5268  item.valueCount = 0;
    5269  item.singleLineMode = singleLine;
    5270  m_Stack.push_back(item);
    5271 }
    5272 
    5273 void VmaJsonWriter::EndObject()
    5274 {
    5275  VMA_ASSERT(!m_InsideString);
    5276 
    5277  WriteIndent(true);
    5278  m_SB.Add('}');
    5279 
    5280  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
    5281  m_Stack.pop_back();
    5282 }
    5283 
    5284 void VmaJsonWriter::BeginArray(bool singleLine)
    5285 {
    5286  VMA_ASSERT(!m_InsideString);
    5287 
    5288  BeginValue(false);
    5289  m_SB.Add('[');
    5290 
    5291  StackItem item;
    5292  item.type = COLLECTION_TYPE_ARRAY;
    5293  item.valueCount = 0;
    5294  item.singleLineMode = singleLine;
    5295  m_Stack.push_back(item);
    5296 }
    5297 
    5298 void VmaJsonWriter::EndArray()
    5299 {
    5300  VMA_ASSERT(!m_InsideString);
    5301 
    5302  WriteIndent(true);
    5303  m_SB.Add(']');
    5304 
    5305  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
    5306  m_Stack.pop_back();
    5307 }
    5308 
    5309 void VmaJsonWriter::WriteString(const char* pStr)
    5310 {
    5311  BeginString(pStr);
    5312  EndString();
    5313 }
    5314 
    5315 void VmaJsonWriter::BeginString(const char* pStr)
    5316 {
    5317  VMA_ASSERT(!m_InsideString);
    5318 
    5319  BeginValue(true);
    5320  m_SB.Add('"');
    5321  m_InsideString = true;
    5322  if(pStr != VMA_NULL && pStr[0] != '\0')
    5323  {
    5324  ContinueString(pStr);
    5325  }
    5326 }
    5327 
    5328 void VmaJsonWriter::ContinueString(const char* pStr)
    5329 {
    5330  VMA_ASSERT(m_InsideString);
    5331 
    5332  const size_t strLen = strlen(pStr);
    5333  for(size_t i = 0; i < strLen; ++i)
    5334  {
    5335  char ch = pStr[i];
    5336  if(ch == '\\')
    5337  {
    5338  m_SB.Add("\\\\");
    5339  }
    5340  else if(ch == '"')
    5341  {
    5342  m_SB.Add("\\\"");
    5343  }
    5344  else if(ch >= 32)
    5345  {
    5346  m_SB.Add(ch);
    5347  }
    5348  else switch(ch)
    5349  {
    5350  case '\b':
    5351  m_SB.Add("\\b");
    5352  break;
    5353  case '\f':
    5354  m_SB.Add("\\f");
    5355  break;
    5356  case '\n':
    5357  m_SB.Add("\\n");
    5358  break;
    5359  case '\r':
    5360  m_SB.Add("\\r");
    5361  break;
    5362  case '\t':
    5363  m_SB.Add("\\t");
    5364  break;
    5365  default:
    5366  VMA_ASSERT(0 && "Character not currently supported.");
    5367  break;
    5368  }
    5369  }
    5370 }
    5371 
    5372 void VmaJsonWriter::ContinueString(uint32_t n)
    5373 {
    5374  VMA_ASSERT(m_InsideString);
    5375  m_SB.AddNumber(n);
    5376 }
    5377 
    5378 void VmaJsonWriter::ContinueString(uint64_t n)
    5379 {
    5380  VMA_ASSERT(m_InsideString);
    5381  m_SB.AddNumber(n);
    5382 }
    5383 
    5384 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
    5385 {
    5386  VMA_ASSERT(m_InsideString);
    5387  m_SB.AddPointer(ptr);
    5388 }
    5389 
    5390 void VmaJsonWriter::EndString(const char* pStr)
    5391 {
    5392  VMA_ASSERT(m_InsideString);
    5393  if(pStr != VMA_NULL && pStr[0] != '\0')
    5394  {
    5395  ContinueString(pStr);
    5396  }
    5397  m_SB.Add('"');
    5398  m_InsideString = false;
    5399 }
    5400 
    5401 void VmaJsonWriter::WriteNumber(uint32_t n)
    5402 {
    5403  VMA_ASSERT(!m_InsideString);
    5404  BeginValue(false);
    5405  m_SB.AddNumber(n);
    5406 }
    5407 
    5408 void VmaJsonWriter::WriteNumber(uint64_t n)
    5409 {
    5410  VMA_ASSERT(!m_InsideString);
    5411  BeginValue(false);
    5412  m_SB.AddNumber(n);
    5413 }
    5414 
    5415 void VmaJsonWriter::WriteBool(bool b)
    5416 {
    5417  VMA_ASSERT(!m_InsideString);
    5418  BeginValue(false);
    5419  m_SB.Add(b ? "true" : "false");
    5420 }
    5421 
    5422 void VmaJsonWriter::WriteNull()
    5423 {
    5424  VMA_ASSERT(!m_InsideString);
    5425  BeginValue(false);
    5426  m_SB.Add("null");
    5427 }
    5428 
    5429 void VmaJsonWriter::BeginValue(bool isString)
    5430 {
    5431  if(!m_Stack.empty())
    5432  {
    5433  StackItem& currItem = m_Stack.back();
    5434  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5435  currItem.valueCount % 2 == 0)
    5436  {
    5437  VMA_ASSERT(isString);
    5438  }
    5439 
    5440  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5441  currItem.valueCount % 2 != 0)
    5442  {
    5443  m_SB.Add(": ");
    5444  }
    5445  else if(currItem.valueCount > 0)
    5446  {
    5447  m_SB.Add(", ");
    5448  WriteIndent();
    5449  }
    5450  else
    5451  {
    5452  WriteIndent();
    5453  }
    5454  ++currItem.valueCount;
    5455  }
    5456 }
    5457 
    5458 void VmaJsonWriter::WriteIndent(bool oneLess)
    5459 {
    5460  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
    5461  {
    5462  m_SB.AddNewLine();
    5463 
    5464  size_t count = m_Stack.size();
    5465  if(count > 0 && oneLess)
    5466  {
    5467  --count;
    5468  }
    5469  for(size_t i = 0; i < count; ++i)
    5470  {
    5471  m_SB.Add(INDENT);
    5472  }
    5473  }
    5474 }
    5475 
    5476 #endif // #if VMA_STATS_STRING_ENABLED
    5477 
    5479 
    5480 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
    5481 {
    5482  if(IsUserDataString())
    5483  {
    5484  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
    5485 
    5486  FreeUserDataString(hAllocator);
    5487 
    5488  if(pUserData != VMA_NULL)
    5489  {
    5490  const char* const newStrSrc = (char*)pUserData;
    5491  const size_t newStrLen = strlen(newStrSrc);
    5492  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
    5493  memcpy(newStrDst, newStrSrc, newStrLen + 1);
    5494  m_pUserData = newStrDst;
    5495  }
    5496  }
    5497  else
    5498  {
    5499  m_pUserData = pUserData;
    5500  }
    5501 }
    5502 
    5503 void VmaAllocation_T::ChangeBlockAllocation(
    5504  VmaAllocator hAllocator,
    5505  VmaDeviceMemoryBlock* block,
    5506  VkDeviceSize offset)
    5507 {
    5508  VMA_ASSERT(block != VMA_NULL);
    5509  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5510 
    5511  // Move mapping reference counter from old block to new block.
    5512  if(block != m_BlockAllocation.m_Block)
    5513  {
    5514  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
    5515  if(IsPersistentMap())
    5516  ++mapRefCount;
    5517  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
    5518  block->Map(hAllocator, mapRefCount, VMA_NULL);
    5519  }
    5520 
    5521  m_BlockAllocation.m_Block = block;
    5522  m_BlockAllocation.m_Offset = offset;
    5523 }
    5524 
    5525 VkDeviceSize VmaAllocation_T::GetOffset() const
    5526 {
    5527  switch(m_Type)
    5528  {
    5529  case ALLOCATION_TYPE_BLOCK:
    5530  return m_BlockAllocation.m_Offset;
    5531  case ALLOCATION_TYPE_DEDICATED:
    5532  return 0;
    5533  default:
    5534  VMA_ASSERT(0);
    5535  return 0;
    5536  }
    5537 }
    5538 
    5539 VkDeviceMemory VmaAllocation_T::GetMemory() const
    5540 {
    5541  switch(m_Type)
    5542  {
    5543  case ALLOCATION_TYPE_BLOCK:
    5544  return m_BlockAllocation.m_Block->GetDeviceMemory();
    5545  case ALLOCATION_TYPE_DEDICATED:
    5546  return m_DedicatedAllocation.m_hMemory;
    5547  default:
    5548  VMA_ASSERT(0);
    5549  return VK_NULL_HANDLE;
    5550  }
    5551 }
    5552 
    5553 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
    5554 {
    5555  switch(m_Type)
    5556  {
    5557  case ALLOCATION_TYPE_BLOCK:
    5558  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
    5559  case ALLOCATION_TYPE_DEDICATED:
    5560  return m_DedicatedAllocation.m_MemoryTypeIndex;
    5561  default:
    5562  VMA_ASSERT(0);
    5563  return UINT32_MAX;
    5564  }
    5565 }
    5566 
    5567 void* VmaAllocation_T::GetMappedData() const
    5568 {
    5569  switch(m_Type)
    5570  {
    5571  case ALLOCATION_TYPE_BLOCK:
    5572  if(m_MapCount != 0)
    5573  {
    5574  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
    5575  VMA_ASSERT(pBlockData != VMA_NULL);
    5576  return (char*)pBlockData + m_BlockAllocation.m_Offset;
    5577  }
    5578  else
    5579  {
    5580  return VMA_NULL;
    5581  }
    5582  break;
    5583  case ALLOCATION_TYPE_DEDICATED:
    5584  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
    5585  return m_DedicatedAllocation.m_pMappedData;
    5586  default:
    5587  VMA_ASSERT(0);
    5588  return VMA_NULL;
    5589  }
    5590 }
    5591 
    5592 bool VmaAllocation_T::CanBecomeLost() const
    5593 {
    5594  switch(m_Type)
    5595  {
    5596  case ALLOCATION_TYPE_BLOCK:
    5597  return m_BlockAllocation.m_CanBecomeLost;
    5598  case ALLOCATION_TYPE_DEDICATED:
    5599  return false;
    5600  default:
    5601  VMA_ASSERT(0);
    5602  return false;
    5603  }
    5604 }
    5605 
    5606 VmaPool VmaAllocation_T::GetPool() const
    5607 {
    5608  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5609  return m_BlockAllocation.m_hPool;
    5610 }
    5611 
    5612 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    5613 {
    5614  VMA_ASSERT(CanBecomeLost());
    5615 
    5616  /*
    5617  Warning: This is a carefully designed algorithm.
    5618  Do not modify unless you really know what you're doing :)
    5619  */
    5620  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
    5621  for(;;)
    5622  {
    5623  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    5624  {
    5625  VMA_ASSERT(0);
    5626  return false;
    5627  }
    5628  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
    5629  {
    5630  return false;
    5631  }
    5632  else // Last use time earlier than current time.
    5633  {
    5634  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
    5635  {
    5636  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
    5637  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
    5638  return true;
    5639  }
    5640  }
    5641  }
    5642 }
    5643 
    5644 #if VMA_STATS_STRING_ENABLED
    5645 
    5646 // Correspond to values of enum VmaSuballocationType.
    5647 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
    5648  "FREE",
    5649  "UNKNOWN",
    5650  "BUFFER",
    5651  "IMAGE_UNKNOWN",
    5652  "IMAGE_LINEAR",
    5653  "IMAGE_OPTIMAL",
    5654 };
    5655 
    5656 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
    5657 {
    5658  json.WriteString("Type");
    5659  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
    5660 
    5661  json.WriteString("Size");
    5662  json.WriteNumber(m_Size);
    5663 
    5664  if(m_pUserData != VMA_NULL)
    5665  {
    5666  json.WriteString("UserData");
    5667  if(IsUserDataString())
    5668  {
    5669  json.WriteString((const char*)m_pUserData);
    5670  }
    5671  else
    5672  {
    5673  json.BeginString();
    5674  json.ContinueString_Pointer(m_pUserData);
    5675  json.EndString();
    5676  }
    5677  }
    5678 
    5679  json.WriteString("CreationFrameIndex");
    5680  json.WriteNumber(m_CreationFrameIndex);
    5681 
    5682  json.WriteString("LastUseFrameIndex");
    5683  json.WriteNumber(GetLastUseFrameIndex());
    5684 
    5685  if(m_BufferImageUsage != 0)
    5686  {
    5687  json.WriteString("Usage");
    5688  json.WriteNumber(m_BufferImageUsage);
    5689  }
    5690 }
    5691 
    5692 #endif
    5693 
    5694 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
    5695 {
    5696  VMA_ASSERT(IsUserDataString());
    5697  if(m_pUserData != VMA_NULL)
    5698  {
    5699  char* const oldStr = (char*)m_pUserData;
    5700  const size_t oldStrLen = strlen(oldStr);
    5701  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
    5702  m_pUserData = VMA_NULL;
    5703  }
    5704 }
    5705 
    5706 void VmaAllocation_T::BlockAllocMap()
    5707 {
    5708  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5709 
    5710  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5711  {
    5712  ++m_MapCount;
    5713  }
    5714  else
    5715  {
    5716  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
    5717  }
    5718 }
    5719 
    5720 void VmaAllocation_T::BlockAllocUnmap()
    5721 {
    5722  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5723 
    5724  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5725  {
    5726  --m_MapCount;
    5727  }
    5728  else
    5729  {
    5730  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
    5731  }
    5732 }
    5733 
    5734 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
    5735 {
    5736  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5737 
    5738  if(m_MapCount != 0)
    5739  {
    5740  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5741  {
    5742  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
    5743  *ppData = m_DedicatedAllocation.m_pMappedData;
    5744  ++m_MapCount;
    5745  return VK_SUCCESS;
    5746  }
    5747  else
    5748  {
    5749  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
    5750  return VK_ERROR_MEMORY_MAP_FAILED;
    5751  }
    5752  }
    5753  else
    5754  {
    5755  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    5756  hAllocator->m_hDevice,
    5757  m_DedicatedAllocation.m_hMemory,
    5758  0, // offset
    5759  VK_WHOLE_SIZE,
    5760  0, // flags
    5761  ppData);
    5762  if(result == VK_SUCCESS)
    5763  {
    5764  m_DedicatedAllocation.m_pMappedData = *ppData;
    5765  m_MapCount = 1;
    5766  }
    5767  return result;
    5768  }
    5769 }
    5770 
    5771 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
    5772 {
    5773  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5774 
    5775  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5776  {
    5777  --m_MapCount;
    5778  if(m_MapCount == 0)
    5779  {
    5780  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
    5781  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
    5782  hAllocator->m_hDevice,
    5783  m_DedicatedAllocation.m_hMemory);
    5784  }
    5785  }
    5786  else
    5787  {
    5788  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
    5789  }
    5790 }
    5791 
    5792 #if VMA_STATS_STRING_ENABLED
    5793 
    5794 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
    5795 {
    5796  json.BeginObject();
    5797 
    5798  json.WriteString("Blocks");
    5799  json.WriteNumber(stat.blockCount);
    5800 
    5801  json.WriteString("Allocations");
    5802  json.WriteNumber(stat.allocationCount);
    5803 
    5804  json.WriteString("UnusedRanges");
    5805  json.WriteNumber(stat.unusedRangeCount);
    5806 
    5807  json.WriteString("UsedBytes");
    5808  json.WriteNumber(stat.usedBytes);
    5809 
    5810  json.WriteString("UnusedBytes");
    5811  json.WriteNumber(stat.unusedBytes);
    5812 
    5813  if(stat.allocationCount > 1)
    5814  {
    5815  json.WriteString("AllocationSize");
    5816  json.BeginObject(true);
    5817  json.WriteString("Min");
    5818  json.WriteNumber(stat.allocationSizeMin);
    5819  json.WriteString("Avg");
    5820  json.WriteNumber(stat.allocationSizeAvg);
    5821  json.WriteString("Max");
    5822  json.WriteNumber(stat.allocationSizeMax);
    5823  json.EndObject();
    5824  }
    5825 
    5826  if(stat.unusedRangeCount > 1)
    5827  {
    5828  json.WriteString("UnusedRangeSize");
    5829  json.BeginObject(true);
    5830  json.WriteString("Min");
    5831  json.WriteNumber(stat.unusedRangeSizeMin);
    5832  json.WriteString("Avg");
    5833  json.WriteNumber(stat.unusedRangeSizeAvg);
    5834  json.WriteString("Max");
    5835  json.WriteNumber(stat.unusedRangeSizeMax);
    5836  json.EndObject();
    5837  }
    5838 
    5839  json.EndObject();
    5840 }
    5841 
    5842 #endif // #if VMA_STATS_STRING_ENABLED
    5843 
    5844 struct VmaSuballocationItemSizeLess
    5845 {
    5846  bool operator()(
    5847  const VmaSuballocationList::iterator lhs,
    5848  const VmaSuballocationList::iterator rhs) const
    5849  {
    5850  return lhs->size < rhs->size;
    5851  }
    5852  bool operator()(
    5853  const VmaSuballocationList::iterator lhs,
    5854  VkDeviceSize rhsSize) const
    5855  {
    5856  return lhs->size < rhsSize;
    5857  }
    5858 };
    5859 
    5861 // class VmaBlockMetadata
    5862 
    5863 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
    5864  m_Size(0),
    5865  m_FreeCount(0),
    5866  m_SumFreeSize(0),
    5867  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
    5868  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
    5869 {
    5870 }
    5871 
    5872 VmaBlockMetadata::~VmaBlockMetadata()
    5873 {
    5874 }
    5875 
    5876 void VmaBlockMetadata::Init(VkDeviceSize size)
    5877 {
    5878  m_Size = size;
    5879  m_FreeCount = 1;
    5880  m_SumFreeSize = size;
    5881 
    5882  VmaSuballocation suballoc = {};
    5883  suballoc.offset = 0;
    5884  suballoc.size = size;
    5885  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    5886  suballoc.hAllocation = VK_NULL_HANDLE;
    5887 
    5888  m_Suballocations.push_back(suballoc);
    5889  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
    5890  --suballocItem;
    5891  m_FreeSuballocationsBySize.push_back(suballocItem);
    5892 }
    5893 
    5894 bool VmaBlockMetadata::Validate() const
    5895 {
    5896  if(m_Suballocations.empty())
    5897  {
    5898  return false;
    5899  }
    5900 
    5901  // Expected offset of new suballocation as calculates from previous ones.
    5902  VkDeviceSize calculatedOffset = 0;
    5903  // Expected number of free suballocations as calculated from traversing their list.
    5904  uint32_t calculatedFreeCount = 0;
    5905  // Expected sum size of free suballocations as calculated from traversing their list.
    5906  VkDeviceSize calculatedSumFreeSize = 0;
    5907  // Expected number of free suballocations that should be registered in
    5908  // m_FreeSuballocationsBySize calculated from traversing their list.
    5909  size_t freeSuballocationsToRegister = 0;
    5910  // True if previous visited suballocation was free.
    5911  bool prevFree = false;
    5912 
    5913  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5914  suballocItem != m_Suballocations.cend();
    5915  ++suballocItem)
    5916  {
    5917  const VmaSuballocation& subAlloc = *suballocItem;
    5918 
    5919  // Actual offset of this suballocation doesn't match expected one.
    5920  if(subAlloc.offset != calculatedOffset)
    5921  {
    5922  return false;
    5923  }
    5924 
    5925  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
    5926  // Two adjacent free suballocations are invalid. They should be merged.
    5927  if(prevFree && currFree)
    5928  {
    5929  return false;
    5930  }
    5931 
    5932  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
    5933  {
    5934  return false;
    5935  }
    5936 
    5937  if(currFree)
    5938  {
    5939  calculatedSumFreeSize += subAlloc.size;
    5940  ++calculatedFreeCount;
    5941  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    5942  {
    5943  ++freeSuballocationsToRegister;
    5944  }
    5945 
    5946  // Margin required between allocations - every free space must be at least that large.
    5947  if(subAlloc.size < VMA_DEBUG_MARGIN)
    5948  {
    5949  return false;
    5950  }
    5951  }
    5952  else
    5953  {
    5954  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
    5955  {
    5956  return false;
    5957  }
    5958  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
    5959  {
    5960  return false;
    5961  }
    5962 
    5963  // Margin required between allocations - previous allocation must be free.
    5964  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
    5965  {
    5966  return false;
    5967  }
    5968  }
    5969 
    5970  calculatedOffset += subAlloc.size;
    5971  prevFree = currFree;
    5972  }
    5973 
    5974  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
    5975  // match expected one.
    5976  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
    5977  {
    5978  return false;
    5979  }
    5980 
    5981  VkDeviceSize lastSize = 0;
    5982  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
    5983  {
    5984  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
    5985 
    5986  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
    5987  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
    5988  {
    5989  return false;
    5990  }
    5991  // They must be sorted by size ascending.
    5992  if(suballocItem->size < lastSize)
    5993  {
    5994  return false;
    5995  }
    5996 
    5997  lastSize = suballocItem->size;
    5998  }
    5999 
    6000  // Check if totals match calculacted values.
    6001  if(!ValidateFreeSuballocationList() ||
    6002  (calculatedOffset != m_Size) ||
    6003  (calculatedSumFreeSize != m_SumFreeSize) ||
    6004  (calculatedFreeCount != m_FreeCount))
    6005  {
    6006  return false;
    6007  }
    6008 
    6009  return true;
    6010 }
    6011 
    6012 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
    6013 {
    6014  if(!m_FreeSuballocationsBySize.empty())
    6015  {
    6016  return m_FreeSuballocationsBySize.back()->size;
    6017  }
    6018  else
    6019  {
    6020  return 0;
    6021  }
    6022 }
    6023 
    6024 bool VmaBlockMetadata::IsEmpty() const
    6025 {
    6026  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
    6027 }
    6028 
    6029 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
    6030 {
    6031  outInfo.blockCount = 1;
    6032 
    6033  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    6034  outInfo.allocationCount = rangeCount - m_FreeCount;
    6035  outInfo.unusedRangeCount = m_FreeCount;
    6036 
    6037  outInfo.unusedBytes = m_SumFreeSize;
    6038  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
    6039 
    6040  outInfo.allocationSizeMin = UINT64_MAX;
    6041  outInfo.allocationSizeMax = 0;
    6042  outInfo.unusedRangeSizeMin = UINT64_MAX;
    6043  outInfo.unusedRangeSizeMax = 0;
    6044 
    6045  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    6046  suballocItem != m_Suballocations.cend();
    6047  ++suballocItem)
    6048  {
    6049  const VmaSuballocation& suballoc = *suballocItem;
    6050  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
    6051  {
    6052  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
    6053  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
    6054  }
    6055  else
    6056  {
    6057  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
    6058  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
    6059  }
    6060  }
    6061 }
    6062 
    6063 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
    6064 {
    6065  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    6066 
    6067  inoutStats.size += m_Size;
    6068  inoutStats.unusedSize += m_SumFreeSize;
    6069  inoutStats.allocationCount += rangeCount - m_FreeCount;
    6070  inoutStats.unusedRangeCount += m_FreeCount;
    6071  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
    6072 }
    6073 
    6074 #if VMA_STATS_STRING_ENABLED
    6075 
    6076 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
    6077 {
    6078  json.BeginObject();
    6079 
    6080  json.WriteString("TotalBytes");
    6081  json.WriteNumber(m_Size);
    6082 
    6083  json.WriteString("UnusedBytes");
    6084  json.WriteNumber(m_SumFreeSize);
    6085 
    6086  json.WriteString("Allocations");
    6087  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
    6088 
    6089  json.WriteString("UnusedRanges");
    6090  json.WriteNumber(m_FreeCount);
    6091 
    6092  json.WriteString("Suballocations");
    6093  json.BeginArray();
    6094  size_t i = 0;
    6095  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    6096  suballocItem != m_Suballocations.cend();
    6097  ++suballocItem, ++i)
    6098  {
    6099  json.BeginObject(true);
    6100 
    6101  json.WriteString("Offset");
    6102  json.WriteNumber(suballocItem->offset);
    6103 
    6104  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6105  {
    6106  json.WriteString("Type");
    6107  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
    6108 
    6109  json.WriteString("Size");
    6110  json.WriteNumber(suballocItem->size);
    6111  }
    6112  else
    6113  {
    6114  suballocItem->hAllocation->PrintParameters(json);
    6115  }
    6116 
    6117  json.EndObject();
    6118  }
    6119  json.EndArray();
    6120 
    6121  json.EndObject();
    6122 }
    6123 
    6124 #endif // #if VMA_STATS_STRING_ENABLED
    6125 
    6126 /*
    6127 How many suitable free suballocations to analyze before choosing best one.
    6128 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
    6129  be chosen.
    6130 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
    6131  suballocations will be analized and best one will be chosen.
    6132 - Any other value is also acceptable.
    6133 */
    6134 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
    6135 
    6136 bool VmaBlockMetadata::CreateAllocationRequest(
    6137  uint32_t currentFrameIndex,
    6138  uint32_t frameInUseCount,
    6139  VkDeviceSize bufferImageGranularity,
    6140  VkDeviceSize allocSize,
    6141  VkDeviceSize allocAlignment,
    6142  VmaSuballocationType allocType,
    6143  bool canMakeOtherLost,
    6144  VmaAllocationRequest* pAllocationRequest)
    6145 {
    6146  VMA_ASSERT(allocSize > 0);
    6147  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6148  VMA_ASSERT(pAllocationRequest != VMA_NULL);
    6149  VMA_HEAVY_ASSERT(Validate());
    6150 
    6151  // There is not enough total free space in this block to fullfill the request: Early return.
    6152  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
    6153  {
    6154  return false;
    6155  }
    6156 
    6157  // New algorithm, efficiently searching freeSuballocationsBySize.
    6158  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
    6159  if(freeSuballocCount > 0)
    6160  {
    6161  if(VMA_BEST_FIT)
    6162  {
    6163  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
    6164  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6165  m_FreeSuballocationsBySize.data(),
    6166  m_FreeSuballocationsBySize.data() + freeSuballocCount,
    6167  allocSize + 2 * VMA_DEBUG_MARGIN,
    6168  VmaSuballocationItemSizeLess());
    6169  size_t index = it - m_FreeSuballocationsBySize.data();
    6170  for(; index < freeSuballocCount; ++index)
    6171  {
    6172  if(CheckAllocation(
    6173  currentFrameIndex,
    6174  frameInUseCount,
    6175  bufferImageGranularity,
    6176  allocSize,
    6177  allocAlignment,
    6178  allocType,
    6179  m_FreeSuballocationsBySize[index],
    6180  false, // canMakeOtherLost
    6181  &pAllocationRequest->offset,
    6182  &pAllocationRequest->itemsToMakeLostCount,
    6183  &pAllocationRequest->sumFreeSize,
    6184  &pAllocationRequest->sumItemSize))
    6185  {
    6186  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6187  return true;
    6188  }
    6189  }
    6190  }
    6191  else
    6192  {
    6193  // Search staring from biggest suballocations.
    6194  for(size_t index = freeSuballocCount; index--; )
    6195  {
    6196  if(CheckAllocation(
    6197  currentFrameIndex,
    6198  frameInUseCount,
    6199  bufferImageGranularity,
    6200  allocSize,
    6201  allocAlignment,
    6202  allocType,
    6203  m_FreeSuballocationsBySize[index],
    6204  false, // canMakeOtherLost
    6205  &pAllocationRequest->offset,
    6206  &pAllocationRequest->itemsToMakeLostCount,
    6207  &pAllocationRequest->sumFreeSize,
    6208  &pAllocationRequest->sumItemSize))
    6209  {
    6210  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6211  return true;
    6212  }
    6213  }
    6214  }
    6215  }
    6216 
    6217  if(canMakeOtherLost)
    6218  {
    6219  // Brute-force algorithm. TODO: Come up with something better.
    6220 
    6221  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
    6222  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
    6223 
    6224  VmaAllocationRequest tmpAllocRequest = {};
    6225  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
    6226  suballocIt != m_Suballocations.end();
    6227  ++suballocIt)
    6228  {
    6229  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
    6230  suballocIt->hAllocation->CanBecomeLost())
    6231  {
    6232  if(CheckAllocation(
    6233  currentFrameIndex,
    6234  frameInUseCount,
    6235  bufferImageGranularity,
    6236  allocSize,
    6237  allocAlignment,
    6238  allocType,
    6239  suballocIt,
    6240  canMakeOtherLost,
    6241  &tmpAllocRequest.offset,
    6242  &tmpAllocRequest.itemsToMakeLostCount,
    6243  &tmpAllocRequest.sumFreeSize,
    6244  &tmpAllocRequest.sumItemSize))
    6245  {
    6246  tmpAllocRequest.item = suballocIt;
    6247 
    6248  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
    6249  {
    6250  *pAllocationRequest = tmpAllocRequest;
    6251  }
    6252  }
    6253  }
    6254  }
    6255 
    6256  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
    6257  {
    6258  return true;
    6259  }
    6260  }
    6261 
    6262  return false;
    6263 }
    6264 
    6265 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
    6266  uint32_t currentFrameIndex,
    6267  uint32_t frameInUseCount,
    6268  VmaAllocationRequest* pAllocationRequest)
    6269 {
    6270  while(pAllocationRequest->itemsToMakeLostCount > 0)
    6271  {
    6272  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
    6273  {
    6274  ++pAllocationRequest->item;
    6275  }
    6276  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6277  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
    6278  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
    6279  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6280  {
    6281  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
    6282  --pAllocationRequest->itemsToMakeLostCount;
    6283  }
    6284  else
    6285  {
    6286  return false;
    6287  }
    6288  }
    6289 
    6290  VMA_HEAVY_ASSERT(Validate());
    6291  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6292  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6293 
    6294  return true;
    6295 }
    6296 
    6297 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    6298 {
    6299  uint32_t lostAllocationCount = 0;
    6300  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6301  it != m_Suballocations.end();
    6302  ++it)
    6303  {
    6304  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
    6305  it->hAllocation->CanBecomeLost() &&
    6306  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6307  {
    6308  it = FreeSuballocation(it);
    6309  ++lostAllocationCount;
    6310  }
    6311  }
    6312  return lostAllocationCount;
    6313 }
    6314 
    6315 VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData)
    6316 {
    6317  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6318  it != m_Suballocations.end();
    6319  ++it)
    6320  {
    6321  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6322  {
    6323  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
    6324  {
    6325  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
    6326  return VK_ERROR_VALIDATION_FAILED_EXT;
    6327  }
    6328  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
    6329  {
    6330  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
    6331  return VK_ERROR_VALIDATION_FAILED_EXT;
    6332  }
    6333  }
    6334  }
    6335 
    6336  return VK_SUCCESS;
    6337 }
    6338 
    6339 void VmaBlockMetadata::Alloc(
    6340  const VmaAllocationRequest& request,
    6341  VmaSuballocationType type,
    6342  VkDeviceSize allocSize,
    6343  VmaAllocation hAllocation)
    6344 {
    6345  VMA_ASSERT(request.item != m_Suballocations.end());
    6346  VmaSuballocation& suballoc = *request.item;
    6347  // Given suballocation is a free block.
    6348  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6349  // Given offset is inside this suballocation.
    6350  VMA_ASSERT(request.offset >= suballoc.offset);
    6351  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
    6352  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
    6353  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
    6354 
    6355  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
    6356  // it to become used.
    6357  UnregisterFreeSuballocation(request.item);
    6358 
    6359  suballoc.offset = request.offset;
    6360  suballoc.size = allocSize;
    6361  suballoc.type = type;
    6362  suballoc.hAllocation = hAllocation;
    6363 
    6364  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
    6365  if(paddingEnd)
    6366  {
    6367  VmaSuballocation paddingSuballoc = {};
    6368  paddingSuballoc.offset = request.offset + allocSize;
    6369  paddingSuballoc.size = paddingEnd;
    6370  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6371  VmaSuballocationList::iterator next = request.item;
    6372  ++next;
    6373  const VmaSuballocationList::iterator paddingEndItem =
    6374  m_Suballocations.insert(next, paddingSuballoc);
    6375  RegisterFreeSuballocation(paddingEndItem);
    6376  }
    6377 
    6378  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
    6379  if(paddingBegin)
    6380  {
    6381  VmaSuballocation paddingSuballoc = {};
    6382  paddingSuballoc.offset = request.offset - paddingBegin;
    6383  paddingSuballoc.size = paddingBegin;
    6384  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6385  const VmaSuballocationList::iterator paddingBeginItem =
    6386  m_Suballocations.insert(request.item, paddingSuballoc);
    6387  RegisterFreeSuballocation(paddingBeginItem);
    6388  }
    6389 
    6390  // Update totals.
    6391  m_FreeCount = m_FreeCount - 1;
    6392  if(paddingBegin > 0)
    6393  {
    6394  ++m_FreeCount;
    6395  }
    6396  if(paddingEnd > 0)
    6397  {
    6398  ++m_FreeCount;
    6399  }
    6400  m_SumFreeSize -= allocSize;
    6401 }
    6402 
    6403 void VmaBlockMetadata::Free(const VmaAllocation allocation)
    6404 {
    6405  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6406  suballocItem != m_Suballocations.end();
    6407  ++suballocItem)
    6408  {
    6409  VmaSuballocation& suballoc = *suballocItem;
    6410  if(suballoc.hAllocation == allocation)
    6411  {
    6412  FreeSuballocation(suballocItem);
    6413  VMA_HEAVY_ASSERT(Validate());
    6414  return;
    6415  }
    6416  }
    6417  VMA_ASSERT(0 && "Not found!");
    6418 }
    6419 
    6420 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
    6421 {
    6422  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6423  suballocItem != m_Suballocations.end();
    6424  ++suballocItem)
    6425  {
    6426  VmaSuballocation& suballoc = *suballocItem;
    6427  if(suballoc.offset == offset)
    6428  {
    6429  FreeSuballocation(suballocItem);
    6430  return;
    6431  }
    6432  }
    6433  VMA_ASSERT(0 && "Not found!");
    6434 }
    6435 
    6436 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
    6437 {
    6438  VkDeviceSize lastSize = 0;
    6439  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
    6440  {
    6441  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
    6442 
    6443  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6444  {
    6445  VMA_ASSERT(0);
    6446  return false;
    6447  }
    6448  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6449  {
    6450  VMA_ASSERT(0);
    6451  return false;
    6452  }
    6453  if(it->size < lastSize)
    6454  {
    6455  VMA_ASSERT(0);
    6456  return false;
    6457  }
    6458 
    6459  lastSize = it->size;
    6460  }
    6461  return true;
    6462 }
    6463 
    6464 bool VmaBlockMetadata::CheckAllocation(
    6465  uint32_t currentFrameIndex,
    6466  uint32_t frameInUseCount,
    6467  VkDeviceSize bufferImageGranularity,
    6468  VkDeviceSize allocSize,
    6469  VkDeviceSize allocAlignment,
    6470  VmaSuballocationType allocType,
    6471  VmaSuballocationList::const_iterator suballocItem,
    6472  bool canMakeOtherLost,
    6473  VkDeviceSize* pOffset,
    6474  size_t* itemsToMakeLostCount,
    6475  VkDeviceSize* pSumFreeSize,
    6476  VkDeviceSize* pSumItemSize) const
    6477 {
    6478  VMA_ASSERT(allocSize > 0);
    6479  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6480  VMA_ASSERT(suballocItem != m_Suballocations.cend());
    6481  VMA_ASSERT(pOffset != VMA_NULL);
    6482 
    6483  *itemsToMakeLostCount = 0;
    6484  *pSumFreeSize = 0;
    6485  *pSumItemSize = 0;
    6486 
    6487  if(canMakeOtherLost)
    6488  {
    6489  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6490  {
    6491  *pSumFreeSize = suballocItem->size;
    6492  }
    6493  else
    6494  {
    6495  if(suballocItem->hAllocation->CanBecomeLost() &&
    6496  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6497  {
    6498  ++*itemsToMakeLostCount;
    6499  *pSumItemSize = suballocItem->size;
    6500  }
    6501  else
    6502  {
    6503  return false;
    6504  }
    6505  }
    6506 
    6507  // Remaining size is too small for this request: Early return.
    6508  if(m_Size - suballocItem->offset < allocSize)
    6509  {
    6510  return false;
    6511  }
    6512 
    6513  // Start from offset equal to beginning of this suballocation.
    6514  *pOffset = suballocItem->offset;
    6515 
    6516  // Apply VMA_DEBUG_MARGIN at the beginning.
    6517  if(VMA_DEBUG_MARGIN > 0)
    6518  {
    6519  *pOffset += VMA_DEBUG_MARGIN;
    6520  }
    6521 
    6522  // Apply alignment.
    6523  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6524 
    6525  // Check previous suballocations for BufferImageGranularity conflicts.
    6526  // Make bigger alignment if necessary.
    6527  if(bufferImageGranularity > 1)
    6528  {
    6529  bool bufferImageGranularityConflict = false;
    6530  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6531  while(prevSuballocItem != m_Suballocations.cbegin())
    6532  {
    6533  --prevSuballocItem;
    6534  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6535  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6536  {
    6537  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6538  {
    6539  bufferImageGranularityConflict = true;
    6540  break;
    6541  }
    6542  }
    6543  else
    6544  // Already on previous page.
    6545  break;
    6546  }
    6547  if(bufferImageGranularityConflict)
    6548  {
    6549  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6550  }
    6551  }
    6552 
    6553  // Now that we have final *pOffset, check if we are past suballocItem.
    6554  // If yes, return false - this function should be called for another suballocItem as starting point.
    6555  if(*pOffset >= suballocItem->offset + suballocItem->size)
    6556  {
    6557  return false;
    6558  }
    6559 
    6560  // Calculate padding at the beginning based on current offset.
    6561  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
    6562 
    6563  // Calculate required margin at the end.
    6564  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6565 
    6566  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
    6567  // Another early return check.
    6568  if(suballocItem->offset + totalSize > m_Size)
    6569  {
    6570  return false;
    6571  }
    6572 
    6573  // Advance lastSuballocItem until desired size is reached.
    6574  // Update itemsToMakeLostCount.
    6575  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
    6576  if(totalSize > suballocItem->size)
    6577  {
    6578  VkDeviceSize remainingSize = totalSize - suballocItem->size;
    6579  while(remainingSize > 0)
    6580  {
    6581  ++lastSuballocItem;
    6582  if(lastSuballocItem == m_Suballocations.cend())
    6583  {
    6584  return false;
    6585  }
    6586  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6587  {
    6588  *pSumFreeSize += lastSuballocItem->size;
    6589  }
    6590  else
    6591  {
    6592  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
    6593  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
    6594  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6595  {
    6596  ++*itemsToMakeLostCount;
    6597  *pSumItemSize += lastSuballocItem->size;
    6598  }
    6599  else
    6600  {
    6601  return false;
    6602  }
    6603  }
    6604  remainingSize = (lastSuballocItem->size < remainingSize) ?
    6605  remainingSize - lastSuballocItem->size : 0;
    6606  }
    6607  }
    6608 
    6609  // Check next suballocations for BufferImageGranularity conflicts.
    6610  // If conflict exists, we must mark more allocations lost or fail.
    6611  if(bufferImageGranularity > 1)
    6612  {
    6613  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
    6614  ++nextSuballocItem;
    6615  while(nextSuballocItem != m_Suballocations.cend())
    6616  {
    6617  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6618  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6619  {
    6620  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6621  {
    6622  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
    6623  if(nextSuballoc.hAllocation->CanBecomeLost() &&
    6624  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6625  {
    6626  ++*itemsToMakeLostCount;
    6627  }
    6628  else
    6629  {
    6630  return false;
    6631  }
    6632  }
    6633  }
    6634  else
    6635  {
    6636  // Already on next page.
    6637  break;
    6638  }
    6639  ++nextSuballocItem;
    6640  }
    6641  }
    6642  }
    6643  else
    6644  {
    6645  const VmaSuballocation& suballoc = *suballocItem;
    6646  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6647 
    6648  *pSumFreeSize = suballoc.size;
    6649 
    6650  // Size of this suballocation is too small for this request: Early return.
    6651  if(suballoc.size < allocSize)
    6652  {
    6653  return false;
    6654  }
    6655 
    6656  // Start from offset equal to beginning of this suballocation.
    6657  *pOffset = suballoc.offset;
    6658 
    6659  // Apply VMA_DEBUG_MARGIN at the beginning.
    6660  if(VMA_DEBUG_MARGIN > 0)
    6661  {
    6662  *pOffset += VMA_DEBUG_MARGIN;
    6663  }
    6664 
    6665  // Apply alignment.
    6666  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6667 
    6668  // Check previous suballocations for BufferImageGranularity conflicts.
    6669  // Make bigger alignment if necessary.
    6670  if(bufferImageGranularity > 1)
    6671  {
    6672  bool bufferImageGranularityConflict = false;
    6673  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6674  while(prevSuballocItem != m_Suballocations.cbegin())
    6675  {
    6676  --prevSuballocItem;
    6677  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6678  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6679  {
    6680  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6681  {
    6682  bufferImageGranularityConflict = true;
    6683  break;
    6684  }
    6685  }
    6686  else
    6687  // Already on previous page.
    6688  break;
    6689  }
    6690  if(bufferImageGranularityConflict)
    6691  {
    6692  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6693  }
    6694  }
    6695 
    6696  // Calculate padding at the beginning based on current offset.
    6697  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
    6698 
    6699  // Calculate required margin at the end.
    6700  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6701 
    6702  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
    6703  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
    6704  {
    6705  return false;
    6706  }
    6707 
    6708  // Check next suballocations for BufferImageGranularity conflicts.
    6709  // If conflict exists, allocation cannot be made here.
    6710  if(bufferImageGranularity > 1)
    6711  {
    6712  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
    6713  ++nextSuballocItem;
    6714  while(nextSuballocItem != m_Suballocations.cend())
    6715  {
    6716  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6717  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6718  {
    6719  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6720  {
    6721  return false;
    6722  }
    6723  }
    6724  else
    6725  {
    6726  // Already on next page.
    6727  break;
    6728  }
    6729  ++nextSuballocItem;
    6730  }
    6731  }
    6732  }
    6733 
    6734  // All tests passed: Success. pOffset is already filled.
    6735  return true;
    6736 }
    6737 
    6738 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
    6739 {
    6740  VMA_ASSERT(item != m_Suballocations.end());
    6741  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6742 
    6743  VmaSuballocationList::iterator nextItem = item;
    6744  ++nextItem;
    6745  VMA_ASSERT(nextItem != m_Suballocations.end());
    6746  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
    6747 
    6748  item->size += nextItem->size;
    6749  --m_FreeCount;
    6750  m_Suballocations.erase(nextItem);
    6751 }
    6752 
    6753 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
    6754 {
    6755  // Change this suballocation to be marked as free.
    6756  VmaSuballocation& suballoc = *suballocItem;
    6757  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6758  suballoc.hAllocation = VK_NULL_HANDLE;
    6759 
    6760  // Update totals.
    6761  ++m_FreeCount;
    6762  m_SumFreeSize += suballoc.size;
    6763 
    6764  // Merge with previous and/or next suballocation if it's also free.
    6765  bool mergeWithNext = false;
    6766  bool mergeWithPrev = false;
    6767 
    6768  VmaSuballocationList::iterator nextItem = suballocItem;
    6769  ++nextItem;
    6770  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
    6771  {
    6772  mergeWithNext = true;
    6773  }
    6774 
    6775  VmaSuballocationList::iterator prevItem = suballocItem;
    6776  if(suballocItem != m_Suballocations.begin())
    6777  {
    6778  --prevItem;
    6779  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6780  {
    6781  mergeWithPrev = true;
    6782  }
    6783  }
    6784 
    6785  if(mergeWithNext)
    6786  {
    6787  UnregisterFreeSuballocation(nextItem);
    6788  MergeFreeWithNext(suballocItem);
    6789  }
    6790 
    6791  if(mergeWithPrev)
    6792  {
    6793  UnregisterFreeSuballocation(prevItem);
    6794  MergeFreeWithNext(prevItem);
    6795  RegisterFreeSuballocation(prevItem);
    6796  return prevItem;
    6797  }
    6798  else
    6799  {
    6800  RegisterFreeSuballocation(suballocItem);
    6801  return suballocItem;
    6802  }
    6803 }
    6804 
    6805 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
    6806 {
    6807  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6808  VMA_ASSERT(item->size > 0);
    6809 
    6810  // You may want to enable this validation at the beginning or at the end of
    6811  // this function, depending on what do you want to check.
    6812  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6813 
    6814  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6815  {
    6816  if(m_FreeSuballocationsBySize.empty())
    6817  {
    6818  m_FreeSuballocationsBySize.push_back(item);
    6819  }
    6820  else
    6821  {
    6822  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
    6823  }
    6824  }
    6825 
    6826  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6827 }
    6828 
    6829 
    6830 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
    6831 {
    6832  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6833  VMA_ASSERT(item->size > 0);
    6834 
    6835  // You may want to enable this validation at the beginning or at the end of
    6836  // this function, depending on what do you want to check.
    6837  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6838 
    6839  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6840  {
    6841  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6842  m_FreeSuballocationsBySize.data(),
    6843  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
    6844  item,
    6845  VmaSuballocationItemSizeLess());
    6846  for(size_t index = it - m_FreeSuballocationsBySize.data();
    6847  index < m_FreeSuballocationsBySize.size();
    6848  ++index)
    6849  {
    6850  if(m_FreeSuballocationsBySize[index] == item)
    6851  {
    6852  VmaVectorRemove(m_FreeSuballocationsBySize, index);
    6853  return;
    6854  }
    6855  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
    6856  }
    6857  VMA_ASSERT(0 && "Not found.");
    6858  }
    6859 
    6860  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6861 }
    6862 
    6864 // class VmaDeviceMemoryBlock
    6865 
    6866 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
    6867  m_Metadata(hAllocator),
    6868  m_MemoryTypeIndex(UINT32_MAX),
    6869  m_Id(0),
    6870  m_hMemory(VK_NULL_HANDLE),
    6871  m_MapCount(0),
    6872  m_pMappedData(VMA_NULL)
    6873 {
    6874 }
    6875 
    6876 void VmaDeviceMemoryBlock::Init(
    6877  uint32_t newMemoryTypeIndex,
    6878  VkDeviceMemory newMemory,
    6879  VkDeviceSize newSize,
    6880  uint32_t id)
    6881 {
    6882  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    6883 
    6884  m_MemoryTypeIndex = newMemoryTypeIndex;
    6885  m_Id = id;
    6886  m_hMemory = newMemory;
    6887 
    6888  m_Metadata.Init(newSize);
    6889 }
    6890 
    6891 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
    6892 {
    6893  // This is the most important assert in the entire library.
    6894  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
    6895  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
    6896 
    6897  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
    6898  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
    6899  m_hMemory = VK_NULL_HANDLE;
    6900 }
    6901 
    6902 bool VmaDeviceMemoryBlock::Validate() const
    6903 {
    6904  if((m_hMemory == VK_NULL_HANDLE) ||
    6905  (m_Metadata.GetSize() == 0))
    6906  {
    6907  return false;
    6908  }
    6909 
    6910  return m_Metadata.Validate();
    6911 }
    6912 
    6913 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
    6914 {
    6915  void* pData = nullptr;
    6916  VkResult res = Map(hAllocator, 1, &pData);
    6917  if(res != VK_SUCCESS)
    6918  {
    6919  return res;
    6920  }
    6921 
    6922  res = m_Metadata.CheckCorruption(pData);
    6923 
    6924  Unmap(hAllocator, 1);
    6925 
    6926  return res;
    6927 }
    6928 
    6929 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
    6930 {
    6931  if(count == 0)
    6932  {
    6933  return VK_SUCCESS;
    6934  }
    6935 
    6936  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6937  if(m_MapCount != 0)
    6938  {
    6939  m_MapCount += count;
    6940  VMA_ASSERT(m_pMappedData != VMA_NULL);
    6941  if(ppData != VMA_NULL)
    6942  {
    6943  *ppData = m_pMappedData;
    6944  }
    6945  return VK_SUCCESS;
    6946  }
    6947  else
    6948  {
    6949  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    6950  hAllocator->m_hDevice,
    6951  m_hMemory,
    6952  0, // offset
    6953  VK_WHOLE_SIZE,
    6954  0, // flags
    6955  &m_pMappedData);
    6956  if(result == VK_SUCCESS)
    6957  {
    6958  if(ppData != VMA_NULL)
    6959  {
    6960  *ppData = m_pMappedData;
    6961  }
    6962  m_MapCount = count;
    6963  }
    6964  return result;
    6965  }
    6966 }
    6967 
    6968 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
    6969 {
    6970  if(count == 0)
    6971  {
    6972  return;
    6973  }
    6974 
    6975  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6976  if(m_MapCount >= count)
    6977  {
    6978  m_MapCount -= count;
    6979  if(m_MapCount == 0)
    6980  {
    6981  m_pMappedData = VMA_NULL;
    6982  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
    6983  }
    6984  }
    6985  else
    6986  {
    6987  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
    6988  }
    6989 }
    6990 
    6991 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6992 {
    6993  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    6994  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    6995 
    6996  void* pData;
    6997  VkResult res = Map(hAllocator, 1, &pData);
    6998  if(res != VK_SUCCESS)
    6999  {
    7000  return res;
    7001  }
    7002 
    7003  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
    7004  VmaWriteMagicValue(pData, allocOffset + allocSize);
    7005 
    7006  Unmap(hAllocator, 1);
    7007 
    7008  return VK_SUCCESS;
    7009 }
    7010 
    7011 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    7012 {
    7013  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    7014  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    7015 
    7016  void* pData;
    7017  VkResult res = Map(hAllocator, 1, &pData);
    7018  if(res != VK_SUCCESS)
    7019  {
    7020  return res;
    7021  }
    7022 
    7023  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
    7024  {
    7025  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
    7026  }
    7027  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
    7028  {
    7029  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
    7030  }
    7031 
    7032  Unmap(hAllocator, 1);
    7033 
    7034  return VK_SUCCESS;
    7035 }
    7036 
    7037 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
    7038  const VmaAllocator hAllocator,
    7039  const VmaAllocation hAllocation,
    7040  VkBuffer hBuffer)
    7041 {
    7042  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    7043  hAllocation->GetBlock() == this);
    7044  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    7045  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    7046  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
    7047  hAllocator->m_hDevice,
    7048  hBuffer,
    7049  m_hMemory,
    7050  hAllocation->GetOffset());
    7051 }
    7052 
    7053 VkResult VmaDeviceMemoryBlock::BindImageMemory(
    7054  const VmaAllocator hAllocator,
    7055  const VmaAllocation hAllocation,
    7056  VkImage hImage)
    7057 {
    7058  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    7059  hAllocation->GetBlock() == this);
    7060  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    7061  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    7062  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
    7063  hAllocator->m_hDevice,
    7064  hImage,
    7065  m_hMemory,
    7066  hAllocation->GetOffset());
    7067 }
    7068 
    7069 static void InitStatInfo(VmaStatInfo& outInfo)
    7070 {
    7071  memset(&outInfo, 0, sizeof(outInfo));
    7072  outInfo.allocationSizeMin = UINT64_MAX;
    7073  outInfo.unusedRangeSizeMin = UINT64_MAX;
    7074 }
    7075 
    7076 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
    7077 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
    7078 {
    7079  inoutInfo.blockCount += srcInfo.blockCount;
    7080  inoutInfo.allocationCount += srcInfo.allocationCount;
    7081  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
    7082  inoutInfo.usedBytes += srcInfo.usedBytes;
    7083  inoutInfo.unusedBytes += srcInfo.unusedBytes;
    7084  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
    7085  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
    7086  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
    7087  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
    7088 }
    7089 
    7090 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
    7091 {
    7092  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
    7093  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
    7094  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
    7095  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
    7096 }
    7097 
    7098 VmaPool_T::VmaPool_T(
    7099  VmaAllocator hAllocator,
    7100  const VmaPoolCreateInfo& createInfo) :
    7101  m_BlockVector(
    7102  hAllocator,
    7103  createInfo.memoryTypeIndex,
    7104  createInfo.blockSize,
    7105  createInfo.minBlockCount,
    7106  createInfo.maxBlockCount,
    7107  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
    7108  createInfo.frameInUseCount,
    7109  true), // isCustomPool
    7110  m_Id(0)
    7111 {
    7112 }
    7113 
    7114 VmaPool_T::~VmaPool_T()
    7115 {
    7116 }
    7117 
    7118 #if VMA_STATS_STRING_ENABLED
    7119 
    7120 #endif // #if VMA_STATS_STRING_ENABLED
    7121 
    7122 VmaBlockVector::VmaBlockVector(
    7123  VmaAllocator hAllocator,
    7124  uint32_t memoryTypeIndex,
    7125  VkDeviceSize preferredBlockSize,
    7126  size_t minBlockCount,
    7127  size_t maxBlockCount,
    7128  VkDeviceSize bufferImageGranularity,
    7129  uint32_t frameInUseCount,
    7130  bool isCustomPool) :
    7131  m_hAllocator(hAllocator),
    7132  m_MemoryTypeIndex(memoryTypeIndex),
    7133  m_PreferredBlockSize(preferredBlockSize),
    7134  m_MinBlockCount(minBlockCount),
    7135  m_MaxBlockCount(maxBlockCount),
    7136  m_BufferImageGranularity(bufferImageGranularity),
    7137  m_FrameInUseCount(frameInUseCount),
    7138  m_IsCustomPool(isCustomPool),
    7139  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
    7140  m_HasEmptyBlock(false),
    7141  m_pDefragmentator(VMA_NULL),
    7142  m_NextBlockId(0)
    7143 {
    7144 }
    7145 
    7146 VmaBlockVector::~VmaBlockVector()
    7147 {
    7148  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
    7149 
    7150  for(size_t i = m_Blocks.size(); i--; )
    7151  {
    7152  m_Blocks[i]->Destroy(m_hAllocator);
    7153  vma_delete(m_hAllocator, m_Blocks[i]);
    7154  }
    7155 }
    7156 
    7157 VkResult VmaBlockVector::CreateMinBlocks()
    7158 {
    7159  for(size_t i = 0; i < m_MinBlockCount; ++i)
    7160  {
    7161  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
    7162  if(res != VK_SUCCESS)
    7163  {
    7164  return res;
    7165  }
    7166  }
    7167  return VK_SUCCESS;
    7168 }
    7169 
    7170 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
    7171 {
    7172  pStats->size = 0;
    7173  pStats->unusedSize = 0;
    7174  pStats->allocationCount = 0;
    7175  pStats->unusedRangeCount = 0;
    7176  pStats->unusedRangeSizeMax = 0;
    7177 
    7178  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7179 
    7180  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7181  {
    7182  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7183  VMA_ASSERT(pBlock);
    7184  VMA_HEAVY_ASSERT(pBlock->Validate());
    7185  pBlock->m_Metadata.AddPoolStats(*pStats);
    7186  }
    7187 }
    7188 
    7189 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
    7190 {
    7191  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    7192  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
    7193  (VMA_DEBUG_MARGIN > 0) &&
    7194  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
    7195 }
    7196 
    7197 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
    7198 
    7199 VkResult VmaBlockVector::Allocate(
    7200  VmaPool hCurrentPool,
    7201  uint32_t currentFrameIndex,
    7202  VkDeviceSize size,
    7203  VkDeviceSize alignment,
    7204  const VmaAllocationCreateInfo& createInfo,
    7205  VmaSuballocationType suballocType,
    7206  VmaAllocation* pAllocation)
    7207 {
    7208  // Early reject: requested allocation size is larger that maximum block size for this block vector.
    7209  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
    7210  {
    7211  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7212  }
    7213 
    7214  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    7215  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
    7216 
    7217  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7218 
    7219  // 1. Search existing allocations. Try to allocate without making other allocations lost.
    7220  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7221  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7222  {
    7223  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7224  VMA_ASSERT(pCurrBlock);
    7225  VmaAllocationRequest currRequest = {};
    7226  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7227  currentFrameIndex,
    7228  m_FrameInUseCount,
    7229  m_BufferImageGranularity,
    7230  size,
    7231  alignment,
    7232  suballocType,
    7233  false, // canMakeOtherLost
    7234  &currRequest))
    7235  {
    7236  // Allocate from pCurrBlock.
    7237  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
    7238 
    7239  if(mapped)
    7240  {
    7241  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
    7242  if(res != VK_SUCCESS)
    7243  {
    7244  return res;
    7245  }
    7246  }
    7247 
    7248  // We no longer have an empty Allocation.
    7249  if(pCurrBlock->m_Metadata.IsEmpty())
    7250  {
    7251  m_HasEmptyBlock = false;
    7252  }
    7253 
    7254  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7255  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation);
    7256  (*pAllocation)->InitBlockAllocation(
    7257  hCurrentPool,
    7258  pCurrBlock,
    7259  currRequest.offset,
    7260  alignment,
    7261  size,
    7262  suballocType,
    7263  mapped,
    7264  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7265  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
    7266  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7267  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7268  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7269  {
    7270  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7271  }
    7272  if(IsCorruptionDetectionEnabled())
    7273  {
    7274  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
    7275  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7276  }
    7277  return VK_SUCCESS;
    7278  }
    7279  }
    7280 
    7281  const bool canCreateNewBlock =
    7282  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
    7283  (m_Blocks.size() < m_MaxBlockCount);
    7284 
    7285  // 2. Try to create new block.
    7286  if(canCreateNewBlock)
    7287  {
    7288  // Calculate optimal size for new block.
    7289  VkDeviceSize newBlockSize = m_PreferredBlockSize;
    7290  uint32_t newBlockSizeShift = 0;
    7291  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
    7292 
    7293  // Allocating blocks of other sizes is allowed only in default pools.
    7294  // In custom pools block size is fixed.
    7295  if(m_IsCustomPool == false)
    7296  {
    7297  // Allocate 1/8, 1/4, 1/2 as first blocks.
    7298  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
    7299  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
    7300  {
    7301  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7302  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
    7303  {
    7304  newBlockSize = smallerNewBlockSize;
    7305  ++newBlockSizeShift;
    7306  }
    7307  else
    7308  {
    7309  break;
    7310  }
    7311  }
    7312  }
    7313 
    7314  size_t newBlockIndex = 0;
    7315  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
    7316  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
    7317  if(m_IsCustomPool == false)
    7318  {
    7319  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
    7320  {
    7321  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7322  if(smallerNewBlockSize >= size)
    7323  {
    7324  newBlockSize = smallerNewBlockSize;
    7325  ++newBlockSizeShift;
    7326  res = CreateBlock(newBlockSize, &newBlockIndex);
    7327  }
    7328  else
    7329  {
    7330  break;
    7331  }
    7332  }
    7333  }
    7334 
    7335  if(res == VK_SUCCESS)
    7336  {
    7337  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
    7338  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size);
    7339 
    7340  if(mapped)
    7341  {
    7342  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
    7343  if(res != VK_SUCCESS)
    7344  {
    7345  return res;
    7346  }
    7347  }
    7348 
    7349  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
    7350  VmaAllocationRequest allocRequest;
    7351  if(pBlock->m_Metadata.CreateAllocationRequest(
    7352  currentFrameIndex,
    7353  m_FrameInUseCount,
    7354  m_BufferImageGranularity,
    7355  size,
    7356  alignment,
    7357  suballocType,
    7358  false, // canMakeOtherLost
    7359  &allocRequest))
    7360  {
    7361  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7362  pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation);
    7363  (*pAllocation)->InitBlockAllocation(
    7364  hCurrentPool,
    7365  pBlock,
    7366  allocRequest.offset,
    7367  alignment,
    7368  size,
    7369  suballocType,
    7370  mapped,
    7371  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7372  VMA_HEAVY_ASSERT(pBlock->Validate());
    7373  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
    7374  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7375  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7376  {
    7377  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7378  }
    7379  if(IsCorruptionDetectionEnabled())
    7380  {
    7381  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
    7382  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7383  }
    7384  return VK_SUCCESS;
    7385  }
    7386  else
    7387  {
    7388  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
    7389  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7390  }
    7391  }
    7392  }
    7393 
    7394  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
    7395 
    7396  // 3. Try to allocate from existing blocks with making other allocations lost.
    7397  if(canMakeOtherLost)
    7398  {
    7399  uint32_t tryIndex = 0;
    7400  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
    7401  {
    7402  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
    7403  VmaAllocationRequest bestRequest = {};
    7404  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
    7405 
    7406  // 1. Search existing allocations.
    7407  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7408  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7409  {
    7410  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7411  VMA_ASSERT(pCurrBlock);
    7412  VmaAllocationRequest currRequest = {};
    7413  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7414  currentFrameIndex,
    7415  m_FrameInUseCount,
    7416  m_BufferImageGranularity,
    7417  size,
    7418  alignment,
    7419  suballocType,
    7420  canMakeOtherLost,
    7421  &currRequest))
    7422  {
    7423  const VkDeviceSize currRequestCost = currRequest.CalcCost();
    7424  if(pBestRequestBlock == VMA_NULL ||
    7425  currRequestCost < bestRequestCost)
    7426  {
    7427  pBestRequestBlock = pCurrBlock;
    7428  bestRequest = currRequest;
    7429  bestRequestCost = currRequestCost;
    7430 
    7431  if(bestRequestCost == 0)
    7432  {
    7433  break;
    7434  }
    7435  }
    7436  }
    7437  }
    7438 
    7439  if(pBestRequestBlock != VMA_NULL)
    7440  {
    7441  if(mapped)
    7442  {
    7443  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
    7444  if(res != VK_SUCCESS)
    7445  {
    7446  return res;
    7447  }
    7448  }
    7449 
    7450  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
    7451  currentFrameIndex,
    7452  m_FrameInUseCount,
    7453  &bestRequest))
    7454  {
    7455  // We no longer have an empty Allocation.
    7456  if(pBestRequestBlock->m_Metadata.IsEmpty())
    7457  {
    7458  m_HasEmptyBlock = false;
    7459  }
    7460  // Allocate from this pBlock.
    7461  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7462  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation);
    7463  (*pAllocation)->InitBlockAllocation(
    7464  hCurrentPool,
    7465  pBestRequestBlock,
    7466  bestRequest.offset,
    7467  alignment,
    7468  size,
    7469  suballocType,
    7470  mapped,
    7471  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7472  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
    7473  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7474  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7475  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7476  {
    7477  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7478  }
    7479  if(IsCorruptionDetectionEnabled())
    7480  {
    7481  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
    7482  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7483  }
    7484  return VK_SUCCESS;
    7485  }
    7486  // else: Some allocations must have been touched while we are here. Next try.
    7487  }
    7488  else
    7489  {
    7490  // Could not find place in any of the blocks - break outer loop.
    7491  break;
    7492  }
    7493  }
    7494  /* Maximum number of tries exceeded - a very unlike event when many other
    7495  threads are simultaneously touching allocations making it impossible to make
    7496  lost at the same time as we try to allocate. */
    7497  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
    7498  {
    7499  return VK_ERROR_TOO_MANY_OBJECTS;
    7500  }
    7501  }
    7502 
    7503  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7504 }
    7505 
    7506 void VmaBlockVector::Free(
    7507  VmaAllocation hAllocation)
    7508 {
    7509  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
    7510 
    7511  // Scope for lock.
    7512  {
    7513  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7514 
    7515  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    7516 
    7517  if(IsCorruptionDetectionEnabled())
    7518  {
    7519  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
    7520  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
    7521  }
    7522 
    7523  if(hAllocation->IsPersistentMap())
    7524  {
    7525  pBlock->Unmap(m_hAllocator, 1);
    7526  }
    7527 
    7528  pBlock->m_Metadata.Free(hAllocation);
    7529  VMA_HEAVY_ASSERT(pBlock->Validate());
    7530 
    7531  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
    7532 
    7533  // pBlock became empty after this deallocation.
    7534  if(pBlock->m_Metadata.IsEmpty())
    7535  {
    7536  // Already has empty Allocation. We don't want to have two, so delete this one.
    7537  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
    7538  {
    7539  pBlockToDelete = pBlock;
    7540  Remove(pBlock);
    7541  }
    7542  // We now have first empty block.
    7543  else
    7544  {
    7545  m_HasEmptyBlock = true;
    7546  }
    7547  }
    7548  // pBlock didn't become empty, but we have another empty block - find and free that one.
    7549  // (This is optional, heuristics.)
    7550  else if(m_HasEmptyBlock)
    7551  {
    7552  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
    7553  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
    7554  {
    7555  pBlockToDelete = pLastBlock;
    7556  m_Blocks.pop_back();
    7557  m_HasEmptyBlock = false;
    7558  }
    7559  }
    7560 
    7561  IncrementallySortBlocks();
    7562  }
    7563 
    7564  // Destruction of a free Allocation. Deferred until this point, outside of mutex
    7565  // lock, for performance reason.
    7566  if(pBlockToDelete != VMA_NULL)
    7567  {
    7568  VMA_DEBUG_LOG(" Deleted empty allocation");
    7569  pBlockToDelete->Destroy(m_hAllocator);
    7570  vma_delete(m_hAllocator, pBlockToDelete);
    7571  }
    7572 }
    7573 
    7574 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
    7575 {
    7576  VkDeviceSize result = 0;
    7577  for(size_t i = m_Blocks.size(); i--; )
    7578  {
    7579  result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize());
    7580  if(result >= m_PreferredBlockSize)
    7581  {
    7582  break;
    7583  }
    7584  }
    7585  return result;
    7586 }
    7587 
    7588 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
    7589 {
    7590  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7591  {
    7592  if(m_Blocks[blockIndex] == pBlock)
    7593  {
    7594  VmaVectorRemove(m_Blocks, blockIndex);
    7595  return;
    7596  }
    7597  }
    7598  VMA_ASSERT(0);
    7599 }
    7600 
    7601 void VmaBlockVector::IncrementallySortBlocks()
    7602 {
    7603  // Bubble sort only until first swap.
    7604  for(size_t i = 1; i < m_Blocks.size(); ++i)
    7605  {
    7606  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
    7607  {
    7608  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
    7609  return;
    7610  }
    7611  }
    7612 }
    7613 
    7614 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
    7615 {
    7616  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    7617  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
    7618  allocInfo.allocationSize = blockSize;
    7619  VkDeviceMemory mem = VK_NULL_HANDLE;
    7620  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
    7621  if(res < 0)
    7622  {
    7623  return res;
    7624  }
    7625 
    7626  // New VkDeviceMemory successfully created.
    7627 
    7628  // Create new Allocation for it.
    7629  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
    7630  pBlock->Init(
    7631  m_MemoryTypeIndex,
    7632  mem,
    7633  allocInfo.allocationSize,
    7634  m_NextBlockId++);
    7635 
    7636  m_Blocks.push_back(pBlock);
    7637  if(pNewBlockIndex != VMA_NULL)
    7638  {
    7639  *pNewBlockIndex = m_Blocks.size() - 1;
    7640  }
    7641 
    7642  return VK_SUCCESS;
    7643 }
    7644 
    7645 #if VMA_STATS_STRING_ENABLED
    7646 
    7647 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
    7648 {
    7649  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7650 
    7651  json.BeginObject();
    7652 
    7653  if(m_IsCustomPool)
    7654  {
    7655  json.WriteString("MemoryTypeIndex");
    7656  json.WriteNumber(m_MemoryTypeIndex);
    7657 
    7658  json.WriteString("BlockSize");
    7659  json.WriteNumber(m_PreferredBlockSize);
    7660 
    7661  json.WriteString("BlockCount");
    7662  json.BeginObject(true);
    7663  if(m_MinBlockCount > 0)
    7664  {
    7665  json.WriteString("Min");
    7666  json.WriteNumber((uint64_t)m_MinBlockCount);
    7667  }
    7668  if(m_MaxBlockCount < SIZE_MAX)
    7669  {
    7670  json.WriteString("Max");
    7671  json.WriteNumber((uint64_t)m_MaxBlockCount);
    7672  }
    7673  json.WriteString("Cur");
    7674  json.WriteNumber((uint64_t)m_Blocks.size());
    7675  json.EndObject();
    7676 
    7677  if(m_FrameInUseCount > 0)
    7678  {
    7679  json.WriteString("FrameInUseCount");
    7680  json.WriteNumber(m_FrameInUseCount);
    7681  }
    7682  }
    7683  else
    7684  {
    7685  json.WriteString("PreferredBlockSize");
    7686  json.WriteNumber(m_PreferredBlockSize);
    7687  }
    7688 
    7689  json.WriteString("Blocks");
    7690  json.BeginObject();
    7691  for(size_t i = 0; i < m_Blocks.size(); ++i)
    7692  {
    7693  json.BeginString();
    7694  json.ContinueString(m_Blocks[i]->GetId());
    7695  json.EndString();
    7696 
    7697  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
    7698  }
    7699  json.EndObject();
    7700 
    7701  json.EndObject();
    7702 }
    7703 
    7704 #endif // #if VMA_STATS_STRING_ENABLED
    7705 
    7706 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
    7707  VmaAllocator hAllocator,
    7708  uint32_t currentFrameIndex)
    7709 {
    7710  if(m_pDefragmentator == VMA_NULL)
    7711  {
    7712  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
    7713  hAllocator,
    7714  this,
    7715  currentFrameIndex);
    7716  }
    7717 
    7718  return m_pDefragmentator;
    7719 }
    7720 
    7721 VkResult VmaBlockVector::Defragment(
    7722  VmaDefragmentationStats* pDefragmentationStats,
    7723  VkDeviceSize& maxBytesToMove,
    7724  uint32_t& maxAllocationsToMove)
    7725 {
    7726  if(m_pDefragmentator == VMA_NULL)
    7727  {
    7728  return VK_SUCCESS;
    7729  }
    7730 
    7731  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7732 
    7733  // Defragment.
    7734  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
    7735 
    7736  // Accumulate statistics.
    7737  if(pDefragmentationStats != VMA_NULL)
    7738  {
    7739  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
    7740  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
    7741  pDefragmentationStats->bytesMoved += bytesMoved;
    7742  pDefragmentationStats->allocationsMoved += allocationsMoved;
    7743  VMA_ASSERT(bytesMoved <= maxBytesToMove);
    7744  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
    7745  maxBytesToMove -= bytesMoved;
    7746  maxAllocationsToMove -= allocationsMoved;
    7747  }
    7748 
    7749  // Free empty blocks.
    7750  m_HasEmptyBlock = false;
    7751  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
    7752  {
    7753  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
    7754  if(pBlock->m_Metadata.IsEmpty())
    7755  {
    7756  if(m_Blocks.size() > m_MinBlockCount)
    7757  {
    7758  if(pDefragmentationStats != VMA_NULL)
    7759  {
    7760  ++pDefragmentationStats->deviceMemoryBlocksFreed;
    7761  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
    7762  }
    7763 
    7764  VmaVectorRemove(m_Blocks, blockIndex);
    7765  pBlock->Destroy(m_hAllocator);
    7766  vma_delete(m_hAllocator, pBlock);
    7767  }
    7768  else
    7769  {
    7770  m_HasEmptyBlock = true;
    7771  }
    7772  }
    7773  }
    7774 
    7775  return result;
    7776 }
    7777 
    7778 void VmaBlockVector::DestroyDefragmentator()
    7779 {
    7780  if(m_pDefragmentator != VMA_NULL)
    7781  {
    7782  vma_delete(m_hAllocator, m_pDefragmentator);
    7783  m_pDefragmentator = VMA_NULL;
    7784  }
    7785 }
    7786 
    7787 void VmaBlockVector::MakePoolAllocationsLost(
    7788  uint32_t currentFrameIndex,
    7789  size_t* pLostAllocationCount)
    7790 {
    7791  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7792  size_t lostAllocationCount = 0;
    7793  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7794  {
    7795  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7796  VMA_ASSERT(pBlock);
    7797  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
    7798  }
    7799  if(pLostAllocationCount != VMA_NULL)
    7800  {
    7801  *pLostAllocationCount = lostAllocationCount;
    7802  }
    7803 }
    7804 
    7805 VkResult VmaBlockVector::CheckCorruption()
    7806 {
    7807  if(!IsCorruptionDetectionEnabled())
    7808  {
    7809  return VK_ERROR_FEATURE_NOT_PRESENT;
    7810  }
    7811 
    7812  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7813  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7814  {
    7815  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7816  VMA_ASSERT(pBlock);
    7817  VkResult res = pBlock->CheckCorruption(m_hAllocator);
    7818  if(res != VK_SUCCESS)
    7819  {
    7820  return res;
    7821  }
    7822  }
    7823  return VK_SUCCESS;
    7824 }
    7825 
    7826 void VmaBlockVector::AddStats(VmaStats* pStats)
    7827 {
    7828  const uint32_t memTypeIndex = m_MemoryTypeIndex;
    7829  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
    7830 
    7831  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7832 
    7833  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7834  {
    7835  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7836  VMA_ASSERT(pBlock);
    7837  VMA_HEAVY_ASSERT(pBlock->Validate());
    7838  VmaStatInfo allocationStatInfo;
    7839  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
    7840  VmaAddStatInfo(pStats->total, allocationStatInfo);
    7841  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    7842  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    7843  }
    7844 }
    7845 
    7847 // VmaDefragmentator members definition
    7848 
    7849 VmaDefragmentator::VmaDefragmentator(
    7850  VmaAllocator hAllocator,
    7851  VmaBlockVector* pBlockVector,
    7852  uint32_t currentFrameIndex) :
    7853  m_hAllocator(hAllocator),
    7854  m_pBlockVector(pBlockVector),
    7855  m_CurrentFrameIndex(currentFrameIndex),
    7856  m_BytesMoved(0),
    7857  m_AllocationsMoved(0),
    7858  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
    7859  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
    7860 {
    7861 }
    7862 
    7863 VmaDefragmentator::~VmaDefragmentator()
    7864 {
    7865  for(size_t i = m_Blocks.size(); i--; )
    7866  {
    7867  vma_delete(m_hAllocator, m_Blocks[i]);
    7868  }
    7869 }
    7870 
    7871 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
    7872 {
    7873  AllocationInfo allocInfo;
    7874  allocInfo.m_hAllocation = hAlloc;
    7875  allocInfo.m_pChanged = pChanged;
    7876  m_Allocations.push_back(allocInfo);
    7877 }
    7878 
    7879 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
    7880 {
    7881  // It has already been mapped for defragmentation.
    7882  if(m_pMappedDataForDefragmentation)
    7883  {
    7884  *ppMappedData = m_pMappedDataForDefragmentation;
    7885  return VK_SUCCESS;
    7886  }
    7887 
    7888  // It is originally mapped.
    7889  if(m_pBlock->GetMappedData())
    7890  {
    7891  *ppMappedData = m_pBlock->GetMappedData();
    7892  return VK_SUCCESS;
    7893  }
    7894 
    7895  // Map on first usage.
    7896  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
    7897  *ppMappedData = m_pMappedDataForDefragmentation;
    7898  return res;
    7899 }
    7900 
    7901 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
    7902 {
    7903  if(m_pMappedDataForDefragmentation != VMA_NULL)
    7904  {
    7905  m_pBlock->Unmap(hAllocator, 1);
    7906  }
    7907 }
    7908 
    7909 VkResult VmaDefragmentator::DefragmentRound(
    7910  VkDeviceSize maxBytesToMove,
    7911  uint32_t maxAllocationsToMove)
    7912 {
    7913  if(m_Blocks.empty())
    7914  {
    7915  return VK_SUCCESS;
    7916  }
    7917 
    7918  size_t srcBlockIndex = m_Blocks.size() - 1;
    7919  size_t srcAllocIndex = SIZE_MAX;
    7920  for(;;)
    7921  {
    7922  // 1. Find next allocation to move.
    7923  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
    7924  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
    7925  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
    7926  {
    7927  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
    7928  {
    7929  // Finished: no more allocations to process.
    7930  if(srcBlockIndex == 0)
    7931  {
    7932  return VK_SUCCESS;
    7933  }
    7934  else
    7935  {
    7936  --srcBlockIndex;
    7937  srcAllocIndex = SIZE_MAX;
    7938  }
    7939  }
    7940  else
    7941  {
    7942  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
    7943  }
    7944  }
    7945 
    7946  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
    7947  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
    7948 
    7949  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
    7950  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
    7951  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
    7952  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
    7953 
    7954  // 2. Try to find new place for this allocation in preceding or current block.
    7955  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
    7956  {
    7957  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
    7958  VmaAllocationRequest dstAllocRequest;
    7959  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
    7960  m_CurrentFrameIndex,
    7961  m_pBlockVector->GetFrameInUseCount(),
    7962  m_pBlockVector->GetBufferImageGranularity(),
    7963  size,
    7964  alignment,
    7965  suballocType,
    7966  false, // canMakeOtherLost
    7967  &dstAllocRequest) &&
    7968  MoveMakesSense(
    7969  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
    7970  {
    7971  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
    7972 
    7973  // Reached limit on number of allocations or bytes to move.
    7974  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
    7975  (m_BytesMoved + size > maxBytesToMove))
    7976  {
    7977  return VK_INCOMPLETE;
    7978  }
    7979 
    7980  void* pDstMappedData = VMA_NULL;
    7981  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
    7982  if(res != VK_SUCCESS)
    7983  {
    7984  return res;
    7985  }
    7986 
    7987  void* pSrcMappedData = VMA_NULL;
    7988  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
    7989  if(res != VK_SUCCESS)
    7990  {
    7991  return res;
    7992  }
    7993 
    7994  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
    7995  memcpy(
    7996  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
    7997  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
    7998  static_cast<size_t>(size));
    7999 
    8000  if(VMA_DEBUG_MARGIN > 0)
    8001  {
    8002  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
    8003  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
    8004  }
    8005 
    8006  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
    8007  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
    8008 
    8009  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
    8010 
    8011  if(allocInfo.m_pChanged != VMA_NULL)
    8012  {
    8013  *allocInfo.m_pChanged = VK_TRUE;
    8014  }
    8015 
    8016  ++m_AllocationsMoved;
    8017  m_BytesMoved += size;
    8018 
    8019  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
    8020 
    8021  break;
    8022  }
    8023  }
    8024 
    8025  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
    8026 
    8027  if(srcAllocIndex > 0)
    8028  {
    8029  --srcAllocIndex;
    8030  }
    8031  else
    8032  {
    8033  if(srcBlockIndex > 0)
    8034  {
    8035  --srcBlockIndex;
    8036  srcAllocIndex = SIZE_MAX;
    8037  }
    8038  else
    8039  {
    8040  return VK_SUCCESS;
    8041  }
    8042  }
    8043  }
    8044 }
    8045 
    8046 VkResult VmaDefragmentator::Defragment(
    8047  VkDeviceSize maxBytesToMove,
    8048  uint32_t maxAllocationsToMove)
    8049 {
    8050  if(m_Allocations.empty())
    8051  {
    8052  return VK_SUCCESS;
    8053  }
    8054 
    8055  // Create block info for each block.
    8056  const size_t blockCount = m_pBlockVector->m_Blocks.size();
    8057  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8058  {
    8059  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
    8060  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
    8061  m_Blocks.push_back(pBlockInfo);
    8062  }
    8063 
    8064  // Sort them by m_pBlock pointer value.
    8065  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
    8066 
    8067  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
    8068  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
    8069  {
    8070  AllocationInfo& allocInfo = m_Allocations[blockIndex];
    8071  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
    8072  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    8073  {
    8074  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
    8075  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
    8076  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
    8077  {
    8078  (*it)->m_Allocations.push_back(allocInfo);
    8079  }
    8080  else
    8081  {
    8082  VMA_ASSERT(0);
    8083  }
    8084  }
    8085  }
    8086  m_Allocations.clear();
    8087 
    8088  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8089  {
    8090  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
    8091  pBlockInfo->CalcHasNonMovableAllocations();
    8092  pBlockInfo->SortAllocationsBySizeDescecnding();
    8093  }
    8094 
    8095  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
    8096  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
    8097 
    8098  // Execute defragmentation rounds (the main part).
    8099  VkResult result = VK_SUCCESS;
    8100  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
    8101  {
    8102  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
    8103  }
    8104 
    8105  // Unmap blocks that were mapped for defragmentation.
    8106  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8107  {
    8108  m_Blocks[blockIndex]->Unmap(m_hAllocator);
    8109  }
    8110 
    8111  return result;
    8112 }
    8113 
    8114 bool VmaDefragmentator::MoveMakesSense(
    8115  size_t dstBlockIndex, VkDeviceSize dstOffset,
    8116  size_t srcBlockIndex, VkDeviceSize srcOffset)
    8117 {
    8118  if(dstBlockIndex < srcBlockIndex)
    8119  {
    8120  return true;
    8121  }
    8122  if(dstBlockIndex > srcBlockIndex)
    8123  {
    8124  return false;
    8125  }
    8126  if(dstOffset < srcOffset)
    8127  {
    8128  return true;
    8129  }
    8130  return false;
    8131 }
    8132 
    8134 // VmaRecorder
    8135 
    8136 #if VMA_RECORDING_ENABLED
    8137 
    8138 VmaRecorder::VmaRecorder() :
    8139  m_UseMutex(true),
    8140  m_Flags(0),
    8141  m_File(VMA_NULL),
    8142  m_Freq(INT64_MAX),
    8143  m_StartCounter(INT64_MAX)
    8144 {
    8145 }
    8146 
    8147 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
    8148 {
    8149  m_UseMutex = useMutex;
    8150  m_Flags = settings.flags;
    8151 
    8152  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
    8153  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
    8154 
    8155  // Open file for writing.
    8156  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
    8157  if(err != 0)
    8158  {
    8159  return VK_ERROR_INITIALIZATION_FAILED;
    8160  }
    8161 
    8162  // Write header.
    8163  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
    8164  fprintf(m_File, "%s\n", "1,2");
    8165 
    8166  return VK_SUCCESS;
    8167 }
    8168 
    8169 VmaRecorder::~VmaRecorder()
    8170 {
    8171  if(m_File != VMA_NULL)
    8172  {
    8173  fclose(m_File);
    8174  }
    8175 }
    8176 
    8177 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
    8178 {
    8179  CallParams callParams;
    8180  GetBasicParams(callParams);
    8181 
    8182  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8183  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8184  Flush();
    8185 }
    8186 
    8187 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
    8188 {
    8189  CallParams callParams;
    8190  GetBasicParams(callParams);
    8191 
    8192  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8193  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8194  Flush();
    8195 }
    8196 
    8197 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
    8198 {
    8199  CallParams callParams;
    8200  GetBasicParams(callParams);
    8201 
    8202  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8203  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
    8204  createInfo.memoryTypeIndex,
    8205  createInfo.flags,
    8206  createInfo.blockSize,
    8207  createInfo.minBlockCount,
    8208  createInfo.maxBlockCount,
    8209  createInfo.frameInUseCount,
    8210  pool);
    8211  Flush();
    8212 }
    8213 
    8214 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
    8215 {
    8216  CallParams callParams;
    8217  GetBasicParams(callParams);
    8218 
    8219  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8220  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
    8221  pool);
    8222  Flush();
    8223 }
    8224 
    8225 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
    8226  const VkMemoryRequirements& vkMemReq,
    8227  const VmaAllocationCreateInfo& createInfo,
    8228  VmaAllocation allocation)
    8229 {
    8230  CallParams callParams;
    8231  GetBasicParams(callParams);
    8232 
    8233  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8234  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8235  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8236  vkMemReq.size,
    8237  vkMemReq.alignment,
    8238  vkMemReq.memoryTypeBits,
    8239  createInfo.flags,
    8240  createInfo.usage,
    8241  createInfo.requiredFlags,
    8242  createInfo.preferredFlags,
    8243  createInfo.memoryTypeBits,
    8244  createInfo.pool,
    8245  allocation,
    8246  userDataStr.GetString());
    8247  Flush();
    8248 }
    8249 
    8250 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    8251  const VkMemoryRequirements& vkMemReq,
    8252  bool requiresDedicatedAllocation,
    8253  bool prefersDedicatedAllocation,
    8254  const VmaAllocationCreateInfo& createInfo,
    8255  VmaAllocation allocation)
    8256 {
    8257  CallParams callParams;
    8258  GetBasicParams(callParams);
    8259 
    8260  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8261  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8262  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8263  vkMemReq.size,
    8264  vkMemReq.alignment,
    8265  vkMemReq.memoryTypeBits,
    8266  requiresDedicatedAllocation ? 1 : 0,
    8267  prefersDedicatedAllocation ? 1 : 0,
    8268  createInfo.flags,
    8269  createInfo.usage,
    8270  createInfo.requiredFlags,
    8271  createInfo.preferredFlags,
    8272  createInfo.memoryTypeBits,
    8273  createInfo.pool,
    8274  allocation,
    8275  userDataStr.GetString());
    8276  Flush();
    8277 }
    8278 
    8279 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
    8280  const VkMemoryRequirements& vkMemReq,
    8281  bool requiresDedicatedAllocation,
    8282  bool prefersDedicatedAllocation,
    8283  const VmaAllocationCreateInfo& createInfo,
    8284  VmaAllocation allocation)
    8285 {
    8286  CallParams callParams;
    8287  GetBasicParams(callParams);
    8288 
    8289  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8290  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8291  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8292  vkMemReq.size,
    8293  vkMemReq.alignment,
    8294  vkMemReq.memoryTypeBits,
    8295  requiresDedicatedAllocation ? 1 : 0,
    8296  prefersDedicatedAllocation ? 1 : 0,
    8297  createInfo.flags,
    8298  createInfo.usage,
    8299  createInfo.requiredFlags,
    8300  createInfo.preferredFlags,
    8301  createInfo.memoryTypeBits,
    8302  createInfo.pool,
    8303  allocation,
    8304  userDataStr.GetString());
    8305  Flush();
    8306 }
    8307 
    8308 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
    8309  VmaAllocation allocation)
    8310 {
    8311  CallParams callParams;
    8312  GetBasicParams(callParams);
    8313 
    8314  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8315  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8316  allocation);
    8317  Flush();
    8318 }
    8319 
    8320 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
    8321  VmaAllocation allocation,
    8322  const void* pUserData)
    8323 {
    8324  CallParams callParams;
    8325  GetBasicParams(callParams);
    8326 
    8327  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8328  UserDataString userDataStr(
    8329  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
    8330  pUserData);
    8331  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8332  allocation,
    8333  userDataStr.GetString());
    8334  Flush();
    8335 }
    8336 
    8337 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
    8338  VmaAllocation allocation)
    8339 {
    8340  CallParams callParams;
    8341  GetBasicParams(callParams);
    8342 
    8343  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8344  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
    8345  allocation);
    8346  Flush();
    8347 }
    8348 
    8349 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
    8350  VmaAllocation allocation)
    8351 {
    8352  CallParams callParams;
    8353  GetBasicParams(callParams);
    8354 
    8355  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8356  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8357  allocation);
    8358  Flush();
    8359 }
    8360 
    8361 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
    8362  VmaAllocation allocation)
    8363 {
    8364  CallParams callParams;
    8365  GetBasicParams(callParams);
    8366 
    8367  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8368  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8369  allocation);
    8370  Flush();
    8371 }
    8372 
    8373 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
    8374  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8375 {
    8376  CallParams callParams;
    8377  GetBasicParams(callParams);
    8378 
    8379  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8380  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8381  allocation,
    8382  offset,
    8383  size);
    8384  Flush();
    8385 }
    8386 
    8387 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
    8388  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8389 {
    8390  CallParams callParams;
    8391  GetBasicParams(callParams);
    8392 
    8393  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8394  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8395  allocation,
    8396  offset,
    8397  size);
    8398  Flush();
    8399 }
    8400 
    8401 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
    8402  const VkBufferCreateInfo& bufCreateInfo,
    8403  const VmaAllocationCreateInfo& allocCreateInfo,
    8404  VmaAllocation allocation)
    8405 {
    8406  CallParams callParams;
    8407  GetBasicParams(callParams);
    8408 
    8409  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8410  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8411  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8412  bufCreateInfo.flags,
    8413  bufCreateInfo.size,
    8414  bufCreateInfo.usage,
    8415  bufCreateInfo.sharingMode,
    8416  allocCreateInfo.flags,
    8417  allocCreateInfo.usage,
    8418  allocCreateInfo.requiredFlags,
    8419  allocCreateInfo.preferredFlags,
    8420  allocCreateInfo.memoryTypeBits,
    8421  allocCreateInfo.pool,
    8422  allocation,
    8423  userDataStr.GetString());
    8424  Flush();
    8425 }
    8426 
    8427 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
    8428  const VkImageCreateInfo& imageCreateInfo,
    8429  const VmaAllocationCreateInfo& allocCreateInfo,
    8430  VmaAllocation allocation)
    8431 {
    8432  CallParams callParams;
    8433  GetBasicParams(callParams);
    8434 
    8435  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8436  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8437  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8438  imageCreateInfo.flags,
    8439  imageCreateInfo.imageType,
    8440  imageCreateInfo.format,
    8441  imageCreateInfo.extent.width,
    8442  imageCreateInfo.extent.height,
    8443  imageCreateInfo.extent.depth,
    8444  imageCreateInfo.mipLevels,
    8445  imageCreateInfo.arrayLayers,
    8446  imageCreateInfo.samples,
    8447  imageCreateInfo.tiling,
    8448  imageCreateInfo.usage,
    8449  imageCreateInfo.sharingMode,
    8450  imageCreateInfo.initialLayout,
    8451  allocCreateInfo.flags,
    8452  allocCreateInfo.usage,
    8453  allocCreateInfo.requiredFlags,
    8454  allocCreateInfo.preferredFlags,
    8455  allocCreateInfo.memoryTypeBits,
    8456  allocCreateInfo.pool,
    8457  allocation,
    8458  userDataStr.GetString());
    8459  Flush();
    8460 }
    8461 
    8462 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
    8463  VmaAllocation allocation)
    8464 {
    8465  CallParams callParams;
    8466  GetBasicParams(callParams);
    8467 
    8468  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8469  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
    8470  allocation);
    8471  Flush();
    8472 }
    8473 
    8474 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
    8475  VmaAllocation allocation)
    8476 {
    8477  CallParams callParams;
    8478  GetBasicParams(callParams);
    8479 
    8480  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8481  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
    8482  allocation);
    8483  Flush();
    8484 }
    8485 
    8486 void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex,
    8487  VmaAllocation allocation)
    8488 {
    8489  CallParams callParams;
    8490  GetBasicParams(callParams);
    8491 
    8492  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8493  fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
    8494  allocation);
    8495  Flush();
    8496 }
    8497 
    8498 void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex,
    8499  VmaAllocation allocation)
    8500 {
    8501  CallParams callParams;
    8502  GetBasicParams(callParams);
    8503 
    8504  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8505  fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex,
    8506  allocation);
    8507  Flush();
    8508 }
    8509 
    8510 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
    8511 {
    8512  if(pUserData != VMA_NULL)
    8513  {
    8514  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
    8515  {
    8516  m_Str = (const char*)pUserData;
    8517  }
    8518  else
    8519  {
    8520  sprintf_s(m_PtrStr, "%p", pUserData);
    8521  m_Str = m_PtrStr;
    8522  }
    8523  }
    8524  else
    8525  {
    8526  m_Str = "";
    8527  }
    8528 }
    8529 
    8530 void VmaRecorder::GetBasicParams(CallParams& outParams)
    8531 {
    8532  outParams.threadId = GetCurrentThreadId();
    8533 
    8534  LARGE_INTEGER counter;
    8535  QueryPerformanceCounter(&counter);
    8536  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
    8537 }
    8538 
    8539 void VmaRecorder::Flush()
    8540 {
    8541  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
    8542  {
    8543  fflush(m_File);
    8544  }
    8545 }
    8546 
    8547 #endif // #if VMA_RECORDING_ENABLED
    8548 
    8550 // VmaAllocator_T
    8551 
    8552 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
    8553  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
    8554  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
    8555  m_hDevice(pCreateInfo->device),
    8556  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
    8557  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
    8558  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
    8559  m_PreferredLargeHeapBlockSize(0),
    8560  m_PhysicalDevice(pCreateInfo->physicalDevice),
    8561  m_CurrentFrameIndex(0),
    8562  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
    8563  m_NextPoolId(0)
    8564 #if VMA_RECORDING_ENABLED
    8565  ,m_pRecorder(VMA_NULL)
    8566 #endif
    8567 {
    8568  if(VMA_DEBUG_DETECT_CORRUPTION)
    8569  {
    8570  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
    8571  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
    8572  }
    8573 
    8574  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
    8575 
    8576 #if !(VMA_DEDICATED_ALLOCATION)
    8577  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
    8578  {
    8579  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
    8580  }
    8581 #endif
    8582 
    8583  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
    8584  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
    8585  memset(&m_MemProps, 0, sizeof(m_MemProps));
    8586 
    8587  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
    8588  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
    8589 
    8590  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    8591  {
    8592  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
    8593  }
    8594 
    8595  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
    8596  {
    8597  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
    8598  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
    8599  }
    8600 
    8601  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
    8602 
    8603  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
    8604  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
    8605 
    8606  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
    8607  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
    8608 
    8609  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
    8610  {
    8611  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
    8612  {
    8613  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
    8614  if(limit != VK_WHOLE_SIZE)
    8615  {
    8616  m_HeapSizeLimit[heapIndex] = limit;
    8617  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
    8618  {
    8619  m_MemProps.memoryHeaps[heapIndex].size = limit;
    8620  }
    8621  }
    8622  }
    8623  }
    8624 
    8625  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    8626  {
    8627  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
    8628 
    8629  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
    8630  this,
    8631  memTypeIndex,
    8632  preferredBlockSize,
    8633  0,
    8634  SIZE_MAX,
    8635  GetBufferImageGranularity(),
    8636  pCreateInfo->frameInUseCount,
    8637  false); // isCustomPool
    8638  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
    8639  // becase minBlockCount is 0.
    8640  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
    8641 
    8642  }
    8643 }
    8644 
    8645 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
    8646 {
    8647  VkResult res = VK_SUCCESS;
    8648 
    8649  if(pCreateInfo->pRecordSettings != VMA_NULL &&
    8650  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
    8651  {
    8652 #if VMA_RECORDING_ENABLED
    8653  m_pRecorder = vma_new(this, VmaRecorder)();
    8654  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
    8655  if(res != VK_SUCCESS)
    8656  {
    8657  return res;
    8658  }
    8659  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
    8660 #else
    8661  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
    8662  return VK_ERROR_FEATURE_NOT_PRESENT;
    8663 #endif
    8664  }
    8665 
    8666  return res;
    8667 }
    8668 
    8669 VmaAllocator_T::~VmaAllocator_T()
    8670 {
    8671 #if VMA_RECORDING_ENABLED
    8672  if(m_pRecorder != VMA_NULL)
    8673  {
    8674  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
    8675  vma_delete(this, m_pRecorder);
    8676  }
    8677 #endif
    8678 
    8679  VMA_ASSERT(m_Pools.empty());
    8680 
    8681  for(size_t i = GetMemoryTypeCount(); i--; )
    8682  {
    8683  vma_delete(this, m_pDedicatedAllocations[i]);
    8684  vma_delete(this, m_pBlockVectors[i]);
    8685  }
    8686 }
    8687 
    8688 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
    8689 {
    8690 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8691  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
    8692  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
    8693  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    8694  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
    8695  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
    8696  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
    8697  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
    8698  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
    8699  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
    8700  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
    8701  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
    8702  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
    8703  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
    8704  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
    8705  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
    8706  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
    8707 #if VMA_DEDICATED_ALLOCATION
    8708  if(m_UseKhrDedicatedAllocation)
    8709  {
    8710  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
    8711  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
    8712  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
    8713  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
    8714  }
    8715 #endif // #if VMA_DEDICATED_ALLOCATION
    8716 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8717 
    8718 #define VMA_COPY_IF_NOT_NULL(funcName) \
    8719  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
    8720 
    8721  if(pVulkanFunctions != VMA_NULL)
    8722  {
    8723  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
    8724  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
    8725  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
    8726  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
    8727  VMA_COPY_IF_NOT_NULL(vkMapMemory);
    8728  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
    8729  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
    8730  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
    8731  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
    8732  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
    8733  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
    8734  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
    8735  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
    8736  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
    8737  VMA_COPY_IF_NOT_NULL(vkCreateImage);
    8738  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
    8739 #if VMA_DEDICATED_ALLOCATION
    8740  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
    8741  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
    8742 #endif
    8743  }
    8744 
    8745 #undef VMA_COPY_IF_NOT_NULL
    8746 
    8747  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
    8748  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
    8749  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
    8750  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
    8751  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
    8752  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
    8753  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
    8754  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
    8755  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
    8756  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
    8757  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
    8758  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
    8759  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
    8760  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
    8761  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
    8762  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
    8763  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
    8764  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
    8765 #if VMA_DEDICATED_ALLOCATION
    8766  if(m_UseKhrDedicatedAllocation)
    8767  {
    8768  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
    8769  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
    8770  }
    8771 #endif
    8772 }
    8773 
    8774 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
    8775 {
    8776  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    8777  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
    8778  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
    8779  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
    8780 }
    8781 
    8782 VkResult VmaAllocator_T::AllocateMemoryOfType(
    8783  VkDeviceSize size,
    8784  VkDeviceSize alignment,
    8785  bool dedicatedAllocation,
    8786  VkBuffer dedicatedBuffer,
    8787  VkImage dedicatedImage,
    8788  const VmaAllocationCreateInfo& createInfo,
    8789  uint32_t memTypeIndex,
    8790  VmaSuballocationType suballocType,
    8791  VmaAllocation* pAllocation)
    8792 {
    8793  VMA_ASSERT(pAllocation != VMA_NULL);
    8794  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
    8795 
    8796  VmaAllocationCreateInfo finalCreateInfo = createInfo;
    8797 
    8798  // If memory type is not HOST_VISIBLE, disable MAPPED.
    8799  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8800  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    8801  {
    8802  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
    8803  }
    8804 
    8805  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
    8806  VMA_ASSERT(blockVector);
    8807 
    8808  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
    8809  bool preferDedicatedMemory =
    8810  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
    8811  dedicatedAllocation ||
    8812  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
    8813  size > preferredBlockSize / 2;
    8814 
    8815  if(preferDedicatedMemory &&
    8816  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
    8817  finalCreateInfo.pool == VK_NULL_HANDLE)
    8818  {
    8819  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
    8820  }
    8821 
    8822  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
    8823  {
    8824  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8825  {
    8826  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8827  }
    8828  else
    8829  {
    8830  return AllocateDedicatedMemory(
    8831  size,
    8832  suballocType,
    8833  memTypeIndex,
    8834  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8835  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8836  finalCreateInfo.pUserData,
    8837  dedicatedBuffer,
    8838  dedicatedImage,
    8839  pAllocation);
    8840  }
    8841  }
    8842  else
    8843  {
    8844  VkResult res = blockVector->Allocate(
    8845  VK_NULL_HANDLE, // hCurrentPool
    8846  m_CurrentFrameIndex.load(),
    8847  size,
    8848  alignment,
    8849  finalCreateInfo,
    8850  suballocType,
    8851  pAllocation);
    8852  if(res == VK_SUCCESS)
    8853  {
    8854  return res;
    8855  }
    8856 
    8857  // 5. Try dedicated memory.
    8858  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8859  {
    8860  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8861  }
    8862  else
    8863  {
    8864  res = AllocateDedicatedMemory(
    8865  size,
    8866  suballocType,
    8867  memTypeIndex,
    8868  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8869  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8870  finalCreateInfo.pUserData,
    8871  dedicatedBuffer,
    8872  dedicatedImage,
    8873  pAllocation);
    8874  if(res == VK_SUCCESS)
    8875  {
    8876  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
    8877  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
    8878  return VK_SUCCESS;
    8879  }
    8880  else
    8881  {
    8882  // Everything failed: Return error code.
    8883  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8884  return res;
    8885  }
    8886  }
    8887  }
    8888 }
    8889 
    8890 VkResult VmaAllocator_T::AllocateDedicatedMemory(
    8891  VkDeviceSize size,
    8892  VmaSuballocationType suballocType,
    8893  uint32_t memTypeIndex,
    8894  bool map,
    8895  bool isUserDataString,
    8896  void* pUserData,
    8897  VkBuffer dedicatedBuffer,
    8898  VkImage dedicatedImage,
    8899  VmaAllocation* pAllocation)
    8900 {
    8901  VMA_ASSERT(pAllocation);
    8902 
    8903  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    8904  allocInfo.memoryTypeIndex = memTypeIndex;
    8905  allocInfo.allocationSize = size;
    8906 
    8907 #if VMA_DEDICATED_ALLOCATION
    8908  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
    8909  if(m_UseKhrDedicatedAllocation)
    8910  {
    8911  if(dedicatedBuffer != VK_NULL_HANDLE)
    8912  {
    8913  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
    8914  dedicatedAllocInfo.buffer = dedicatedBuffer;
    8915  allocInfo.pNext = &dedicatedAllocInfo;
    8916  }
    8917  else if(dedicatedImage != VK_NULL_HANDLE)
    8918  {
    8919  dedicatedAllocInfo.image = dedicatedImage;
    8920  allocInfo.pNext = &dedicatedAllocInfo;
    8921  }
    8922  }
    8923 #endif // #if VMA_DEDICATED_ALLOCATION
    8924 
    8925  // Allocate VkDeviceMemory.
    8926  VkDeviceMemory hMemory = VK_NULL_HANDLE;
    8927  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
    8928  if(res < 0)
    8929  {
    8930  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8931  return res;
    8932  }
    8933 
    8934  void* pMappedData = VMA_NULL;
    8935  if(map)
    8936  {
    8937  res = (*m_VulkanFunctions.vkMapMemory)(
    8938  m_hDevice,
    8939  hMemory,
    8940  0,
    8941  VK_WHOLE_SIZE,
    8942  0,
    8943  &pMappedData);
    8944  if(res < 0)
    8945  {
    8946  VMA_DEBUG_LOG(" vkMapMemory FAILED");
    8947  FreeVulkanMemory(memTypeIndex, size, hMemory);
    8948  return res;
    8949  }
    8950  }
    8951 
    8952  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
    8953  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
    8954  (*pAllocation)->SetUserData(this, pUserData);
    8955  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    8956  {
    8957  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    8958  }
    8959 
    8960  // Register it in m_pDedicatedAllocations.
    8961  {
    8962  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    8963  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    8964  VMA_ASSERT(pDedicatedAllocations);
    8965  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
    8966  }
    8967 
    8968  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
    8969 
    8970  return VK_SUCCESS;
    8971 }
    8972 
    8973 void VmaAllocator_T::GetBufferMemoryRequirements(
    8974  VkBuffer hBuffer,
    8975  VkMemoryRequirements& memReq,
    8976  bool& requiresDedicatedAllocation,
    8977  bool& prefersDedicatedAllocation) const
    8978 {
    8979 #if VMA_DEDICATED_ALLOCATION
    8980  if(m_UseKhrDedicatedAllocation)
    8981  {
    8982  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
    8983  memReqInfo.buffer = hBuffer;
    8984 
    8985  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    8986 
    8987  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    8988  memReq2.pNext = &memDedicatedReq;
    8989 
    8990  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    8991 
    8992  memReq = memReq2.memoryRequirements;
    8993  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    8994  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    8995  }
    8996  else
    8997 #endif // #if VMA_DEDICATED_ALLOCATION
    8998  {
    8999  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
    9000  requiresDedicatedAllocation = false;
    9001  prefersDedicatedAllocation = false;
    9002  }
    9003 }
    9004 
    9005 void VmaAllocator_T::GetImageMemoryRequirements(
    9006  VkImage hImage,
    9007  VkMemoryRequirements& memReq,
    9008  bool& requiresDedicatedAllocation,
    9009  bool& prefersDedicatedAllocation) const
    9010 {
    9011 #if VMA_DEDICATED_ALLOCATION
    9012  if(m_UseKhrDedicatedAllocation)
    9013  {
    9014  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
    9015  memReqInfo.image = hImage;
    9016 
    9017  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    9018 
    9019  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    9020  memReq2.pNext = &memDedicatedReq;
    9021 
    9022  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    9023 
    9024  memReq = memReq2.memoryRequirements;
    9025  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    9026  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    9027  }
    9028  else
    9029 #endif // #if VMA_DEDICATED_ALLOCATION
    9030  {
    9031  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
    9032  requiresDedicatedAllocation = false;
    9033  prefersDedicatedAllocation = false;
    9034  }
    9035 }
    9036 
    9037 VkResult VmaAllocator_T::AllocateMemory(
    9038  const VkMemoryRequirements& vkMemReq,
    9039  bool requiresDedicatedAllocation,
    9040  bool prefersDedicatedAllocation,
    9041  VkBuffer dedicatedBuffer,
    9042  VkImage dedicatedImage,
    9043  const VmaAllocationCreateInfo& createInfo,
    9044  VmaSuballocationType suballocType,
    9045  VmaAllocation* pAllocation)
    9046 {
    9047  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
    9048  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    9049  {
    9050  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
    9051  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9052  }
    9053  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    9054  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
    9055  {
    9056  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
    9057  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9058  }
    9059  if(requiresDedicatedAllocation)
    9060  {
    9061  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    9062  {
    9063  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
    9064  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9065  }
    9066  if(createInfo.pool != VK_NULL_HANDLE)
    9067  {
    9068  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
    9069  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9070  }
    9071  }
    9072  if((createInfo.pool != VK_NULL_HANDLE) &&
    9073  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
    9074  {
    9075  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
    9076  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9077  }
    9078 
    9079  if(createInfo.pool != VK_NULL_HANDLE)
    9080  {
    9081  const VkDeviceSize alignmentForPool = VMA_MAX(
    9082  vkMemReq.alignment,
    9083  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
    9084  return createInfo.pool->m_BlockVector.Allocate(
    9085  createInfo.pool,
    9086  m_CurrentFrameIndex.load(),
    9087  vkMemReq.size,
    9088  alignmentForPool,
    9089  createInfo,
    9090  suballocType,
    9091  pAllocation);
    9092  }
    9093  else
    9094  {
    9095  // Bit mask of memory Vulkan types acceptable for this allocation.
    9096  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
    9097  uint32_t memTypeIndex = UINT32_MAX;
    9098  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9099  if(res == VK_SUCCESS)
    9100  {
    9101  VkDeviceSize alignmentForMemType = VMA_MAX(
    9102  vkMemReq.alignment,
    9103  GetMemoryTypeMinAlignment(memTypeIndex));
    9104 
    9105  res = AllocateMemoryOfType(
    9106  vkMemReq.size,
    9107  alignmentForMemType,
    9108  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9109  dedicatedBuffer,
    9110  dedicatedImage,
    9111  createInfo,
    9112  memTypeIndex,
    9113  suballocType,
    9114  pAllocation);
    9115  // Succeeded on first try.
    9116  if(res == VK_SUCCESS)
    9117  {
    9118  return res;
    9119  }
    9120  // Allocation from this memory type failed. Try other compatible memory types.
    9121  else
    9122  {
    9123  for(;;)
    9124  {
    9125  // Remove old memTypeIndex from list of possibilities.
    9126  memoryTypeBits &= ~(1u << memTypeIndex);
    9127  // Find alternative memTypeIndex.
    9128  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9129  if(res == VK_SUCCESS)
    9130  {
    9131  alignmentForMemType = VMA_MAX(
    9132  vkMemReq.alignment,
    9133  GetMemoryTypeMinAlignment(memTypeIndex));
    9134 
    9135  res = AllocateMemoryOfType(
    9136  vkMemReq.size,
    9137  alignmentForMemType,
    9138  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9139  dedicatedBuffer,
    9140  dedicatedImage,
    9141  createInfo,
    9142  memTypeIndex,
    9143  suballocType,
    9144  pAllocation);
    9145  // Allocation from this alternative memory type succeeded.
    9146  if(res == VK_SUCCESS)
    9147  {
    9148  return res;
    9149  }
    9150  // else: Allocation from this memory type failed. Try next one - next loop iteration.
    9151  }
    9152  // No other matching memory type index could be found.
    9153  else
    9154  {
    9155  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
    9156  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9157  }
    9158  }
    9159  }
    9160  }
    9161  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
    9162  else
    9163  return res;
    9164  }
    9165 }
    9166 
    9167 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
    9168 {
    9169  VMA_ASSERT(allocation);
    9170 
    9171  if(allocation->CanBecomeLost() == false ||
    9172  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    9173  {
    9174  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    9175  {
    9176  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
    9177  }
    9178 
    9179  switch(allocation->GetType())
    9180  {
    9181  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9182  {
    9183  VmaBlockVector* pBlockVector = VMA_NULL;
    9184  VmaPool hPool = allocation->GetPool();
    9185  if(hPool != VK_NULL_HANDLE)
    9186  {
    9187  pBlockVector = &hPool->m_BlockVector;
    9188  }
    9189  else
    9190  {
    9191  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9192  pBlockVector = m_pBlockVectors[memTypeIndex];
    9193  }
    9194  pBlockVector->Free(allocation);
    9195  }
    9196  break;
    9197  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9198  FreeDedicatedMemory(allocation);
    9199  break;
    9200  default:
    9201  VMA_ASSERT(0);
    9202  }
    9203  }
    9204 
    9205  allocation->SetUserData(this, VMA_NULL);
    9206  vma_delete(this, allocation);
    9207 }
    9208 
    9209 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
    9210 {
    9211  // Initialize.
    9212  InitStatInfo(pStats->total);
    9213  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
    9214  InitStatInfo(pStats->memoryType[i]);
    9215  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    9216  InitStatInfo(pStats->memoryHeap[i]);
    9217 
    9218  // Process default pools.
    9219  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9220  {
    9221  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9222  VMA_ASSERT(pBlockVector);
    9223  pBlockVector->AddStats(pStats);
    9224  }
    9225 
    9226  // Process custom pools.
    9227  {
    9228  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9229  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9230  {
    9231  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
    9232  }
    9233  }
    9234 
    9235  // Process dedicated allocations.
    9236  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9237  {
    9238  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    9239  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9240  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9241  VMA_ASSERT(pDedicatedAllocVector);
    9242  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
    9243  {
    9244  VmaStatInfo allocationStatInfo;
    9245  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
    9246  VmaAddStatInfo(pStats->total, allocationStatInfo);
    9247  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    9248  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    9249  }
    9250  }
    9251 
    9252  // Postprocess.
    9253  VmaPostprocessCalcStatInfo(pStats->total);
    9254  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
    9255  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
    9256  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
    9257  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
    9258 }
    9259 
    9260 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
    9261 
    9262 VkResult VmaAllocator_T::Defragment(
    9263  VmaAllocation* pAllocations,
    9264  size_t allocationCount,
    9265  VkBool32* pAllocationsChanged,
    9266  const VmaDefragmentationInfo* pDefragmentationInfo,
    9267  VmaDefragmentationStats* pDefragmentationStats)
    9268 {
    9269  if(pAllocationsChanged != VMA_NULL)
    9270  {
    9271  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
    9272  }
    9273  if(pDefragmentationStats != VMA_NULL)
    9274  {
    9275  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
    9276  }
    9277 
    9278  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
    9279 
    9280  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
    9281 
    9282  const size_t poolCount = m_Pools.size();
    9283 
    9284  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
    9285  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
    9286  {
    9287  VmaAllocation hAlloc = pAllocations[allocIndex];
    9288  VMA_ASSERT(hAlloc);
    9289  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
    9290  // DedicatedAlloc cannot be defragmented.
    9291  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
    9292  // Only HOST_VISIBLE memory types can be defragmented.
    9293  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
    9294  // Lost allocation cannot be defragmented.
    9295  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
    9296  {
    9297  VmaBlockVector* pAllocBlockVector = VMA_NULL;
    9298 
    9299  const VmaPool hAllocPool = hAlloc->GetPool();
    9300  // This allocation belongs to custom pool.
    9301  if(hAllocPool != VK_NULL_HANDLE)
    9302  {
    9303  pAllocBlockVector = &hAllocPool->GetBlockVector();
    9304  }
    9305  // This allocation belongs to general pool.
    9306  else
    9307  {
    9308  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
    9309  }
    9310 
    9311  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
    9312 
    9313  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
    9314  &pAllocationsChanged[allocIndex] : VMA_NULL;
    9315  pDefragmentator->AddAllocation(hAlloc, pChanged);
    9316  }
    9317  }
    9318 
    9319  VkResult result = VK_SUCCESS;
    9320 
    9321  // ======== Main processing.
    9322 
    9323  VkDeviceSize maxBytesToMove = SIZE_MAX;
    9324  uint32_t maxAllocationsToMove = UINT32_MAX;
    9325  if(pDefragmentationInfo != VMA_NULL)
    9326  {
    9327  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
    9328  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
    9329  }
    9330 
    9331  // Process standard memory.
    9332  for(uint32_t memTypeIndex = 0;
    9333  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
    9334  ++memTypeIndex)
    9335  {
    9336  // Only HOST_VISIBLE memory types can be defragmented.
    9337  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9338  {
    9339  result = m_pBlockVectors[memTypeIndex]->Defragment(
    9340  pDefragmentationStats,
    9341  maxBytesToMove,
    9342  maxAllocationsToMove);
    9343  }
    9344  }
    9345 
    9346  // Process custom pools.
    9347  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
    9348  {
    9349  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
    9350  pDefragmentationStats,
    9351  maxBytesToMove,
    9352  maxAllocationsToMove);
    9353  }
    9354 
    9355  // ======== Destroy defragmentators.
    9356 
    9357  // Process custom pools.
    9358  for(size_t poolIndex = poolCount; poolIndex--; )
    9359  {
    9360  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
    9361  }
    9362 
    9363  // Process standard memory.
    9364  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
    9365  {
    9366  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9367  {
    9368  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
    9369  }
    9370  }
    9371 
    9372  return result;
    9373 }
    9374 
    9375 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
    9376 {
    9377  if(hAllocation->CanBecomeLost())
    9378  {
    9379  /*
    9380  Warning: This is a carefully designed algorithm.
    9381  Do not modify unless you really know what you're doing :)
    9382  */
    9383  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9384  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9385  for(;;)
    9386  {
    9387  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9388  {
    9389  pAllocationInfo->memoryType = UINT32_MAX;
    9390  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
    9391  pAllocationInfo->offset = 0;
    9392  pAllocationInfo->size = hAllocation->GetSize();
    9393  pAllocationInfo->pMappedData = VMA_NULL;
    9394  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9395  return;
    9396  }
    9397  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9398  {
    9399  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9400  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9401  pAllocationInfo->offset = hAllocation->GetOffset();
    9402  pAllocationInfo->size = hAllocation->GetSize();
    9403  pAllocationInfo->pMappedData = VMA_NULL;
    9404  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9405  return;
    9406  }
    9407  else // Last use time earlier than current time.
    9408  {
    9409  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9410  {
    9411  localLastUseFrameIndex = localCurrFrameIndex;
    9412  }
    9413  }
    9414  }
    9415  }
    9416  else
    9417  {
    9418 #if VMA_STATS_STRING_ENABLED
    9419  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9420  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9421  for(;;)
    9422  {
    9423  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9424  if(localLastUseFrameIndex == localCurrFrameIndex)
    9425  {
    9426  break;
    9427  }
    9428  else // Last use time earlier than current time.
    9429  {
    9430  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9431  {
    9432  localLastUseFrameIndex = localCurrFrameIndex;
    9433  }
    9434  }
    9435  }
    9436 #endif
    9437 
    9438  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9439  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9440  pAllocationInfo->offset = hAllocation->GetOffset();
    9441  pAllocationInfo->size = hAllocation->GetSize();
    9442  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
    9443  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9444  }
    9445 }
    9446 
    9447 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
    9448 {
    9449  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
    9450  if(hAllocation->CanBecomeLost())
    9451  {
    9452  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9453  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9454  for(;;)
    9455  {
    9456  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9457  {
    9458  return false;
    9459  }
    9460  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9461  {
    9462  return true;
    9463  }
    9464  else // Last use time earlier than current time.
    9465  {
    9466  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9467  {
    9468  localLastUseFrameIndex = localCurrFrameIndex;
    9469  }
    9470  }
    9471  }
    9472  }
    9473  else
    9474  {
    9475 #if VMA_STATS_STRING_ENABLED
    9476  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9477  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9478  for(;;)
    9479  {
    9480  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9481  if(localLastUseFrameIndex == localCurrFrameIndex)
    9482  {
    9483  break;
    9484  }
    9485  else // Last use time earlier than current time.
    9486  {
    9487  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9488  {
    9489  localLastUseFrameIndex = localCurrFrameIndex;
    9490  }
    9491  }
    9492  }
    9493 #endif
    9494 
    9495  return true;
    9496  }
    9497 }
    9498 
    9499 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
    9500 {
    9501  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
    9502 
    9503  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
    9504 
    9505  if(newCreateInfo.maxBlockCount == 0)
    9506  {
    9507  newCreateInfo.maxBlockCount = SIZE_MAX;
    9508  }
    9509  if(newCreateInfo.blockSize == 0)
    9510  {
    9511  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
    9512  }
    9513 
    9514  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
    9515 
    9516  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
    9517  if(res != VK_SUCCESS)
    9518  {
    9519  vma_delete(this, *pPool);
    9520  *pPool = VMA_NULL;
    9521  return res;
    9522  }
    9523 
    9524  // Add to m_Pools.
    9525  {
    9526  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9527  (*pPool)->SetId(m_NextPoolId++);
    9528  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
    9529  }
    9530 
    9531  return VK_SUCCESS;
    9532 }
    9533 
    9534 void VmaAllocator_T::DestroyPool(VmaPool pool)
    9535 {
    9536  // Remove from m_Pools.
    9537  {
    9538  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9539  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
    9540  VMA_ASSERT(success && "Pool not found in Allocator.");
    9541  }
    9542 
    9543  vma_delete(this, pool);
    9544 }
    9545 
    9546 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
    9547 {
    9548  pool->m_BlockVector.GetPoolStats(pPoolStats);
    9549 }
    9550 
    9551 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
    9552 {
    9553  m_CurrentFrameIndex.store(frameIndex);
    9554 }
    9555 
    9556 void VmaAllocator_T::MakePoolAllocationsLost(
    9557  VmaPool hPool,
    9558  size_t* pLostAllocationCount)
    9559 {
    9560  hPool->m_BlockVector.MakePoolAllocationsLost(
    9561  m_CurrentFrameIndex.load(),
    9562  pLostAllocationCount);
    9563 }
    9564 
    9565 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
    9566 {
    9567  return hPool->m_BlockVector.CheckCorruption();
    9568 }
    9569 
    9570 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
    9571 {
    9572  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
    9573 
    9574  // Process default pools.
    9575  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9576  {
    9577  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
    9578  {
    9579  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9580  VMA_ASSERT(pBlockVector);
    9581  VkResult localRes = pBlockVector->CheckCorruption();
    9582  switch(localRes)
    9583  {
    9584  case VK_ERROR_FEATURE_NOT_PRESENT:
    9585  break;
    9586  case VK_SUCCESS:
    9587  finalRes = VK_SUCCESS;
    9588  break;
    9589  default:
    9590  return localRes;
    9591  }
    9592  }
    9593  }
    9594 
    9595  // Process custom pools.
    9596  {
    9597  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9598  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9599  {
    9600  if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0)
    9601  {
    9602  VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption();
    9603  switch(localRes)
    9604  {
    9605  case VK_ERROR_FEATURE_NOT_PRESENT:
    9606  break;
    9607  case VK_SUCCESS:
    9608  finalRes = VK_SUCCESS;
    9609  break;
    9610  default:
    9611  return localRes;
    9612  }
    9613  }
    9614  }
    9615  }
    9616 
    9617  return finalRes;
    9618 }
    9619 
    9620 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
    9621 {
    9622  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
    9623  (*pAllocation)->InitLost();
    9624 }
    9625 
    9626 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
    9627 {
    9628  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
    9629 
    9630  VkResult res;
    9631  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9632  {
    9633  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9634  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
    9635  {
    9636  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9637  if(res == VK_SUCCESS)
    9638  {
    9639  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
    9640  }
    9641  }
    9642  else
    9643  {
    9644  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9645  }
    9646  }
    9647  else
    9648  {
    9649  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9650  }
    9651 
    9652  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
    9653  {
    9654  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
    9655  }
    9656 
    9657  return res;
    9658 }
    9659 
    9660 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
    9661 {
    9662  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
    9663  {
    9664  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
    9665  }
    9666 
    9667  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
    9668 
    9669  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
    9670  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9671  {
    9672  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9673  m_HeapSizeLimit[heapIndex] += size;
    9674  }
    9675 }
    9676 
    9677 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
    9678 {
    9679  if(hAllocation->CanBecomeLost())
    9680  {
    9681  return VK_ERROR_MEMORY_MAP_FAILED;
    9682  }
    9683 
    9684  switch(hAllocation->GetType())
    9685  {
    9686  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9687  {
    9688  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9689  char *pBytes = VMA_NULL;
    9690  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
    9691  if(res == VK_SUCCESS)
    9692  {
    9693  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
    9694  hAllocation->BlockAllocMap();
    9695  }
    9696  return res;
    9697  }
    9698  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9699  return hAllocation->DedicatedAllocMap(this, ppData);
    9700  default:
    9701  VMA_ASSERT(0);
    9702  return VK_ERROR_MEMORY_MAP_FAILED;
    9703  }
    9704 }
    9705 
    9706 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
    9707 {
    9708  switch(hAllocation->GetType())
    9709  {
    9710  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9711  {
    9712  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9713  hAllocation->BlockAllocUnmap();
    9714  pBlock->Unmap(this, 1);
    9715  }
    9716  break;
    9717  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9718  hAllocation->DedicatedAllocUnmap(this);
    9719  break;
    9720  default:
    9721  VMA_ASSERT(0);
    9722  }
    9723 }
    9724 
    9725 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
    9726 {
    9727  VkResult res = VK_SUCCESS;
    9728  switch(hAllocation->GetType())
    9729  {
    9730  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9731  res = GetVulkanFunctions().vkBindBufferMemory(
    9732  m_hDevice,
    9733  hBuffer,
    9734  hAllocation->GetMemory(),
    9735  0); //memoryOffset
    9736  break;
    9737  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9738  {
    9739  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9740  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
    9741  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
    9742  break;
    9743  }
    9744  default:
    9745  VMA_ASSERT(0);
    9746  }
    9747  return res;
    9748 }
    9749 
    9750 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
    9751 {
    9752  VkResult res = VK_SUCCESS;
    9753  switch(hAllocation->GetType())
    9754  {
    9755  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9756  res = GetVulkanFunctions().vkBindImageMemory(
    9757  m_hDevice,
    9758  hImage,
    9759  hAllocation->GetMemory(),
    9760  0); //memoryOffset
    9761  break;
    9762  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9763  {
    9764  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9765  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
    9766  res = pBlock->BindImageMemory(this, hAllocation, hImage);
    9767  break;
    9768  }
    9769  default:
    9770  VMA_ASSERT(0);
    9771  }
    9772  return res;
    9773 }
    9774 
    9775 void VmaAllocator_T::FlushOrInvalidateAllocation(
    9776  VmaAllocation hAllocation,
    9777  VkDeviceSize offset, VkDeviceSize size,
    9778  VMA_CACHE_OPERATION op)
    9779 {
    9780  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
    9781  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
    9782  {
    9783  const VkDeviceSize allocationSize = hAllocation->GetSize();
    9784  VMA_ASSERT(offset <= allocationSize);
    9785 
    9786  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
    9787 
    9788  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
    9789  memRange.memory = hAllocation->GetMemory();
    9790 
    9791  switch(hAllocation->GetType())
    9792  {
    9793  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9794  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9795  if(size == VK_WHOLE_SIZE)
    9796  {
    9797  memRange.size = allocationSize - memRange.offset;
    9798  }
    9799  else
    9800  {
    9801  VMA_ASSERT(offset + size <= allocationSize);
    9802  memRange.size = VMA_MIN(
    9803  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
    9804  allocationSize - memRange.offset);
    9805  }
    9806  break;
    9807 
    9808  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9809  {
    9810  // 1. Still within this allocation.
    9811  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9812  if(size == VK_WHOLE_SIZE)
    9813  {
    9814  size = allocationSize - offset;
    9815  }
    9816  else
    9817  {
    9818  VMA_ASSERT(offset + size <= allocationSize);
    9819  }
    9820  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
    9821 
    9822  // 2. Adjust to whole block.
    9823  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
    9824  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
    9825  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize();
    9826  memRange.offset += allocationOffset;
    9827  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
    9828 
    9829  break;
    9830  }
    9831 
    9832  default:
    9833  VMA_ASSERT(0);
    9834  }
    9835 
    9836  switch(op)
    9837  {
    9838  case VMA_CACHE_FLUSH:
    9839  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9840  break;
    9841  case VMA_CACHE_INVALIDATE:
    9842  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9843  break;
    9844  default:
    9845  VMA_ASSERT(0);
    9846  }
    9847  }
    9848  // else: Just ignore this call.
    9849 }
    9850 
    9851 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
    9852 {
    9853  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
    9854 
    9855  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9856  {
    9857  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9858  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    9859  VMA_ASSERT(pDedicatedAllocations);
    9860  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
    9861  VMA_ASSERT(success);
    9862  }
    9863 
    9864  VkDeviceMemory hMemory = allocation->GetMemory();
    9865 
    9866  if(allocation->GetMappedData() != VMA_NULL)
    9867  {
    9868  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
    9869  }
    9870 
    9871  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
    9872 
    9873  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
    9874 }
    9875 
    9876 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
    9877 {
    9878  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
    9879  !hAllocation->CanBecomeLost() &&
    9880  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9881  {
    9882  void* pData = VMA_NULL;
    9883  VkResult res = Map(hAllocation, &pData);
    9884  if(res == VK_SUCCESS)
    9885  {
    9886  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
    9887  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
    9888  Unmap(hAllocation);
    9889  }
    9890  else
    9891  {
    9892  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
    9893  }
    9894  }
    9895 }
    9896 
    9897 #if VMA_STATS_STRING_ENABLED
    9898 
    9899 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
    9900 {
    9901  bool dedicatedAllocationsStarted = false;
    9902  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9903  {
    9904  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9905  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9906  VMA_ASSERT(pDedicatedAllocVector);
    9907  if(pDedicatedAllocVector->empty() == false)
    9908  {
    9909  if(dedicatedAllocationsStarted == false)
    9910  {
    9911  dedicatedAllocationsStarted = true;
    9912  json.WriteString("DedicatedAllocations");
    9913  json.BeginObject();
    9914  }
    9915 
    9916  json.BeginString("Type ");
    9917  json.ContinueString(memTypeIndex);
    9918  json.EndString();
    9919 
    9920  json.BeginArray();
    9921 
    9922  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
    9923  {
    9924  json.BeginObject(true);
    9925  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
    9926  hAlloc->PrintParameters(json);
    9927  json.EndObject();
    9928  }
    9929 
    9930  json.EndArray();
    9931  }
    9932  }
    9933  if(dedicatedAllocationsStarted)
    9934  {
    9935  json.EndObject();
    9936  }
    9937 
    9938  {
    9939  bool allocationsStarted = false;
    9940  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9941  {
    9942  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
    9943  {
    9944  if(allocationsStarted == false)
    9945  {
    9946  allocationsStarted = true;
    9947  json.WriteString("DefaultPools");
    9948  json.BeginObject();
    9949  }
    9950 
    9951  json.BeginString("Type ");
    9952  json.ContinueString(memTypeIndex);
    9953  json.EndString();
    9954 
    9955  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
    9956  }
    9957  }
    9958  if(allocationsStarted)
    9959  {
    9960  json.EndObject();
    9961  }
    9962  }
    9963 
    9964  {
    9965  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9966  const size_t poolCount = m_Pools.size();
    9967  if(poolCount > 0)
    9968  {
    9969  json.WriteString("Pools");
    9970  json.BeginObject();
    9971  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
    9972  {
    9973  json.BeginString();
    9974  json.ContinueString(m_Pools[poolIndex]->GetId());
    9975  json.EndString();
    9976 
    9977  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
    9978  }
    9979  json.EndObject();
    9980  }
    9981  }
    9982 }
    9983 
    9984 #endif // #if VMA_STATS_STRING_ENABLED
    9985 
    9987 // Public interface
    9988 
    9989 VkResult vmaCreateAllocator(
    9990  const VmaAllocatorCreateInfo* pCreateInfo,
    9991  VmaAllocator* pAllocator)
    9992 {
    9993  VMA_ASSERT(pCreateInfo && pAllocator);
    9994  VMA_DEBUG_LOG("vmaCreateAllocator");
    9995  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
    9996  return (*pAllocator)->Init(pCreateInfo);
    9997 }
    9998 
    9999 void vmaDestroyAllocator(
    10000  VmaAllocator allocator)
    10001 {
    10002  if(allocator != VK_NULL_HANDLE)
    10003  {
    10004  VMA_DEBUG_LOG("vmaDestroyAllocator");
    10005  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
    10006  vma_delete(&allocationCallbacks, allocator);
    10007  }
    10008 }
    10009 
    10010 void vmaGetPhysicalDeviceProperties(
    10011  VmaAllocator allocator,
    10012  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
    10013 {
    10014  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
    10015  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
    10016 }
    10017 
    10018 void vmaGetMemoryProperties(
    10019  VmaAllocator allocator,
    10020  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
    10021 {
    10022  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
    10023  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
    10024 }
    10025 
    10026 void vmaGetMemoryTypeProperties(
    10027  VmaAllocator allocator,
    10028  uint32_t memoryTypeIndex,
    10029  VkMemoryPropertyFlags* pFlags)
    10030 {
    10031  VMA_ASSERT(allocator && pFlags);
    10032  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
    10033  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
    10034 }
    10035 
    10036 void vmaSetCurrentFrameIndex(
    10037  VmaAllocator allocator,
    10038  uint32_t frameIndex)
    10039 {
    10040  VMA_ASSERT(allocator);
    10041  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
    10042 
    10043  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10044 
    10045  allocator->SetCurrentFrameIndex(frameIndex);
    10046 }
    10047 
    10048 void vmaCalculateStats(
    10049  VmaAllocator allocator,
    10050  VmaStats* pStats)
    10051 {
    10052  VMA_ASSERT(allocator && pStats);
    10053  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10054  allocator->CalculateStats(pStats);
    10055 }
    10056 
    10057 #if VMA_STATS_STRING_ENABLED
    10058 
    10059 void vmaBuildStatsString(
    10060  VmaAllocator allocator,
    10061  char** ppStatsString,
    10062  VkBool32 detailedMap)
    10063 {
    10064  VMA_ASSERT(allocator && ppStatsString);
    10065  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10066 
    10067  VmaStringBuilder sb(allocator);
    10068  {
    10069  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
    10070  json.BeginObject();
    10071 
    10072  VmaStats stats;
    10073  allocator->CalculateStats(&stats);
    10074 
    10075  json.WriteString("Total");
    10076  VmaPrintStatInfo(json, stats.total);
    10077 
    10078  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
    10079  {
    10080  json.BeginString("Heap ");
    10081  json.ContinueString(heapIndex);
    10082  json.EndString();
    10083  json.BeginObject();
    10084 
    10085  json.WriteString("Size");
    10086  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
    10087 
    10088  json.WriteString("Flags");
    10089  json.BeginArray(true);
    10090  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
    10091  {
    10092  json.WriteString("DEVICE_LOCAL");
    10093  }
    10094  json.EndArray();
    10095 
    10096  if(stats.memoryHeap[heapIndex].blockCount > 0)
    10097  {
    10098  json.WriteString("Stats");
    10099  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
    10100  }
    10101 
    10102  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
    10103  {
    10104  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
    10105  {
    10106  json.BeginString("Type ");
    10107  json.ContinueString(typeIndex);
    10108  json.EndString();
    10109 
    10110  json.BeginObject();
    10111 
    10112  json.WriteString("Flags");
    10113  json.BeginArray(true);
    10114  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
    10115  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
    10116  {
    10117  json.WriteString("DEVICE_LOCAL");
    10118  }
    10119  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    10120  {
    10121  json.WriteString("HOST_VISIBLE");
    10122  }
    10123  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
    10124  {
    10125  json.WriteString("HOST_COHERENT");
    10126  }
    10127  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
    10128  {
    10129  json.WriteString("HOST_CACHED");
    10130  }
    10131  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
    10132  {
    10133  json.WriteString("LAZILY_ALLOCATED");
    10134  }
    10135  json.EndArray();
    10136 
    10137  if(stats.memoryType[typeIndex].blockCount > 0)
    10138  {
    10139  json.WriteString("Stats");
    10140  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
    10141  }
    10142 
    10143  json.EndObject();
    10144  }
    10145  }
    10146 
    10147  json.EndObject();
    10148  }
    10149  if(detailedMap == VK_TRUE)
    10150  {
    10151  allocator->PrintDetailedMap(json);
    10152  }
    10153 
    10154  json.EndObject();
    10155  }
    10156 
    10157  const size_t len = sb.GetLength();
    10158  char* const pChars = vma_new_array(allocator, char, len + 1);
    10159  if(len > 0)
    10160  {
    10161  memcpy(pChars, sb.GetData(), len);
    10162  }
    10163  pChars[len] = '\0';
    10164  *ppStatsString = pChars;
    10165 }
    10166 
    10167 void vmaFreeStatsString(
    10168  VmaAllocator allocator,
    10169  char* pStatsString)
    10170 {
    10171  if(pStatsString != VMA_NULL)
    10172  {
    10173  VMA_ASSERT(allocator);
    10174  size_t len = strlen(pStatsString);
    10175  vma_delete_array(allocator, pStatsString, len + 1);
    10176  }
    10177 }
    10178 
    10179 #endif // #if VMA_STATS_STRING_ENABLED
    10180 
    10181 /*
    10182 This function is not protected by any mutex because it just reads immutable data.
    10183 */
    10184 VkResult vmaFindMemoryTypeIndex(
    10185  VmaAllocator allocator,
    10186  uint32_t memoryTypeBits,
    10187  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10188  uint32_t* pMemoryTypeIndex)
    10189 {
    10190  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10191  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10192  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10193 
    10194  if(pAllocationCreateInfo->memoryTypeBits != 0)
    10195  {
    10196  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
    10197  }
    10198 
    10199  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
    10200  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
    10201 
    10202  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    10203  if(mapped)
    10204  {
    10205  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10206  }
    10207 
    10208  // Convert usage to requiredFlags and preferredFlags.
    10209  switch(pAllocationCreateInfo->usage)
    10210  {
    10211  case VMA_MEMORY_USAGE_UNKNOWN:
    10212  break;
    10213  case VMA_MEMORY_USAGE_GPU_ONLY:
    10214  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10215  {
    10216  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10217  }
    10218  break;
    10219  case VMA_MEMORY_USAGE_CPU_ONLY:
    10220  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    10221  break;
    10222  case VMA_MEMORY_USAGE_CPU_TO_GPU:
    10223  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10224  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10225  {
    10226  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10227  }
    10228  break;
    10229  case VMA_MEMORY_USAGE_GPU_TO_CPU:
    10230  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10231  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
    10232  break;
    10233  default:
    10234  break;
    10235  }
    10236 
    10237  *pMemoryTypeIndex = UINT32_MAX;
    10238  uint32_t minCost = UINT32_MAX;
    10239  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
    10240  memTypeIndex < allocator->GetMemoryTypeCount();
    10241  ++memTypeIndex, memTypeBit <<= 1)
    10242  {
    10243  // This memory type is acceptable according to memoryTypeBits bitmask.
    10244  if((memTypeBit & memoryTypeBits) != 0)
    10245  {
    10246  const VkMemoryPropertyFlags currFlags =
    10247  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
    10248  // This memory type contains requiredFlags.
    10249  if((requiredFlags & ~currFlags) == 0)
    10250  {
    10251  // Calculate cost as number of bits from preferredFlags not present in this memory type.
    10252  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
    10253  // Remember memory type with lowest cost.
    10254  if(currCost < minCost)
    10255  {
    10256  *pMemoryTypeIndex = memTypeIndex;
    10257  if(currCost == 0)
    10258  {
    10259  return VK_SUCCESS;
    10260  }
    10261  minCost = currCost;
    10262  }
    10263  }
    10264  }
    10265  }
    10266  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
    10267 }
    10268 
    10269 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    10270  VmaAllocator allocator,
    10271  const VkBufferCreateInfo* pBufferCreateInfo,
    10272  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10273  uint32_t* pMemoryTypeIndex)
    10274 {
    10275  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10276  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
    10277  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10278  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10279 
    10280  const VkDevice hDev = allocator->m_hDevice;
    10281  VkBuffer hBuffer = VK_NULL_HANDLE;
    10282  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
    10283  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
    10284  if(res == VK_SUCCESS)
    10285  {
    10286  VkMemoryRequirements memReq = {};
    10287  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
    10288  hDev, hBuffer, &memReq);
    10289 
    10290  res = vmaFindMemoryTypeIndex(
    10291  allocator,
    10292  memReq.memoryTypeBits,
    10293  pAllocationCreateInfo,
    10294  pMemoryTypeIndex);
    10295 
    10296  allocator->GetVulkanFunctions().vkDestroyBuffer(
    10297  hDev, hBuffer, allocator->GetAllocationCallbacks());
    10298  }
    10299  return res;
    10300 }
    10301 
    10302 VkResult vmaFindMemoryTypeIndexForImageInfo(
    10303  VmaAllocator allocator,
    10304  const VkImageCreateInfo* pImageCreateInfo,
    10305  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10306  uint32_t* pMemoryTypeIndex)
    10307 {
    10308  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10309  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
    10310  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10311  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10312 
    10313  const VkDevice hDev = allocator->m_hDevice;
    10314  VkImage hImage = VK_NULL_HANDLE;
    10315  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
    10316  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
    10317  if(res == VK_SUCCESS)
    10318  {
    10319  VkMemoryRequirements memReq = {};
    10320  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
    10321  hDev, hImage, &memReq);
    10322 
    10323  res = vmaFindMemoryTypeIndex(
    10324  allocator,
    10325  memReq.memoryTypeBits,
    10326  pAllocationCreateInfo,
    10327  pMemoryTypeIndex);
    10328 
    10329  allocator->GetVulkanFunctions().vkDestroyImage(
    10330  hDev, hImage, allocator->GetAllocationCallbacks());
    10331  }
    10332  return res;
    10333 }
    10334 
    10335 VkResult vmaCreatePool(
    10336  VmaAllocator allocator,
    10337  const VmaPoolCreateInfo* pCreateInfo,
    10338  VmaPool* pPool)
    10339 {
    10340  VMA_ASSERT(allocator && pCreateInfo && pPool);
    10341 
    10342  VMA_DEBUG_LOG("vmaCreatePool");
    10343 
    10344  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10345 
    10346  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
    10347 
    10348 #if VMA_RECORDING_ENABLED
    10349  if(allocator->GetRecorder() != VMA_NULL)
    10350  {
    10351  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
    10352  }
    10353 #endif
    10354 
    10355  return res;
    10356 }
    10357 
    10358 void vmaDestroyPool(
    10359  VmaAllocator allocator,
    10360  VmaPool pool)
    10361 {
    10362  VMA_ASSERT(allocator);
    10363 
    10364  if(pool == VK_NULL_HANDLE)
    10365  {
    10366  return;
    10367  }
    10368 
    10369  VMA_DEBUG_LOG("vmaDestroyPool");
    10370 
    10371  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10372 
    10373 #if VMA_RECORDING_ENABLED
    10374  if(allocator->GetRecorder() != VMA_NULL)
    10375  {
    10376  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
    10377  }
    10378 #endif
    10379 
    10380  allocator->DestroyPool(pool);
    10381 }
    10382 
    10383 void vmaGetPoolStats(
    10384  VmaAllocator allocator,
    10385  VmaPool pool,
    10386  VmaPoolStats* pPoolStats)
    10387 {
    10388  VMA_ASSERT(allocator && pool && pPoolStats);
    10389 
    10390  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10391 
    10392  allocator->GetPoolStats(pool, pPoolStats);
    10393 }
    10394 
    10395 void vmaMakePoolAllocationsLost(
    10396  VmaAllocator allocator,
    10397  VmaPool pool,
    10398  size_t* pLostAllocationCount)
    10399 {
    10400  VMA_ASSERT(allocator && pool);
    10401 
    10402  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10403 
    10404  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
    10405 }
    10406 
    10407 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
    10408 {
    10409  VMA_ASSERT(allocator && pool);
    10410 
    10411  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10412 
    10413  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
    10414 
    10415  return allocator->CheckPoolCorruption(pool);
    10416 }
    10417 
    10418 VkResult vmaAllocateMemory(
    10419  VmaAllocator allocator,
    10420  const VkMemoryRequirements* pVkMemoryRequirements,
    10421  const VmaAllocationCreateInfo* pCreateInfo,
    10422  VmaAllocation* pAllocation,
    10423  VmaAllocationInfo* pAllocationInfo)
    10424 {
    10425  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
    10426 
    10427  VMA_DEBUG_LOG("vmaAllocateMemory");
    10428 
    10429  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10430 
    10431  VkResult result = allocator->AllocateMemory(
    10432  *pVkMemoryRequirements,
    10433  false, // requiresDedicatedAllocation
    10434  false, // prefersDedicatedAllocation
    10435  VK_NULL_HANDLE, // dedicatedBuffer
    10436  VK_NULL_HANDLE, // dedicatedImage
    10437  *pCreateInfo,
    10438  VMA_SUBALLOCATION_TYPE_UNKNOWN,
    10439  pAllocation);
    10440 
    10441 #if VMA_RECORDING_ENABLED
    10442  if(allocator->GetRecorder() != VMA_NULL)
    10443  {
    10444  allocator->GetRecorder()->RecordAllocateMemory(
    10445  allocator->GetCurrentFrameIndex(),
    10446  *pVkMemoryRequirements,
    10447  *pCreateInfo,
    10448  *pAllocation);
    10449  }
    10450 #endif
    10451 
    10452  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
    10453  {
    10454  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10455  }
    10456 
    10457  return result;
    10458 }
    10459 
    10460 VkResult vmaAllocateMemoryForBuffer(
    10461  VmaAllocator allocator,
    10462  VkBuffer buffer,
    10463  const VmaAllocationCreateInfo* pCreateInfo,
    10464  VmaAllocation* pAllocation,
    10465  VmaAllocationInfo* pAllocationInfo)
    10466 {
    10467  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10468 
    10469  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
    10470 
    10471  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10472 
    10473  VkMemoryRequirements vkMemReq = {};
    10474  bool requiresDedicatedAllocation = false;
    10475  bool prefersDedicatedAllocation = false;
    10476  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
    10477  requiresDedicatedAllocation,
    10478  prefersDedicatedAllocation);
    10479 
    10480  VkResult result = allocator->AllocateMemory(
    10481  vkMemReq,
    10482  requiresDedicatedAllocation,
    10483  prefersDedicatedAllocation,
    10484  buffer, // dedicatedBuffer
    10485  VK_NULL_HANDLE, // dedicatedImage
    10486  *pCreateInfo,
    10487  VMA_SUBALLOCATION_TYPE_BUFFER,
    10488  pAllocation);
    10489 
    10490 #if VMA_RECORDING_ENABLED
    10491  if(allocator->GetRecorder() != VMA_NULL)
    10492  {
    10493  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
    10494  allocator->GetCurrentFrameIndex(),
    10495  vkMemReq,
    10496  requiresDedicatedAllocation,
    10497  prefersDedicatedAllocation,
    10498  *pCreateInfo,
    10499  *pAllocation);
    10500  }
    10501 #endif
    10502 
    10503  if(pAllocationInfo && result == VK_SUCCESS)
    10504  {
    10505  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10506  }
    10507 
    10508  return result;
    10509 }
    10510 
    10511 VkResult vmaAllocateMemoryForImage(
    10512  VmaAllocator allocator,
    10513  VkImage image,
    10514  const VmaAllocationCreateInfo* pCreateInfo,
    10515  VmaAllocation* pAllocation,
    10516  VmaAllocationInfo* pAllocationInfo)
    10517 {
    10518  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10519 
    10520  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
    10521 
    10522  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10523 
    10524  VkMemoryRequirements vkMemReq = {};
    10525  bool requiresDedicatedAllocation = false;
    10526  bool prefersDedicatedAllocation = false;
    10527  allocator->GetImageMemoryRequirements(image, vkMemReq,
    10528  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10529 
    10530  VkResult result = allocator->AllocateMemory(
    10531  vkMemReq,
    10532  requiresDedicatedAllocation,
    10533  prefersDedicatedAllocation,
    10534  VK_NULL_HANDLE, // dedicatedBuffer
    10535  image, // dedicatedImage
    10536  *pCreateInfo,
    10537  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
    10538  pAllocation);
    10539 
    10540 #if VMA_RECORDING_ENABLED
    10541  if(allocator->GetRecorder() != VMA_NULL)
    10542  {
    10543  allocator->GetRecorder()->RecordAllocateMemoryForImage(
    10544  allocator->GetCurrentFrameIndex(),
    10545  vkMemReq,
    10546  requiresDedicatedAllocation,
    10547  prefersDedicatedAllocation,
    10548  *pCreateInfo,
    10549  *pAllocation);
    10550  }
    10551 #endif
    10552 
    10553  if(pAllocationInfo && result == VK_SUCCESS)
    10554  {
    10555  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10556  }
    10557 
    10558  return result;
    10559 }
    10560 
    10561 void vmaFreeMemory(
    10562  VmaAllocator allocator,
    10563  VmaAllocation allocation)
    10564 {
    10565  VMA_ASSERT(allocator);
    10566 
    10567  if(allocation == VK_NULL_HANDLE)
    10568  {
    10569  return;
    10570  }
    10571 
    10572  VMA_DEBUG_LOG("vmaFreeMemory");
    10573 
    10574  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10575 
    10576 #if VMA_RECORDING_ENABLED
    10577  if(allocator->GetRecorder() != VMA_NULL)
    10578  {
    10579  allocator->GetRecorder()->RecordFreeMemory(
    10580  allocator->GetCurrentFrameIndex(),
    10581  allocation);
    10582  }
    10583 #endif
    10584 
    10585  allocator->FreeMemory(allocation);
    10586 }
    10587 
    10588 void vmaGetAllocationInfo(
    10589  VmaAllocator allocator,
    10590  VmaAllocation allocation,
    10591  VmaAllocationInfo* pAllocationInfo)
    10592 {
    10593  VMA_ASSERT(allocator && allocation && pAllocationInfo);
    10594 
    10595  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10596 
    10597 #if VMA_RECORDING_ENABLED
    10598  if(allocator->GetRecorder() != VMA_NULL)
    10599  {
    10600  allocator->GetRecorder()->RecordGetAllocationInfo(
    10601  allocator->GetCurrentFrameIndex(),
    10602  allocation);
    10603  }
    10604 #endif
    10605 
    10606  allocator->GetAllocationInfo(allocation, pAllocationInfo);
    10607 }
    10608 
    10609 VkBool32 vmaTouchAllocation(
    10610  VmaAllocator allocator,
    10611  VmaAllocation allocation)
    10612 {
    10613  VMA_ASSERT(allocator && allocation);
    10614 
    10615  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10616 
    10617 #if VMA_RECORDING_ENABLED
    10618  if(allocator->GetRecorder() != VMA_NULL)
    10619  {
    10620  allocator->GetRecorder()->RecordTouchAllocation(
    10621  allocator->GetCurrentFrameIndex(),
    10622  allocation);
    10623  }
    10624 #endif
    10625 
    10626  return allocator->TouchAllocation(allocation);
    10627 }
    10628 
    10629 void vmaSetAllocationUserData(
    10630  VmaAllocator allocator,
    10631  VmaAllocation allocation,
    10632  void* pUserData)
    10633 {
    10634  VMA_ASSERT(allocator && allocation);
    10635 
    10636  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10637 
    10638  allocation->SetUserData(allocator, pUserData);
    10639 
    10640 #if VMA_RECORDING_ENABLED
    10641  if(allocator->GetRecorder() != VMA_NULL)
    10642  {
    10643  allocator->GetRecorder()->RecordSetAllocationUserData(
    10644  allocator->GetCurrentFrameIndex(),
    10645  allocation,
    10646  pUserData);
    10647  }
    10648 #endif
    10649 }
    10650 
    10651 void vmaCreateLostAllocation(
    10652  VmaAllocator allocator,
    10653  VmaAllocation* pAllocation)
    10654 {
    10655  VMA_ASSERT(allocator && pAllocation);
    10656 
    10657  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
    10658 
    10659  allocator->CreateLostAllocation(pAllocation);
    10660 
    10661 #if VMA_RECORDING_ENABLED
    10662  if(allocator->GetRecorder() != VMA_NULL)
    10663  {
    10664  allocator->GetRecorder()->RecordCreateLostAllocation(
    10665  allocator->GetCurrentFrameIndex(),
    10666  *pAllocation);
    10667  }
    10668 #endif
    10669 }
    10670 
    10671 VkResult vmaMapMemory(
    10672  VmaAllocator allocator,
    10673  VmaAllocation allocation,
    10674  void** ppData)
    10675 {
    10676  VMA_ASSERT(allocator && allocation && ppData);
    10677 
    10678  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10679 
    10680  VkResult res = allocator->Map(allocation, ppData);
    10681 
    10682 #if VMA_RECORDING_ENABLED
    10683  if(allocator->GetRecorder() != VMA_NULL)
    10684  {
    10685  allocator->GetRecorder()->RecordMapMemory(
    10686  allocator->GetCurrentFrameIndex(),
    10687  allocation);
    10688  }
    10689 #endif
    10690 
    10691  return res;
    10692 }
    10693 
    10694 void vmaUnmapMemory(
    10695  VmaAllocator allocator,
    10696  VmaAllocation allocation)
    10697 {
    10698  VMA_ASSERT(allocator && allocation);
    10699 
    10700  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10701 
    10702 #if VMA_RECORDING_ENABLED
    10703  if(allocator->GetRecorder() != VMA_NULL)
    10704  {
    10705  allocator->GetRecorder()->RecordUnmapMemory(
    10706  allocator->GetCurrentFrameIndex(),
    10707  allocation);
    10708  }
    10709 #endif
    10710 
    10711  allocator->Unmap(allocation);
    10712 }
    10713 
    10714 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10715 {
    10716  VMA_ASSERT(allocator && allocation);
    10717 
    10718  VMA_DEBUG_LOG("vmaFlushAllocation");
    10719 
    10720  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10721 
    10722  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
    10723 
    10724 #if VMA_RECORDING_ENABLED
    10725  if(allocator->GetRecorder() != VMA_NULL)
    10726  {
    10727  allocator->GetRecorder()->RecordFlushAllocation(
    10728  allocator->GetCurrentFrameIndex(),
    10729  allocation, offset, size);
    10730  }
    10731 #endif
    10732 }
    10733 
    10734 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10735 {
    10736  VMA_ASSERT(allocator && allocation);
    10737 
    10738  VMA_DEBUG_LOG("vmaInvalidateAllocation");
    10739 
    10740  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10741 
    10742  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
    10743 
    10744 #if VMA_RECORDING_ENABLED
    10745  if(allocator->GetRecorder() != VMA_NULL)
    10746  {
    10747  allocator->GetRecorder()->RecordInvalidateAllocation(
    10748  allocator->GetCurrentFrameIndex(),
    10749  allocation, offset, size);
    10750  }
    10751 #endif
    10752 }
    10753 
    10754 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
    10755 {
    10756  VMA_ASSERT(allocator);
    10757 
    10758  VMA_DEBUG_LOG("vmaCheckCorruption");
    10759 
    10760  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10761 
    10762  return allocator->CheckCorruption(memoryTypeBits);
    10763 }
    10764 
    10765 VkResult vmaDefragment(
    10766  VmaAllocator allocator,
    10767  VmaAllocation* pAllocations,
    10768  size_t allocationCount,
    10769  VkBool32* pAllocationsChanged,
    10770  const VmaDefragmentationInfo *pDefragmentationInfo,
    10771  VmaDefragmentationStats* pDefragmentationStats)
    10772 {
    10773  VMA_ASSERT(allocator && pAllocations);
    10774 
    10775  VMA_DEBUG_LOG("vmaDefragment");
    10776 
    10777  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10778 
    10779  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
    10780 }
    10781 
    10782 VkResult vmaBindBufferMemory(
    10783  VmaAllocator allocator,
    10784  VmaAllocation allocation,
    10785  VkBuffer buffer)
    10786 {
    10787  VMA_ASSERT(allocator && allocation && buffer);
    10788 
    10789  VMA_DEBUG_LOG("vmaBindBufferMemory");
    10790 
    10791  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10792 
    10793  return allocator->BindBufferMemory(allocation, buffer);
    10794 }
    10795 
    10796 VkResult vmaBindImageMemory(
    10797  VmaAllocator allocator,
    10798  VmaAllocation allocation,
    10799  VkImage image)
    10800 {
    10801  VMA_ASSERT(allocator && allocation && image);
    10802 
    10803  VMA_DEBUG_LOG("vmaBindImageMemory");
    10804 
    10805  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10806 
    10807  return allocator->BindImageMemory(allocation, image);
    10808 }
    10809 
    10810 VkResult vmaCreateBuffer(
    10811  VmaAllocator allocator,
    10812  const VkBufferCreateInfo* pBufferCreateInfo,
    10813  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10814  VkBuffer* pBuffer,
    10815  VmaAllocation* pAllocation,
    10816  VmaAllocationInfo* pAllocationInfo)
    10817 {
    10818  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
    10819 
    10820  VMA_DEBUG_LOG("vmaCreateBuffer");
    10821 
    10822  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10823 
    10824  *pBuffer = VK_NULL_HANDLE;
    10825  *pAllocation = VK_NULL_HANDLE;
    10826 
    10827  // 1. Create VkBuffer.
    10828  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
    10829  allocator->m_hDevice,
    10830  pBufferCreateInfo,
    10831  allocator->GetAllocationCallbacks(),
    10832  pBuffer);
    10833  if(res >= 0)
    10834  {
    10835  // 2. vkGetBufferMemoryRequirements.
    10836  VkMemoryRequirements vkMemReq = {};
    10837  bool requiresDedicatedAllocation = false;
    10838  bool prefersDedicatedAllocation = false;
    10839  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
    10840  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10841 
    10842  // Make sure alignment requirements for specific buffer usages reported
    10843  // in Physical Device Properties are included in alignment reported by memory requirements.
    10844  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
    10845  {
    10846  VMA_ASSERT(vkMemReq.alignment %
    10847  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
    10848  }
    10849  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
    10850  {
    10851  VMA_ASSERT(vkMemReq.alignment %
    10852  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
    10853  }
    10854  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
    10855  {
    10856  VMA_ASSERT(vkMemReq.alignment %
    10857  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
    10858  }
    10859 
    10860  // 3. Allocate memory using allocator.
    10861  res = allocator->AllocateMemory(
    10862  vkMemReq,
    10863  requiresDedicatedAllocation,
    10864  prefersDedicatedAllocation,
    10865  *pBuffer, // dedicatedBuffer
    10866  VK_NULL_HANDLE, // dedicatedImage
    10867  *pAllocationCreateInfo,
    10868  VMA_SUBALLOCATION_TYPE_BUFFER,
    10869  pAllocation);
    10870 
    10871 #if VMA_RECORDING_ENABLED
    10872  if(allocator->GetRecorder() != VMA_NULL)
    10873  {
    10874  allocator->GetRecorder()->RecordCreateBuffer(
    10875  allocator->GetCurrentFrameIndex(),
    10876  *pBufferCreateInfo,
    10877  *pAllocationCreateInfo,
    10878  *pAllocation);
    10879  }
    10880 #endif
    10881 
    10882  if(res >= 0)
    10883  {
    10884  // 3. Bind buffer with memory.
    10885  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
    10886  if(res >= 0)
    10887  {
    10888  // All steps succeeded.
    10889  #if VMA_STATS_STRING_ENABLED
    10890  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
    10891  #endif
    10892  if(pAllocationInfo != VMA_NULL)
    10893  {
    10894  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10895  }
    10896 
    10897  return VK_SUCCESS;
    10898  }
    10899  allocator->FreeMemory(*pAllocation);
    10900  *pAllocation = VK_NULL_HANDLE;
    10901  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10902  *pBuffer = VK_NULL_HANDLE;
    10903  return res;
    10904  }
    10905  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10906  *pBuffer = VK_NULL_HANDLE;
    10907  return res;
    10908  }
    10909  return res;
    10910 }
    10911 
    10912 void vmaDestroyBuffer(
    10913  VmaAllocator allocator,
    10914  VkBuffer buffer,
    10915  VmaAllocation allocation)
    10916 {
    10917  VMA_ASSERT(allocator);
    10918 
    10919  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    10920  {
    10921  return;
    10922  }
    10923 
    10924  VMA_DEBUG_LOG("vmaDestroyBuffer");
    10925 
    10926  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10927 
    10928 #if VMA_RECORDING_ENABLED
    10929  if(allocator->GetRecorder() != VMA_NULL)
    10930  {
    10931  allocator->GetRecorder()->RecordDestroyBuffer(
    10932  allocator->GetCurrentFrameIndex(),
    10933  allocation);
    10934  }
    10935 #endif
    10936 
    10937  if(buffer != VK_NULL_HANDLE)
    10938  {
    10939  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
    10940  }
    10941 
    10942  if(allocation != VK_NULL_HANDLE)
    10943  {
    10944  allocator->FreeMemory(allocation);
    10945  }
    10946 }
    10947 
    10948 VkResult vmaCreateImage(
    10949  VmaAllocator allocator,
    10950  const VkImageCreateInfo* pImageCreateInfo,
    10951  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10952  VkImage* pImage,
    10953  VmaAllocation* pAllocation,
    10954  VmaAllocationInfo* pAllocationInfo)
    10955 {
    10956  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
    10957 
    10958  VMA_DEBUG_LOG("vmaCreateImage");
    10959 
    10960  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10961 
    10962  *pImage = VK_NULL_HANDLE;
    10963  *pAllocation = VK_NULL_HANDLE;
    10964 
    10965  // 1. Create VkImage.
    10966  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
    10967  allocator->m_hDevice,
    10968  pImageCreateInfo,
    10969  allocator->GetAllocationCallbacks(),
    10970  pImage);
    10971  if(res >= 0)
    10972  {
    10973  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
    10974  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
    10975  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
    10976 
    10977  // 2. Allocate memory using allocator.
    10978  VkMemoryRequirements vkMemReq = {};
    10979  bool requiresDedicatedAllocation = false;
    10980  bool prefersDedicatedAllocation = false;
    10981  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
    10982  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10983 
    10984  res = allocator->AllocateMemory(
    10985  vkMemReq,
    10986  requiresDedicatedAllocation,
    10987  prefersDedicatedAllocation,
    10988  VK_NULL_HANDLE, // dedicatedBuffer
    10989  *pImage, // dedicatedImage
    10990  *pAllocationCreateInfo,
    10991  suballocType,
    10992  pAllocation);
    10993 
    10994 #if VMA_RECORDING_ENABLED
    10995  if(allocator->GetRecorder() != VMA_NULL)
    10996  {
    10997  allocator->GetRecorder()->RecordCreateImage(
    10998  allocator->GetCurrentFrameIndex(),
    10999  *pImageCreateInfo,
    11000  *pAllocationCreateInfo,
    11001  *pAllocation);
    11002  }
    11003 #endif
    11004 
    11005  if(res >= 0)
    11006  {
    11007  // 3. Bind image with memory.
    11008  res = allocator->BindImageMemory(*pAllocation, *pImage);
    11009  if(res >= 0)
    11010  {
    11011  // All steps succeeded.
    11012  #if VMA_STATS_STRING_ENABLED
    11013  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
    11014  #endif
    11015  if(pAllocationInfo != VMA_NULL)
    11016  {
    11017  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    11018  }
    11019 
    11020  return VK_SUCCESS;
    11021  }
    11022  allocator->FreeMemory(*pAllocation);
    11023  *pAllocation = VK_NULL_HANDLE;
    11024  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    11025  *pImage = VK_NULL_HANDLE;
    11026  return res;
    11027  }
    11028  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    11029  *pImage = VK_NULL_HANDLE;
    11030  return res;
    11031  }
    11032  return res;
    11033 }
    11034 
    11035 void vmaDestroyImage(
    11036  VmaAllocator allocator,
    11037  VkImage image,
    11038  VmaAllocation allocation)
    11039 {
    11040  VMA_ASSERT(allocator);
    11041 
    11042  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    11043  {
    11044  return;
    11045  }
    11046 
    11047  VMA_DEBUG_LOG("vmaDestroyImage");
    11048 
    11049  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    11050 
    11051 #if VMA_RECORDING_ENABLED
    11052  if(allocator->GetRecorder() != VMA_NULL)
    11053  {
    11054  allocator->GetRecorder()->RecordDestroyImage(
    11055  allocator->GetCurrentFrameIndex(),
    11056  allocation);
    11057  }
    11058 #endif
    11059 
    11060  if(image != VK_NULL_HANDLE)
    11061  {
    11062  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
    11063  }
    11064  if(allocation != VK_NULL_HANDLE)
    11065  {
    11066  allocator->FreeMemory(allocation);
    11067  }
    11068 }
    11069 
    11070 #endif // #ifdef VMA_IMPLEMENTATION
    VmaVulkanFunctions::vkGetPhysicalDeviceProperties
    PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
    Definition: vk_mem_alloc.h:1345
    +
    VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
    Set this flag if the allocation should have its own memory block.
    Definition: vk_mem_alloc.h:1658
    vmaUnmapMemory
    void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
    Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
    -
    VmaAllocatorCreateInfo::physicalDevice
    VkPhysicalDevice physicalDevice
    Vulkan physical device.
    Definition: vk_mem_alloc.h:1367
    +
    VmaAllocatorCreateInfo::physicalDevice
    VkPhysicalDevice physicalDevice
    Vulkan physical device.
    Definition: vk_mem_alloc.h:1414
    vmaDefragment
    VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
    Compacts memory by moving allocations.
    vmaInvalidateAllocation
    void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    Invalidates memory of given allocation.
    VmaAllocation
    Represents single memory allocation.
    -
    VMA_RECORD_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1329
    -
    VmaVulkanFunctions::vkCreateBuffer
    PFN_vkCreateBuffer vkCreateBuffer
    Definition: vk_mem_alloc.h:1310
    +
    VMA_RECORD_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1376
    +
    VmaVulkanFunctions::vkCreateBuffer
    PFN_vkCreateBuffer vkCreateBuffer
    Definition: vk_mem_alloc.h:1357
    vmaFreeStatsString
    void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
    VmaStats
    struct VmaStats VmaStats
    General statistics from current state of Allocator.
    -
    VMA_MEMORY_USAGE_GPU_ONLY
    Definition: vk_mem_alloc.h:1568
    -
    VmaVulkanFunctions::vkMapMemory
    PFN_vkMapMemory vkMapMemory
    Definition: vk_mem_alloc.h:1302
    -
    VmaAllocationInfo::deviceMemory
    VkDeviceMemory deviceMemory
    Handle to Vulkan memory object.
    Definition: vk_mem_alloc.h:1957
    -
    VmaAllocatorCreateInfo::flags
    VmaAllocatorCreateFlags flags
    Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
    Definition: vk_mem_alloc.h:1364
    -
    VmaDefragmentationInfo::maxAllocationsToMove
    uint32_t maxAllocationsToMove
    Maximum number of allocations that can be moved to different place.
    Definition: vk_mem_alloc.h:2202
    -
    VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
    Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
    Definition: vk_mem_alloc.h:1787
    -
    VMA_RECORDING_ENABLED
    #define VMA_RECORDING_ENABLED
    Definition: vk_mem_alloc.h:1341
    +
    VMA_MEMORY_USAGE_GPU_ONLY
    Definition: vk_mem_alloc.h:1615
    +
    VmaVulkanFunctions::vkMapMemory
    PFN_vkMapMemory vkMapMemory
    Definition: vk_mem_alloc.h:1349
    +
    VmaAllocationInfo::deviceMemory
    VkDeviceMemory deviceMemory
    Handle to Vulkan memory object.
    Definition: vk_mem_alloc.h:2004
    +
    VmaAllocatorCreateInfo::flags
    VmaAllocatorCreateFlags flags
    Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
    Definition: vk_mem_alloc.h:1411
    +
    VmaDefragmentationInfo::maxAllocationsToMove
    uint32_t maxAllocationsToMove
    Maximum number of allocations that can be moved to different place.
    Definition: vk_mem_alloc.h:2249
    +
    VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
    Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
    Definition: vk_mem_alloc.h:1834
    +
    VMA_RECORDING_ENABLED
    #define VMA_RECORDING_ENABLED
    Definition: vk_mem_alloc.h:1388
    vmaMakePoolAllocationsLost
    void vmaMakePoolAllocationsLost(VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
    Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInf...
    -
    VmaPoolStats::size
    VkDeviceSize size
    Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
    Definition: vk_mem_alloc.h:1841
    -
    VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
    Definition: vk_mem_alloc.h:1648
    -
    VmaAllocatorCreateFlags
    VkFlags VmaAllocatorCreateFlags
    Definition: vk_mem_alloc.h:1291
    -
    VmaAllocationCreateInfo::preferredFlags
    VkMemoryPropertyFlags preferredFlags
    Flags that preferably should be set in a memory type chosen for an allocation.
    Definition: vk_mem_alloc.h:1686
    -
    VMA_MEMORY_USAGE_MAX_ENUM
    Definition: vk_mem_alloc.h:1595
    -
    VmaAllocatorCreateInfo::pAllocationCallbacks
    const VkAllocationCallbacks * pAllocationCallbacks
    Custom CPU memory allocation callbacks. Optional.
    Definition: vk_mem_alloc.h:1376
    +
    VmaPoolStats::size
    VkDeviceSize size
    Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
    Definition: vk_mem_alloc.h:1888
    +
    VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
    Definition: vk_mem_alloc.h:1695
    +
    VmaAllocatorCreateFlags
    VkFlags VmaAllocatorCreateFlags
    Definition: vk_mem_alloc.h:1338
    +
    VmaAllocationCreateInfo::preferredFlags
    VkMemoryPropertyFlags preferredFlags
    Flags that preferably should be set in a memory type chosen for an allocation.
    Definition: vk_mem_alloc.h:1733
    +
    VMA_MEMORY_USAGE_MAX_ENUM
    Definition: vk_mem_alloc.h:1642
    +
    VmaAllocatorCreateInfo::pAllocationCallbacks
    const VkAllocationCallbacks * pAllocationCallbacks
    Custom CPU memory allocation callbacks. Optional.
    Definition: vk_mem_alloc.h:1423
    vmaCalculateStats
    void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
    Retrieves statistics from current state of the Allocator.
    -
    VmaAllocatorCreateInfo::pVulkanFunctions
    const VmaVulkanFunctions * pVulkanFunctions
    Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
    Definition: vk_mem_alloc.h:1429
    -
    VmaAllocatorCreateInfo
    Description of a Allocator to be created.
    Definition: vk_mem_alloc.h:1361
    +
    VmaAllocatorCreateInfo::pVulkanFunctions
    const VmaVulkanFunctions * pVulkanFunctions
    Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
    Definition: vk_mem_alloc.h:1476
    +
    VmaAllocatorCreateInfo
    Description of a Allocator to be created.
    Definition: vk_mem_alloc.h:1408
    vmaDestroyAllocator
    void vmaDestroyAllocator(VmaAllocator allocator)
    Destroys allocator object.
    -
    VmaAllocationCreateFlagBits
    VmaAllocationCreateFlagBits
    Flags to be passed as VmaAllocationCreateInfo::flags.
    Definition: vk_mem_alloc.h:1599
    +
    VmaAllocationCreateFlagBits
    VmaAllocationCreateFlagBits
    Flags to be passed as VmaAllocationCreateInfo::flags.
    Definition: vk_mem_alloc.h:1646
    vmaGetAllocationInfo
    void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
    Returns current information about specified allocation and atomically marks it as used in current fra...
    -
    VmaStatInfo::allocationSizeMax
    VkDeviceSize allocationSizeMax
    Definition: vk_mem_alloc.h:1501
    -
    VmaVulkanFunctions::vkBindImageMemory
    PFN_vkBindImageMemory vkBindImageMemory
    Definition: vk_mem_alloc.h:1307
    -
    VmaStatInfo::unusedBytes
    VkDeviceSize unusedBytes
    Total number of bytes occupied by unused ranges.
    Definition: vk_mem_alloc.h:1500
    -
    VmaDefragmentationStats
    Statistics returned by function vmaDefragment().
    Definition: vk_mem_alloc.h:2206
    +
    VmaStatInfo::allocationSizeMax
    VkDeviceSize allocationSizeMax
    Definition: vk_mem_alloc.h:1548
    +
    VmaVulkanFunctions::vkBindImageMemory
    PFN_vkBindImageMemory vkBindImageMemory
    Definition: vk_mem_alloc.h:1354
    +
    VmaStatInfo::unusedBytes
    VkDeviceSize unusedBytes
    Total number of bytes occupied by unused ranges.
    Definition: vk_mem_alloc.h:1547
    +
    VmaDefragmentationStats
    Statistics returned by function vmaDefragment().
    Definition: vk_mem_alloc.h:2253
    vmaFreeMemory
    void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
    Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
    -
    VmaAllocatorCreateInfo::frameInUseCount
    uint32_t frameInUseCount
    Maximum number of additional frames that are in use at the same time as current frame.
    Definition: vk_mem_alloc.h:1393
    -
    VmaStats::total
    VmaStatInfo total
    Definition: vk_mem_alloc.h:1510
    -
    VmaDefragmentationStats::deviceMemoryBlocksFreed
    uint32_t deviceMemoryBlocksFreed
    Number of empty VkDeviceMemory objects that have been released to the system.
    Definition: vk_mem_alloc.h:2214
    -
    VmaAllocationCreateInfo::flags
    VmaAllocationCreateFlags flags
    Use VmaAllocationCreateFlagBits enum.
    Definition: vk_mem_alloc.h:1670
    -
    VmaDefragmentationInfo::maxBytesToMove
    VkDeviceSize maxBytesToMove
    Maximum total numbers of bytes that can be copied while moving allocations to different places...
    Definition: vk_mem_alloc.h:2197
    -
    VmaVulkanFunctions::vkGetBufferMemoryRequirements
    PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
    Definition: vk_mem_alloc.h:1308
    -
    PFN_vmaAllocateDeviceMemoryFunction
    void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
    Callback function called after successful vkAllocateMemory.
    Definition: vk_mem_alloc.h:1233
    +
    VmaAllocatorCreateInfo::frameInUseCount
    uint32_t frameInUseCount
    Maximum number of additional frames that are in use at the same time as current frame.
    Definition: vk_mem_alloc.h:1440
    +
    VmaStats::total
    VmaStatInfo total
    Definition: vk_mem_alloc.h:1557
    +
    VmaDefragmentationStats::deviceMemoryBlocksFreed
    uint32_t deviceMemoryBlocksFreed
    Number of empty VkDeviceMemory objects that have been released to the system.
    Definition: vk_mem_alloc.h:2261
    +
    VmaAllocationCreateInfo::flags
    VmaAllocationCreateFlags flags
    Use VmaAllocationCreateFlagBits enum.
    Definition: vk_mem_alloc.h:1717
    +
    VmaDefragmentationInfo::maxBytesToMove
    VkDeviceSize maxBytesToMove
    Maximum total numbers of bytes that can be copied while moving allocations to different places...
    Definition: vk_mem_alloc.h:2244
    +
    VmaVulkanFunctions::vkGetBufferMemoryRequirements
    PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
    Definition: vk_mem_alloc.h:1355
    +
    PFN_vmaAllocateDeviceMemoryFunction
    void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
    Callback function called after successful vkAllocateMemory.
    Definition: vk_mem_alloc.h:1280
    VmaAllocator
    Represents main object of this library initialized.
    -
    VmaAllocatorCreateInfo::device
    VkDevice device
    Vulkan device.
    Definition: vk_mem_alloc.h:1370
    +
    VmaAllocatorCreateInfo::device
    VkDevice device
    Vulkan device.
    Definition: vk_mem_alloc.h:1417
    vmaBindBufferMemory
    VkResult vmaBindBufferMemory(VmaAllocator allocator, VmaAllocation allocation, VkBuffer buffer)
    Binds buffer to allocation.
    -
    VmaPoolCreateInfo
    Describes parameter of created VmaPool.
    Definition: vk_mem_alloc.h:1795
    -
    VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1789
    -
    VmaAllocatorCreateInfo::pRecordSettings
    const VmaRecordSettings * pRecordSettings
    Parameters for recording of VMA calls. Can be null.
    Definition: vk_mem_alloc.h:1436
    -
    VmaAllocationInfo::size
    VkDeviceSize size
    Size of this allocation, in bytes.
    Definition: vk_mem_alloc.h:1967
    +
    VmaPoolCreateInfo
    Describes parameter of created VmaPool.
    Definition: vk_mem_alloc.h:1842
    +
    VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1836
    +
    VmaAllocatorCreateInfo::pRecordSettings
    const VmaRecordSettings * pRecordSettings
    Parameters for recording of VMA calls. Can be null.
    Definition: vk_mem_alloc.h:1483
    +
    VmaAllocationInfo::size
    VkDeviceSize size
    Size of this allocation, in bytes.
    Definition: vk_mem_alloc.h:2014
    vmaGetMemoryTypeProperties
    void vmaGetMemoryTypeProperties(VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
    Given Memory Type Index, returns Property Flags of this memory type.
    -
    VmaVulkanFunctions::vkUnmapMemory
    PFN_vkUnmapMemory vkUnmapMemory
    Definition: vk_mem_alloc.h:1303
    -
    VMA_RECORD_FLUSH_AFTER_CALL_BIT
    Enables flush after recording every function call.
    Definition: vk_mem_alloc.h:1327
    -
    VmaAllocationCreateInfo::pUserData
    void * pUserData
    Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
    Definition: vk_mem_alloc.h:1707
    -
    VmaPoolCreateInfo::minBlockCount
    size_t minBlockCount
    Minimum number of blocks to be always allocated in this pool, even if they stay empty.
    Definition: vk_mem_alloc.h:1811
    -
    VmaPoolStats::allocationCount
    size_t allocationCount
    Number of VmaAllocation objects created from this pool that were not destroyed or lost...
    Definition: vk_mem_alloc.h:1847
    +
    VmaVulkanFunctions::vkUnmapMemory
    PFN_vkUnmapMemory vkUnmapMemory
    Definition: vk_mem_alloc.h:1350
    +
    VMA_RECORD_FLUSH_AFTER_CALL_BIT
    Enables flush after recording every function call.
    Definition: vk_mem_alloc.h:1374
    +
    VmaAllocationCreateInfo::pUserData
    void * pUserData
    Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
    Definition: vk_mem_alloc.h:1754
    +
    VmaPoolCreateInfo::minBlockCount
    size_t minBlockCount
    Minimum number of blocks to be always allocated in this pool, even if they stay empty.
    Definition: vk_mem_alloc.h:1858
    +
    VmaPoolStats::allocationCount
    size_t allocationCount
    Number of VmaAllocation objects created from this pool that were not destroyed or lost...
    Definition: vk_mem_alloc.h:1894
    VmaVulkanFunctions
    struct VmaVulkanFunctions VmaVulkanFunctions
    Pointers to some Vulkan functions - a subset used by the library.
    -
    VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1289
    -
    VmaPoolCreateInfo::memoryTypeIndex
    uint32_t memoryTypeIndex
    Vulkan memory type index to allocate this pool from.
    Definition: vk_mem_alloc.h:1798
    +
    VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1336
    +
    VmaPoolCreateInfo::memoryTypeIndex
    uint32_t memoryTypeIndex
    Vulkan memory type index to allocate this pool from.
    Definition: vk_mem_alloc.h:1845
    vmaFindMemoryTypeIndex
    VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
    Helps to find memoryTypeIndex, given memoryTypeBits and VmaAllocationCreateInfo.
    -
    VmaMemoryUsage
    VmaMemoryUsage
    Definition: vk_mem_alloc.h:1546
    +
    VmaMemoryUsage
    VmaMemoryUsage
    Definition: vk_mem_alloc.h:1593
    VmaAllocationInfo
    struct VmaAllocationInfo VmaAllocationInfo
    Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
    vmaFlushAllocation
    void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    Flushes memory of given allocation.
    -
    VmaDefragmentationInfo
    Optional configuration parameters to be passed to function vmaDefragment().
    Definition: vk_mem_alloc.h:2192
    +
    VmaDefragmentationInfo
    Optional configuration parameters to be passed to function vmaDefragment().
    Definition: vk_mem_alloc.h:2239
    VmaPoolCreateInfo
    struct VmaPoolCreateInfo VmaPoolCreateInfo
    Describes parameter of created VmaPool.
    vmaDestroyPool
    void vmaDestroyPool(VmaAllocator allocator, VmaPool pool)
    Destroys VmaPool object and frees Vulkan device memory.
    -
    VmaDefragmentationStats::bytesFreed
    VkDeviceSize bytesFreed
    Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
    Definition: vk_mem_alloc.h:2210
    -
    VMA_MEMORY_USAGE_CPU_TO_GPU
    Definition: vk_mem_alloc.h:1585
    -
    VmaAllocationCreateInfo::memoryTypeBits
    uint32_t memoryTypeBits
    Bitmask containing one bit set for every memory type acceptable for this allocation.
    Definition: vk_mem_alloc.h:1694
    -
    VmaVulkanFunctions::vkBindBufferMemory
    PFN_vkBindBufferMemory vkBindBufferMemory
    Definition: vk_mem_alloc.h:1306
    +
    VmaDefragmentationStats::bytesFreed
    VkDeviceSize bytesFreed
    Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
    Definition: vk_mem_alloc.h:2257
    +
    VMA_MEMORY_USAGE_CPU_TO_GPU
    Definition: vk_mem_alloc.h:1632
    +
    VmaAllocationCreateInfo::memoryTypeBits
    uint32_t memoryTypeBits
    Bitmask containing one bit set for every memory type acceptable for this allocation.
    Definition: vk_mem_alloc.h:1741
    +
    VmaVulkanFunctions::vkBindBufferMemory
    PFN_vkBindBufferMemory vkBindBufferMemory
    Definition: vk_mem_alloc.h:1353
    VmaPool
    Represents custom memory pool.
    vmaGetPoolStats
    void vmaGetPoolStats(VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
    Retrieves statistics of existing VmaPool object.
    VmaDefragmentationInfo
    struct VmaDefragmentationInfo VmaDefragmentationInfo
    Optional configuration parameters to be passed to function vmaDefragment().
    -
    VmaStats
    General statistics from current state of Allocator.
    Definition: vk_mem_alloc.h:1506
    -
    PFN_vmaFreeDeviceMemoryFunction
    void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
    Callback function called before vkFreeMemory.
    Definition: vk_mem_alloc.h:1239
    +
    VmaStats
    General statistics from current state of Allocator.
    Definition: vk_mem_alloc.h:1553
    +
    PFN_vmaFreeDeviceMemoryFunction
    void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
    Callback function called before vkFreeMemory.
    Definition: vk_mem_alloc.h:1286
    vmaSetAllocationUserData
    void vmaSetAllocationUserData(VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
    Sets pUserData in given allocation to new value.
    vmaCreatePool
    VkResult vmaCreatePool(VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
    Allocates Vulkan device memory and creates VmaPool object.
    -
    VmaAllocatorCreateFlagBits
    VmaAllocatorCreateFlagBits
    Flags for created VmaAllocator.
    Definition: vk_mem_alloc.h:1260
    +
    VmaAllocatorCreateFlagBits
    VmaAllocatorCreateFlagBits
    Flags for created VmaAllocator.
    Definition: vk_mem_alloc.h:1307
    vmaBindImageMemory
    VkResult vmaBindImageMemory(VmaAllocator allocator, VmaAllocation allocation, VkImage image)
    Binds image to allocation.
    VmaStatInfo
    struct VmaStatInfo VmaStatInfo
    Calculated statistics of memory usage in entire allocator.
    -
    VmaRecordFlags
    VkFlags VmaRecordFlags
    Definition: vk_mem_alloc.h:1331
    -
    VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
    Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
    Definition: vk_mem_alloc.h:1265
    -
    VmaDefragmentationStats::allocationsMoved
    uint32_t allocationsMoved
    Number of allocations that have been moved to different places.
    Definition: vk_mem_alloc.h:2212
    +
    VmaRecordFlags
    VkFlags VmaRecordFlags
    Definition: vk_mem_alloc.h:1378
    +
    VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
    Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
    Definition: vk_mem_alloc.h:1312
    +
    VmaDefragmentationStats::allocationsMoved
    uint32_t allocationsMoved
    Number of allocations that have been moved to different places.
    Definition: vk_mem_alloc.h:2259
    vmaCreateLostAllocation
    void vmaCreateLostAllocation(VmaAllocator allocator, VmaAllocation *pAllocation)
    Creates new allocation that is in lost state from the beginning.
    -
    VmaAllocationCreateInfo::requiredFlags
    VkMemoryPropertyFlags requiredFlags
    Flags that must be set in a Memory Type chosen for an allocation.
    Definition: vk_mem_alloc.h:1681
    -
    VmaPoolStats::unusedRangeSizeMax
    VkDeviceSize unusedRangeSizeMax
    Size of the largest continuous free memory region.
    Definition: vk_mem_alloc.h:1857
    +
    VmaAllocationCreateInfo::requiredFlags
    VkMemoryPropertyFlags requiredFlags
    Flags that must be set in a Memory Type chosen for an allocation.
    Definition: vk_mem_alloc.h:1728
    +
    VmaPoolStats::unusedRangeSizeMax
    VkDeviceSize unusedRangeSizeMax
    Size of the largest continuous free memory region.
    Definition: vk_mem_alloc.h:1904
    vmaBuildStatsString
    void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
    Builds and returns statistics as string in JSON format.
    -
    VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
    PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
    Definition: vk_mem_alloc.h:1299
    -
    VmaStatInfo
    Calculated statistics of memory usage in entire allocator.
    Definition: vk_mem_alloc.h:1489
    -
    VmaPoolCreateInfo::blockSize
    VkDeviceSize blockSize
    Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
    Definition: vk_mem_alloc.h:1806
    -
    VmaDeviceMemoryCallbacks
    Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
    Definition: vk_mem_alloc.h:1252
    +
    VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
    PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
    Definition: vk_mem_alloc.h:1346
    +
    VmaStatInfo
    Calculated statistics of memory usage in entire allocator.
    Definition: vk_mem_alloc.h:1536
    +
    VmaPoolCreateInfo::blockSize
    VkDeviceSize blockSize
    Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
    Definition: vk_mem_alloc.h:1853
    +
    VmaDeviceMemoryCallbacks
    Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
    Definition: vk_mem_alloc.h:1299
    vmaCreateBuffer
    VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    -
    VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
    Definition: vk_mem_alloc.h:1655
    -
    VmaStatInfo::unusedRangeSizeMin
    VkDeviceSize unusedRangeSizeMin
    Definition: vk_mem_alloc.h:1502
    -
    VmaDeviceMemoryCallbacks::pfnFree
    PFN_vmaFreeDeviceMemoryFunction pfnFree
    Optional, can be null.
    Definition: vk_mem_alloc.h:1256
    -
    VmaPoolCreateInfo::flags
    VmaPoolCreateFlags flags
    Use combination of VmaPoolCreateFlagBits.
    Definition: vk_mem_alloc.h:1801
    -
    VMA_MEMORY_USAGE_GPU_TO_CPU
    Definition: vk_mem_alloc.h:1594
    -
    VmaVulkanFunctions::vkInvalidateMappedMemoryRanges
    PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges
    Definition: vk_mem_alloc.h:1305
    +
    VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
    Definition: vk_mem_alloc.h:1702
    +
    VmaStatInfo::unusedRangeSizeMin
    VkDeviceSize unusedRangeSizeMin
    Definition: vk_mem_alloc.h:1549
    +
    VmaDeviceMemoryCallbacks::pfnFree
    PFN_vmaFreeDeviceMemoryFunction pfnFree
    Optional, can be null.
    Definition: vk_mem_alloc.h:1303
    +
    VmaPoolCreateInfo::flags
    VmaPoolCreateFlags flags
    Use combination of VmaPoolCreateFlagBits.
    Definition: vk_mem_alloc.h:1848
    +
    VMA_MEMORY_USAGE_GPU_TO_CPU
    Definition: vk_mem_alloc.h:1641
    +
    VmaVulkanFunctions::vkInvalidateMappedMemoryRanges
    PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges
    Definition: vk_mem_alloc.h:1352
    VmaPoolStats
    struct VmaPoolStats VmaPoolStats
    Describes parameter of existing VmaPool.
    vmaCreateImage
    VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    Function similar to vmaCreateBuffer().
    -
    VmaAllocationCreateInfo::usage
    VmaMemoryUsage usage
    Intended usage of memory.
    Definition: vk_mem_alloc.h:1676
    -
    VmaAllocationCreateInfo
    Definition: vk_mem_alloc.h:1667
    +
    VmaAllocationCreateInfo::usage
    VmaMemoryUsage usage
    Intended usage of memory.
    Definition: vk_mem_alloc.h:1723
    +
    VmaAllocationCreateInfo
    Definition: vk_mem_alloc.h:1714
    vmaFindMemoryTypeIndexForImageInfo
    VkResult vmaFindMemoryTypeIndexForImageInfo(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
    Helps to find memoryTypeIndex, given VkImageCreateInfo and VmaAllocationCreateInfo.
    -
    VmaStatInfo::blockCount
    uint32_t blockCount
    Number of VkDeviceMemory Vulkan memory blocks allocated.
    Definition: vk_mem_alloc.h:1492
    -
    VmaVulkanFunctions::vkFreeMemory
    PFN_vkFreeMemory vkFreeMemory
    Definition: vk_mem_alloc.h:1301
    -
    VmaPoolCreateInfo::maxBlockCount
    size_t maxBlockCount
    Maximum number of blocks that can be allocated in this pool. Optional.
    Definition: vk_mem_alloc.h:1819
    -
    VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
    const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
    Informative callbacks for vkAllocateMemory, vkFreeMemory. Optional.
    Definition: vk_mem_alloc.h:1379
    -
    VmaPoolStats::unusedRangeCount
    size_t unusedRangeCount
    Number of continuous memory ranges in the pool not used by any VmaAllocation.
    Definition: vk_mem_alloc.h:1850
    -
    VmaAllocationCreateFlags
    VkFlags VmaAllocationCreateFlags
    Definition: vk_mem_alloc.h:1665
    -
    VmaAllocationCreateInfo::pool
    VmaPool pool
    Pool that this allocation should be created in.
    Definition: vk_mem_alloc.h:1700
    +
    VmaStatInfo::blockCount
    uint32_t blockCount
    Number of VkDeviceMemory Vulkan memory blocks allocated.
    Definition: vk_mem_alloc.h:1539
    +
    VmaVulkanFunctions::vkFreeMemory
    PFN_vkFreeMemory vkFreeMemory
    Definition: vk_mem_alloc.h:1348
    +
    VmaPoolCreateInfo::maxBlockCount
    size_t maxBlockCount
    Maximum number of blocks that can be allocated in this pool. Optional.
    Definition: vk_mem_alloc.h:1866
    +
    VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
    const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
    Informative callbacks for vkAllocateMemory, vkFreeMemory. Optional.
    Definition: vk_mem_alloc.h:1426
    +
    VmaPoolStats::unusedRangeCount
    size_t unusedRangeCount
    Number of continuous memory ranges in the pool not used by any VmaAllocation.
    Definition: vk_mem_alloc.h:1897
    +
    VmaAllocationCreateFlags
    VkFlags VmaAllocationCreateFlags
    Definition: vk_mem_alloc.h:1712
    +
    VmaAllocationCreateInfo::pool
    VmaPool pool
    Pool that this allocation should be created in.
    Definition: vk_mem_alloc.h:1747
    vmaGetMemoryProperties
    void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
    -
    VmaAllocatorCreateInfo::pHeapSizeLimit
    const VkDeviceSize * pHeapSizeLimit
    Either null or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
    Definition: vk_mem_alloc.h:1417
    -
    VmaStats::memoryType
    VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
    Definition: vk_mem_alloc.h:1508
    -
    VMA_ALLOCATION_CREATE_MAPPED_BIT
    Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
    Definition: vk_mem_alloc.h:1635
    -
    VmaStatInfo::allocationSizeMin
    VkDeviceSize allocationSizeMin
    Definition: vk_mem_alloc.h:1501
    +
    VmaAllocatorCreateInfo::pHeapSizeLimit
    const VkDeviceSize * pHeapSizeLimit
    Either null or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
    Definition: vk_mem_alloc.h:1464
    +
    VmaStats::memoryType
    VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
    Definition: vk_mem_alloc.h:1555
    +
    VMA_ALLOCATION_CREATE_MAPPED_BIT
    Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
    Definition: vk_mem_alloc.h:1682
    +
    VmaStatInfo::allocationSizeMin
    VkDeviceSize allocationSizeMin
    Definition: vk_mem_alloc.h:1548
    vmaFindMemoryTypeIndexForBufferInfo
    VkResult vmaFindMemoryTypeIndexForBufferInfo(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
    Helps to find memoryTypeIndex, given VkBufferCreateInfo and VmaAllocationCreateInfo.
    -
    VmaVulkanFunctions::vkCreateImage
    PFN_vkCreateImage vkCreateImage
    Definition: vk_mem_alloc.h:1312
    -
    VmaRecordSettings::flags
    VmaRecordFlags flags
    Flags for recording. Use VmaRecordFlagBits enum.
    Definition: vk_mem_alloc.h:1349
    -
    VmaDeviceMemoryCallbacks::pfnAllocate
    PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
    Optional, can be null.
    Definition: vk_mem_alloc.h:1254
    -
    VmaVulkanFunctions::vkDestroyBuffer
    PFN_vkDestroyBuffer vkDestroyBuffer
    Definition: vk_mem_alloc.h:1311
    +
    VmaVulkanFunctions::vkCreateImage
    PFN_vkCreateImage vkCreateImage
    Definition: vk_mem_alloc.h:1359
    +
    VmaRecordSettings::flags
    VmaRecordFlags flags
    Flags for recording. Use VmaRecordFlagBits enum.
    Definition: vk_mem_alloc.h:1396
    +
    VmaDeviceMemoryCallbacks::pfnAllocate
    PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
    Optional, can be null.
    Definition: vk_mem_alloc.h:1301
    +
    VmaVulkanFunctions::vkDestroyBuffer
    PFN_vkDestroyBuffer vkDestroyBuffer
    Definition: vk_mem_alloc.h:1358
    vmaMapMemory
    VkResult vmaMapMemory(VmaAllocator allocator, VmaAllocation allocation, void **ppData)
    Maps memory represented by given allocation and returns pointer to it.
    -
    VmaPoolCreateInfo::frameInUseCount
    uint32_t frameInUseCount
    Maximum number of additional frames that are in use at the same time as current frame.
    Definition: vk_mem_alloc.h:1833
    -
    VmaVulkanFunctions::vkFlushMappedMemoryRanges
    PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges
    Definition: vk_mem_alloc.h:1304
    +
    VmaPoolCreateInfo::frameInUseCount
    uint32_t frameInUseCount
    Maximum number of additional frames that are in use at the same time as current frame.
    Definition: vk_mem_alloc.h:1880
    +
    VmaVulkanFunctions::vkFlushMappedMemoryRanges
    PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges
    Definition: vk_mem_alloc.h:1351
    vmaAllocateMemoryForImage
    VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    Function similar to vmaAllocateMemoryForBuffer().
    VmaAllocatorCreateInfo
    struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
    Description of a Allocator to be created.
    -
    VmaRecordSettings::pFilePath
    const char * pFilePath
    Path to the file that should be written by the recording.
    Definition: vk_mem_alloc.h:1357
    -
    VmaAllocationInfo::pUserData
    void * pUserData
    Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
    Definition: vk_mem_alloc.h:1981
    -
    VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
    VkDeviceSize preferredLargeHeapBlockSize
    Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
    Definition: vk_mem_alloc.h:1373
    -
    VmaStatInfo::allocationSizeAvg
    VkDeviceSize allocationSizeAvg
    Definition: vk_mem_alloc.h:1501
    -
    VmaStatInfo::usedBytes
    VkDeviceSize usedBytes
    Total number of bytes occupied by all allocations.
    Definition: vk_mem_alloc.h:1498
    +
    VmaRecordSettings::pFilePath
    const char * pFilePath
    Path to the file that should be written by the recording.
    Definition: vk_mem_alloc.h:1404
    +
    VmaAllocationInfo::pUserData
    void * pUserData
    Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
    Definition: vk_mem_alloc.h:2028
    +
    VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
    VkDeviceSize preferredLargeHeapBlockSize
    Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
    Definition: vk_mem_alloc.h:1420
    +
    VmaStatInfo::allocationSizeAvg
    VkDeviceSize allocationSizeAvg
    Definition: vk_mem_alloc.h:1548
    +
    VmaStatInfo::usedBytes
    VkDeviceSize usedBytes
    Total number of bytes occupied by all allocations.
    Definition: vk_mem_alloc.h:1545
    VmaDeviceMemoryCallbacks
    struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
    Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
    vmaCheckCorruption
    VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
    Checks magic number in margins around all allocations in given memory types (in both default and cust...
    -
    VmaPoolStats
    Describes parameter of existing VmaPool.
    Definition: vk_mem_alloc.h:1838
    +
    VmaPoolStats
    Describes parameter of existing VmaPool.
    Definition: vk_mem_alloc.h:1885
    vmaCheckPoolCorruption
    VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
    Checks magic number in margins around all allocations in given memory pool in search for corruptions...
    -
    VmaAllocationInfo::offset
    VkDeviceSize offset
    Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
    Definition: vk_mem_alloc.h:1962
    -
    VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1663
    -
    VmaDefragmentationStats::bytesMoved
    VkDeviceSize bytesMoved
    Total number of bytes that have been copied while moving allocations to different places...
    Definition: vk_mem_alloc.h:2208
    -
    VmaVulkanFunctions
    Pointers to some Vulkan functions - a subset used by the library.
    Definition: vk_mem_alloc.h:1297
    +
    VmaAllocationInfo::offset
    VkDeviceSize offset
    Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
    Definition: vk_mem_alloc.h:2009
    +
    VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
    Definition: vk_mem_alloc.h:1710
    +
    VmaDefragmentationStats::bytesMoved
    VkDeviceSize bytesMoved
    Total number of bytes that have been copied while moving allocations to different places...
    Definition: vk_mem_alloc.h:2255
    +
    VmaVulkanFunctions
    Pointers to some Vulkan functions - a subset used by the library.
    Definition: vk_mem_alloc.h:1344
    vmaCreateAllocator
    VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
    Creates Allocator object.
    -
    VmaStatInfo::unusedRangeCount
    uint32_t unusedRangeCount
    Number of free ranges of memory between allocations.
    Definition: vk_mem_alloc.h:1496
    -
    VMA_MEMORY_USAGE_UNKNOWN
    Definition: vk_mem_alloc.h:1551
    -
    VmaPoolCreateFlags
    VkFlags VmaPoolCreateFlags
    Definition: vk_mem_alloc.h:1791
    +
    VmaStatInfo::unusedRangeCount
    uint32_t unusedRangeCount
    Number of free ranges of memory between allocations.
    Definition: vk_mem_alloc.h:1543
    +
    VMA_MEMORY_USAGE_UNKNOWN
    Definition: vk_mem_alloc.h:1598
    +
    VmaPoolCreateFlags
    VkFlags VmaPoolCreateFlags
    Definition: vk_mem_alloc.h:1838
    vmaGetPhysicalDeviceProperties
    void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
    -
    VmaRecordSettings
    Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSetting...
    Definition: vk_mem_alloc.h:1346
    -
    VmaStatInfo::allocationCount
    uint32_t allocationCount
    Number of VmaAllocation allocation objects allocated.
    Definition: vk_mem_alloc.h:1494
    -
    VmaVulkanFunctions::vkGetImageMemoryRequirements
    PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
    Definition: vk_mem_alloc.h:1309
    -
    VmaVulkanFunctions::vkDestroyImage
    PFN_vkDestroyImage vkDestroyImage
    Definition: vk_mem_alloc.h:1313
    -
    VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
    Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
    Definition: vk_mem_alloc.h:1622
    -
    VMA_MEMORY_USAGE_CPU_ONLY
    Definition: vk_mem_alloc.h:1578
    -
    VmaAllocationInfo::pMappedData
    void * pMappedData
    Pointer to the beginning of this allocation as mapped data.
    Definition: vk_mem_alloc.h:1976
    +
    VmaRecordSettings
    Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSetting...
    Definition: vk_mem_alloc.h:1393
    +
    VmaStatInfo::allocationCount
    uint32_t allocationCount
    Number of VmaAllocation allocation objects allocated.
    Definition: vk_mem_alloc.h:1541
    +
    VmaVulkanFunctions::vkGetImageMemoryRequirements
    PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
    Definition: vk_mem_alloc.h:1356
    +
    VmaVulkanFunctions::vkDestroyImage
    PFN_vkDestroyImage vkDestroyImage
    Definition: vk_mem_alloc.h:1360
    +
    VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
    Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
    Definition: vk_mem_alloc.h:1669
    +
    VMA_MEMORY_USAGE_CPU_ONLY
    Definition: vk_mem_alloc.h:1625
    +
    VmaAllocationInfo::pMappedData
    void * pMappedData
    Pointer to the beginning of this allocation as mapped data.
    Definition: vk_mem_alloc.h:2023
    vmaDestroyImage
    void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
    Destroys Vulkan image and frees allocated memory.
    -
    VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
    Enables usage of VK_KHR_dedicated_allocation extension.
    Definition: vk_mem_alloc.h:1287
    +
    VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
    Enables usage of VK_KHR_dedicated_allocation extension.
    Definition: vk_mem_alloc.h:1334
    VmaDefragmentationStats
    struct VmaDefragmentationStats VmaDefragmentationStats
    Statistics returned by function vmaDefragment().
    -
    VmaVulkanFunctions::vkAllocateMemory
    PFN_vkAllocateMemory vkAllocateMemory
    Definition: vk_mem_alloc.h:1300
    -
    VmaAllocationInfo
    Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
    Definition: vk_mem_alloc.h:1943
    +
    VmaVulkanFunctions::vkAllocateMemory
    PFN_vkAllocateMemory vkAllocateMemory
    Definition: vk_mem_alloc.h:1347
    +
    VmaAllocationInfo
    Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
    Definition: vk_mem_alloc.h:1990
    vmaAllocateMemory
    VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    General purpose memory allocation.
    vmaSetCurrentFrameIndex
    void vmaSetCurrentFrameIndex(VmaAllocator allocator, uint32_t frameIndex)
    Sets index of the current frame.
    VmaAllocationCreateInfo
    struct VmaAllocationCreateInfo VmaAllocationCreateInfo
    vmaAllocateMemoryForBuffer
    VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
    -
    VmaPoolCreateFlagBits
    VmaPoolCreateFlagBits
    Flags to be passed as VmaPoolCreateInfo::flags.
    Definition: vk_mem_alloc.h:1769
    -
    VmaStatInfo::unusedRangeSizeAvg
    VkDeviceSize unusedRangeSizeAvg
    Definition: vk_mem_alloc.h:1502
    +
    VmaPoolCreateFlagBits
    VmaPoolCreateFlagBits
    Flags to be passed as VmaPoolCreateInfo::flags.
    Definition: vk_mem_alloc.h:1816
    +
    VmaStatInfo::unusedRangeSizeAvg
    VkDeviceSize unusedRangeSizeAvg
    Definition: vk_mem_alloc.h:1549
    vmaTouchAllocation
    VkBool32 vmaTouchAllocation(VmaAllocator allocator, VmaAllocation allocation)
    Returns VK_TRUE if allocation is not lost and atomically marks it as used in current frame...
    -
    VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
    Definition: vk_mem_alloc.h:1661
    -
    VmaRecordFlagBits
    VmaRecordFlagBits
    Flags to be used in VmaRecordSettings::flags.
    Definition: vk_mem_alloc.h:1321
    -
    VmaStats::memoryHeap
    VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
    Definition: vk_mem_alloc.h:1509
    +
    VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
    Definition: vk_mem_alloc.h:1708
    +
    VmaRecordFlagBits
    VmaRecordFlagBits
    Flags to be used in VmaRecordSettings::flags.
    Definition: vk_mem_alloc.h:1368
    +
    VmaStats::memoryHeap
    VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
    Definition: vk_mem_alloc.h:1556
    vmaDestroyBuffer
    void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
    Destroys Vulkan buffer and frees allocated memory.
    -
    VmaPoolStats::unusedSize
    VkDeviceSize unusedSize
    Total number of bytes in the pool not used by any VmaAllocation.
    Definition: vk_mem_alloc.h:1844
    -
    VmaStatInfo::unusedRangeSizeMax
    VkDeviceSize unusedRangeSizeMax
    Definition: vk_mem_alloc.h:1502
    +
    VmaPoolStats::unusedSize
    VkDeviceSize unusedSize
    Total number of bytes in the pool not used by any VmaAllocation.
    Definition: vk_mem_alloc.h:1891
    +
    VmaStatInfo::unusedRangeSizeMax
    VkDeviceSize unusedRangeSizeMax
    Definition: vk_mem_alloc.h:1549
    VmaRecordSettings
    struct VmaRecordSettings VmaRecordSettings
    Parameters for recording calls to VMA functions. To be used in VmaAllocatorCreateInfo::pRecordSetting...
    -
    VmaAllocationInfo::memoryType
    uint32_t memoryType
    Memory type index that this allocation was allocated from.
    Definition: vk_mem_alloc.h:1948
    +
    VmaAllocationInfo::memoryType
    uint32_t memoryType
    Memory type index that this allocation was allocated from.
    Definition: vk_mem_alloc.h:1995
    diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 8fda710..95e6260 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -67,6 +67,7 @@ Documentation of all members: vk_mem_alloc.h - [Memory initialization](@ref debugging_memory_usage_initialization) - [Margins](@ref debugging_memory_usage_margins) - [Corruption detection](@ref debugging_memory_usage_corruption_detection) + - \subpage record_and_replay - \subpage usage_patterns - [Simple patterns](@ref usage_patterns_simple) - [Advanced patterns](@ref usage_patterns_advanced) @@ -920,6 +921,52 @@ Margin validation (corruption detection) works only for memory types that are `HOST_VISIBLE` and `HOST_COHERENT`. +\page record_and_replay Record and replay + +\section record_and_replay_introduction Introduction + +While using the library, sequence of calls to its functions together with their +parameters can be recorded to a file and later replayed using standalone player +application. It can be useful to: + +- Test correctness - check if same sequence of calls will not cause crash or + failures on a target platform. +- Gather statistics - see number of allocations, peak memory usage, number of + calls etc. +- Benchmark performance - see how much time it takes to replay the whole + sequence. + +\section record_and_replay_usage Usage + +To record sequence of calls to a file: Fill in +VmaAllocatorCreateInfo::pRecordSettings member while creating #VmaAllocator +object. File is opened and written during whole lifetime of the allocator. + +To replay file: Use VmaReplay - standalone command-line program. +Precompiled binary can be found in "bin" directory. +Its source can be found in "src/VmaReplay" directory. +Its project is generated by Premake. +Command line syntax is printed when the program is launched without parameters. +Basic usage: + + VmaReplay.exe MyRecording.csv + +Documentation of file format can be found in file: "docs/Recording file format.md". +It's a human-readable, text file in CSV format (Comma Separated Values). + +\section record_and_replay_additional_considerations Additional considerations + +- Replaying file that was recorded on a different GPU (with different parameters + like `bufferImageGranularity`, `nonCoherentAtomSize`, and especially different + set of memory heaps and types) may give different performance and memory usage + results, as well as issue some warnings and errors. +- Current implementation of recording in VMA, as well as VmaReplay application, is + coded and tested only on Windows. Inclusion of recording code is driven by + `VMA_RECORDING_ENABLED` macro. Support for other platforms should be easy to + add. Contributions are welcomed. +- Currently calls to vmaDefragment() function are not recorded. + + \page usage_patterns Recommended usage patterns See also slides from talk: @@ -1198,7 +1245,7 @@ The library uses following algorithm for allocation, in order: Features deliberately excluded from the scope of this library: - Data transfer - issuing commands that transfer data between buffers or images, any usage of - `VkCommandList` or `VkCommandQueue` and related synchronization is responsibility of the user. + `VkCommandList` or `VkQueue` and related synchronization is responsibility of the user. - Allocations for imported/exported external memory. They tend to require explicit memory type index and dedicated allocation anyway, so they don't interact with main features of this library. Such special purpose allocations From 5de71e1d283ac15b40a5694939e36a240a094bc1 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 11:54:49 +0200 Subject: [PATCH 36/51] VmaReplay: Added statistics for number of calls to VMA functions. --- src/VmaReplay/VmaReplay.cpp | 111 +++++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 3 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 7ab8df3..114169b 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -45,6 +45,52 @@ static enum class VERBOSITY enum class OBJECT_TYPE { BUFFER, IMAGE }; +enum class VMA_FUNCTION +{ + CreatePool, + DestroyPool, + SetAllocationUserData, + CreateBuffer, + DestroyBuffer, + CreateImage, + DestroyImage, + FreeMemory, + CreateLostAllocation, + AllocateMemory, + AllocateMemoryForBuffer, + AllocateMemoryForImage, + MapMemory, + UnmapMemory, + FlushAllocation, + InvalidateAllocation, + TouchAllocation, + GetAllocationInfo, + Count +}; +static const char* VMA_FUNCTION_NAMES[] = { + "vmaCreatePool", + "vmaDestroyPool", + "vmaSetAllocationUserData", + "vmaCreateBuffer", + "vmaDestroyBuffer", + "vmaCreateImage", + "vmaDestroyImage", + "vmaFreeMemory", + "vmaCreateLostAllocation", + "vmaAllocateMemory", + "vmaAllocateMemoryForBuffer", + "vmaAllocateMemoryForImage", + "vmaMapMemory", + "vmaUnmapMemory", + "vmaFlushAllocation", + "vmaInvalidateAllocation", + "vmaTouchAllocation", + "vmaGetAllocationInfo", +}; +static_assert( + _countof(VMA_FUNCTION_NAMES) == (size_t)VMA_FUNCTION::Count, + "VMA_FUNCTION_NAMES array doesn't match VMA_FUNCTION enum."); + static std::string g_FilePath; // Most significant 16 bits are major version, least significant 16 bits are minor version. static uint32_t g_FileVersion; @@ -242,18 +288,21 @@ public: Statistics() { } + const size_t* GetFunctionCallCount() const { return m_FunctionCallCount; } size_t GetImageCreationCount(uint32_t imgClass) const { return m_ImageCreationCount[imgClass]; } size_t GetLinearImageCreationCount() const { return m_LinearImageCreationCount; } size_t GetBufferCreationCount(uint32_t bufClass) const { return m_BufferCreationCount[bufClass]; } size_t GetAllocationCreationCount() const { return m_AllocationCreationCount; } size_t GetPoolCreationCount() const { return m_PoolCreationCount; } + void RegisterFunctionCall(VMA_FUNCTION func); void RegisterCreateImage(uint32_t usage, uint32_t tiling); void RegisterCreateBuffer(uint32_t usage); void RegisterCreatePool(); void RegisterCreateAllocation(); private: + size_t m_FunctionCallCount[(size_t)VMA_FUNCTION::Count] = {}; size_t m_ImageCreationCount[4] = { }; size_t m_LinearImageCreationCount = 0; size_t m_BufferCreationCount[4] = { }; @@ -324,6 +373,11 @@ uint32_t Statistics::ImageUsageToClass(uint32_t usage) } } +void Statistics::RegisterFunctionCall(VMA_FUNCTION func) +{ + ++m_FunctionCallCount[(size_t)func]; +} + void Statistics::RegisterCreateImage(uint32_t usage, uint32_t tiling) { if(tiling == VK_IMAGE_TILING_LINEAR) @@ -506,10 +560,10 @@ private: void ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteSetAllocationUserData(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit); - void ExecuteDestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } + void ExecuteDestroyBuffer(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::DestroyBuffer); DestroyAllocation(lineNumber, csvSplit); } void ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit); - void ExecuteDestroyImage(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } - void ExecuteFreeMemory(size_t lineNumber, const CsvSplit& csvSplit) { DestroyAllocation(lineNumber, csvSplit); } + void ExecuteDestroyImage(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::DestroyImage); DestroyAllocation(lineNumber, csvSplit); } + void ExecuteFreeMemory(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::FreeMemory); DestroyAllocation(lineNumber, csvSplit); } void ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvSplit& csvSplit, OBJECT_TYPE objType); @@ -1120,6 +1174,20 @@ void Player::PrintStats() { printf(" VMA used from only one thread.\n"); } + + // Function call count + if(g_Verbosity == VERBOSITY::MAXIMUM) + { + printf(" Function call count:\n"); + const size_t* const functionCallCount = m_Stats.GetFunctionCallCount(); + for(size_t i = 0; i < (size_t)VMA_FUNCTION::Count; ++i) + { + if(functionCallCount[i] > 0) + { + printf(" %s %zu\n", VMA_FUNCTION_NAMES[i], functionCallCount[i]); + } + } + } } bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound) @@ -1174,6 +1242,8 @@ bool Player::PrepareUserData(size_t lineNumber, uint32_t allocCreateFlags, const void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::CreatePool); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 7, false)) { VmaPoolCreateInfo poolCreateInfo = {}; @@ -1255,6 +1325,8 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::DestroyPool); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { uint64_t origPtr = 0; @@ -1290,6 +1362,8 @@ void Player::ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteSetAllocationUserData(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::SetAllocationUserData); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 2, true)) { uint64_t origPtr = 0; @@ -1331,6 +1405,8 @@ void Player::ExecuteSetAllocationUserData(size_t lineNumber, const CsvSplit& csv void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::CreateBuffer); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 12, true)) { VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; @@ -1416,6 +1492,8 @@ void Player::DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::CreateImage); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 21, true)) { VkImageCreateInfo imageCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; @@ -1475,6 +1553,8 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::CreateLostAllocation); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { uint64_t origPtr = 0; @@ -1499,6 +1579,8 @@ void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvS void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::AllocateMemory); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 11, true)) { VkMemoryRequirements memReq = {}; @@ -1548,6 +1630,17 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvSplit& csvSplit, OBJECT_TYPE objType) { + switch(objType) + { + case OBJECT_TYPE::BUFFER: + m_Stats.RegisterFunctionCall(VMA_FUNCTION::AllocateMemoryForBuffer); + break; + case OBJECT_TYPE::IMAGE: + m_Stats.RegisterFunctionCall(VMA_FUNCTION::AllocateMemoryForImage); + break; + default: assert(0); + } + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 13, true)) { VkMemoryRequirements memReq = {}; @@ -1615,6 +1708,8 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS void Player::ExecuteMapMemory(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::MapMemory); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { uint64_t origPtr = 0; @@ -1664,6 +1759,8 @@ void Player::ExecuteMapMemory(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteUnmapMemory(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::UnmapMemory); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { uint64_t origPtr = 0; @@ -1708,6 +1805,8 @@ void Player::ExecuteUnmapMemory(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteFlushAllocation(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::FlushAllocation); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 3, false)) { uint64_t origPtr = 0; @@ -1756,6 +1855,8 @@ void Player::ExecuteFlushAllocation(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteInvalidateAllocation(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::InvalidateAllocation); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 3, false)) { uint64_t origPtr = 0; @@ -1804,6 +1905,8 @@ void Player::ExecuteInvalidateAllocation(size_t lineNumber, const CsvSplit& csvS void Player::ExecuteTouchAllocation(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::TouchAllocation); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { uint64_t origPtr = 0; @@ -1844,6 +1947,8 @@ void Player::ExecuteTouchAllocation(size_t lineNumber, const CsvSplit& csvSplit) void Player::ExecuteGetAllocationInfo(size_t lineNumber, const CsvSplit& csvSplit) { + m_Stats.RegisterFunctionCall(VMA_FUNCTION::GetAllocationInfo); + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) { uint64_t origPtr = 0; From 2556b377f70a8265b39ac1d94855978126966206 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 12:04:19 +0200 Subject: [PATCH 37/51] Record and replay: Added support for function vmaMakePoolAllocationsLost. --- docs/Recording file format.md | 4 ++++ src/VmaReplay/VmaReplay.cpp | 41 +++++++++++++++++++++++++++++++++++ src/vk_mem_alloc.h | 21 ++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/docs/Recording file format.md b/docs/Recording file format.md index 02e414b..d6c2878 100644 --- a/docs/Recording file format.md +++ b/docs/Recording file format.md @@ -158,6 +158,10 @@ No parameters. - allocation : pointer +**vmaMakePoolAllocationsLost** (min format version: 1.2) + +- pool : pointer + ## Data types **bool** diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 114169b..58cd279 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -65,6 +65,7 @@ enum class VMA_FUNCTION InvalidateAllocation, TouchAllocation, GetAllocationInfo, + MakePoolAllocationsLost, Count }; static const char* VMA_FUNCTION_NAMES[] = { @@ -86,6 +87,7 @@ static const char* VMA_FUNCTION_NAMES[] = { "vmaInvalidateAllocation", "vmaTouchAllocation", "vmaGetAllocationInfo", + "vmaMakePoolAllocationsLost", }; static_assert( _countof(VMA_FUNCTION_NAMES) == (size_t)VMA_FUNCTION::Count, @@ -573,6 +575,7 @@ private: void ExecuteInvalidateAllocation(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteTouchAllocation(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteGetAllocationInfo(size_t lineNumber, const CsvSplit& csvSplit); + void ExecuteMakePoolAllocationsLost(size_t lineNumber, const CsvSplit& csvSplit); void DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit); }; @@ -704,6 +707,8 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) ExecuteTouchAllocation(lineNumber, csvSplit); else if(StrRangeEq(functionName, "vmaGetAllocationInfo")) ExecuteGetAllocationInfo(lineNumber, csvSplit); + else if(StrRangeEq(functionName, "vmaMakePoolAllocationsLost")) + ExecuteMakePoolAllocationsLost(lineNumber, csvSplit); else { if(IssueWarning()) @@ -1988,6 +1993,42 @@ void Player::ExecuteGetAllocationInfo(size_t lineNumber, const CsvSplit& csvSpli } } +void Player::ExecuteMakePoolAllocationsLost(size_t lineNumber, const CsvSplit& csvSplit) +{ + m_Stats.RegisterFunctionCall(VMA_FUNCTION::MakePoolAllocationsLost); + + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 1, false)) + { + uint64_t origPtr = 0; + + if(StrRangeToPtr(csvSplit.GetRange(FIRST_PARAM_INDEX), origPtr)) + { + if(origPtr != 0) + { + const auto it = m_Pools.find(origPtr); + if(it != m_Pools.end()) + { + vmaMakePoolAllocationsLost(m_Allocator, it->second.pool, nullptr); + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Pool %llX not found.\n", lineNumber, origPtr); + } + } + } + } + else + { + if(IssueWarning()) + { + printf("Line %zu: Invalid parameters for vmaMakePoolAllocationsLost.\n", lineNumber); + } + } + } +} + //////////////////////////////////////////////////////////////////////////////// // Main functions diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 95e6260..479b4b4 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -4854,6 +4854,8 @@ public: VmaAllocation allocation); void RecordGetAllocationInfo(uint32_t frameIndex, VmaAllocation allocation); + void RecordMakePoolAllocationsLost(uint32_t frameIndex, + VmaPool pool); private: struct CallParams @@ -8507,6 +8509,18 @@ void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex, Flush(); } +void VmaRecorder::RecordMakePoolAllocationsLost(uint32_t frameIndex, + VmaPool pool) +{ + CallParams callParams; + GetBasicParams(callParams); + + VmaMutexLock lock(m_FileMutex, m_UseMutex); + fprintf(m_File, "%u,%.3f,%u,vmaMakePoolAllocationsLost,%p\n", callParams.threadId, callParams.time, frameIndex, + pool); + Flush(); +} + VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData) { if(pUserData != VMA_NULL) @@ -10401,6 +10415,13 @@ void vmaMakePoolAllocationsLost( VMA_DEBUG_GLOBAL_MUTEX_LOCK +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + allocator->GetRecorder()->RecordMakePoolAllocationsLost(allocator->GetCurrentFrameIndex(), pool); + } +#endif + allocator->MakePoolAllocationsLost(pool, pLostAllocationCount); } From 069dc11452eb2a1ad9fa0c463621b0e2934b6cd4 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 13:19:27 +0200 Subject: [PATCH 38/51] VmaReplay: added --Lines command line argument to limit playback to only range of file lines. Created template class RangeSequence. Moved struct StrRange, class LineSplit, class CsvSplit and related functions to Common.*. --- src/VmaReplay/Common.cpp | 42 ++++++ src/VmaReplay/Common.h | 247 +++++++++++++++++++++++++++++++++++- src/VmaReplay/VmaReplay.cpp | 197 ++++++---------------------- 3 files changed, 325 insertions(+), 161 deletions(-) diff --git a/src/VmaReplay/Common.cpp b/src/VmaReplay/Common.cpp index cfb44dc..0350343 100644 --- a/src/VmaReplay/Common.cpp +++ b/src/VmaReplay/Common.cpp @@ -1,5 +1,47 @@ #include "Common.h" +//////////////////////////////////////////////////////////////////////////////// +// LineSplit class + +bool LineSplit::GetNextLine(StrRange& out) +{ + if(m_NextLineBeg < m_NumBytes) + { + out.beg = m_Data + m_NextLineBeg; + size_t currLineEnd = m_NextLineBeg; + while(currLineEnd < m_NumBytes && m_Data[currLineEnd] != '\n') + ++currLineEnd; + out.end = m_Data + currLineEnd; + m_NextLineBeg = currLineEnd + 1; // Past '\n' + ++m_NextLineIndex; + return true; + } + else + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// CsvSplit class + +void CsvSplit::Set(const StrRange& line, size_t maxCount) +{ + assert(maxCount <= RANGE_COUNT_MAX); + m_Line = line; + const size_t strLen = line.length(); + size_t rangeIndex = 0; + size_t charIndex = 0; + while(charIndex < strLen && rangeIndex < maxCount) + { + m_Ranges[rangeIndex * 2] = charIndex; + while(charIndex < strLen && (rangeIndex + 1 == maxCount || m_Line.beg[charIndex] != ',')) + ++charIndex; + m_Ranges[rangeIndex * 2 + 1] = charIndex; + ++rangeIndex; + ++charIndex; // Past ',' + } + m_Count = rangeIndex; +} + //////////////////////////////////////////////////////////////////////////////// // class CmdLineParser diff --git a/src/VmaReplay/Common.h b/src/VmaReplay/Common.h index f5f4c44..e4b6251 100644 --- a/src/VmaReplay/Common.h +++ b/src/VmaReplay/Common.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -30,17 +31,131 @@ inline float ToFloatSeconds(duration d) void SecondsToFriendlyStr(float seconds, std::string& out); template -inline T ceil_div(T x, T y) +T ceil_div(T x, T y) { return (x+y-1) / y; } template -static inline T align_up(T val, T align) +inline T align_up(T val, T align) { return (val + align - 1) / align * align; } +struct StrRange +{ + const char* beg; + const char* end; + + StrRange() { } + StrRange(const char* beg, const char* end) : beg(beg), end(end) { } + explicit StrRange(const char* sz) : beg(sz), end(sz + strlen(sz)) { } + explicit StrRange(const std::string& s) : beg(s.data()), end(s.data() + s.length()) { } + + size_t length() const { return end - beg; } + void to_str(std::string& out) { out.assign(beg, end); } +}; + +inline bool StrRangeEq(const StrRange& lhs, const char* rhsSz) +{ + const size_t rhsLen = strlen(rhsSz); + return rhsLen == lhs.length() && + memcmp(lhs.beg, rhsSz, rhsLen) == 0; +} + +inline bool StrRangeToUint(const StrRange& s, uint32_t& out) +{ + char* end = (char*)s.end; + out = (uint32_t)strtoul(s.beg, &end, 10); + return end == s.end; +} +inline bool StrRangeToUint(const StrRange& s, uint64_t& out) +{ + char* end = (char*)s.end; + out = (uint64_t)strtoull(s.beg, &end, 10); + return end == s.end; +} +inline bool StrRangeToPtr(const StrRange& s, uint64_t& out) +{ + char* end = (char*)s.end; + out = (uint64_t)strtoull(s.beg, &end, 16); + return end == s.end; +} +inline bool StrRangeToFloat(const StrRange& s, float& out) +{ + char* end = (char*)s.end; + out = strtof(s.beg, &end); + return end == s.end; +} +inline bool StrRangeToBool(const StrRange& s, bool& out) +{ + if(s.end - s.beg == 1) + { + if(*s.beg == '1') + { + out = true; + } + else if(*s.beg == '0') + { + out = false; + } + else + { + return false; + } + } + else + { + return false; + } + + return true; +} + +class LineSplit +{ +public: + LineSplit(const char* data, size_t numBytes) : + m_Data(data), + m_NumBytes(numBytes), + m_NextLineBeg(0), + m_NextLineIndex(0) + { + } + + bool GetNextLine(StrRange& out); + size_t GetNextLineIndex() const { return m_NextLineIndex; } + +private: + const char* const m_Data; + const size_t m_NumBytes; + size_t m_NextLineBeg; + size_t m_NextLineIndex; +}; + +class CsvSplit +{ +public: + static const size_t RANGE_COUNT_MAX = 32; + + void Set(const StrRange& line, size_t maxCount = RANGE_COUNT_MAX); + + const StrRange& GetLine() const { return m_Line; } + + size_t GetCount() const { return m_Count; } + StrRange GetRange(size_t index) const + { + return StrRange { + m_Line.beg + m_Ranges[index * 2], + m_Line.beg + m_Ranges[index * 2 + 1] }; + } + +private: + StrRange m_Line = { nullptr, nullptr }; + size_t m_Count = 0; + size_t m_Ranges[RANGE_COUNT_MAX * 2]; // Pairs of begin-end. +}; + class CmdLineParser { public: @@ -102,6 +217,134 @@ private: std::string m_LastParameter; }; +/* +Parses and stores a sequence of ranges. + +Upper range is inclusive. + +Examples: + + "1" -> [ {1, 1} ] + "1,10" -> [ {1, 1}, {10, 10} ] + "2-6" -> [ {2, 6} ] + "-8" -> [ {MIN, 8} ] + "12-" -> [ {12, MAX} ] + "1-10,12,15-" -> [ {1, 10}, {12, 12}, {15, MAX} ] + +TODO: Optimize it: Do sorting and merging while parsing. Do binary search while +reading. +*/ +template +class RangeSequence +{ +public: + typedef std::pair RangeType; + + void Clear() { m_Ranges.clear(); } + bool Parse(const StrRange& str); + + bool IsEmpty() const { return m_Ranges.empty(); } + size_t GetCount() const { return m_Ranges.size(); } + const RangeType* GetRanges() const { return m_Ranges.data(); } + + bool Includes(T number) const; + +private: + std::vector m_Ranges; +}; + +template +bool RangeSequence::Parse(const StrRange& str) +{ + m_Ranges.clear(); + + StrRange currRange = { str.beg, str.beg }; + while(currRange.beg < str.end) + { + currRange.end = currRange.beg + 1; + // Find next ',' or the end. + while(currRange.end < str.end && *currRange.end != ',') + { + ++currRange.end; + } + + // Find '-' within this range. + const char* hyphenPos = currRange.beg; + while(hyphenPos < currRange.end && *hyphenPos != '-') + { + ++hyphenPos; + } + + // No hyphen - single number like '10'. + if(hyphenPos == currRange.end) + { + RangeType range; + if(!StrRangeToUint(currRange, range.first)) + { + return false; + } + range.second = range.first; + m_Ranges.push_back(range); + } + // Hyphen at the end, like '10-'. + else if(hyphenPos + 1 == currRange.end) + { + const StrRange numberRange = { currRange.beg, hyphenPos }; + RangeType range; + if(!StrRangeToUint(numberRange, range.first)) + { + return false; + } + range.second = std::numeric_limits::max(); + m_Ranges.push_back(range); + } + // Hyphen at the beginning, like "-10". + else if(hyphenPos == currRange.beg) + { + const StrRange numberRange = { currRange.beg + 1, currRange.end }; + RangeType range; + range.first = std::numeric_limits::min(); + if(!StrRangeToUint(numberRange, range.second)) + { + return false; + } + m_Ranges.push_back(range); + } + // Hyphen in the middle, like "1-10". + else + { + const StrRange numberRange1 = { currRange.beg, hyphenPos }; + const StrRange numberRange2 = { hyphenPos + 1, currRange.end }; + RangeType range; + if(!StrRangeToUint(numberRange1, range.first) || + !StrRangeToUint(numberRange2, range.second) || + range.second < range.first) + { + return false; + } + m_Ranges.push_back(range); + } + + // Skip ',' + currRange.beg = currRange.end + 1; + } + + return true; +} + +template +bool RangeSequence::Includes(T number) const +{ + for(const auto& it : m_Ranges) + { + if(number >= it.first && number <= it.second) + { + return true; + } + } + return false; +} + /* class RandomNumberGenerator { diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 58cd279..dba1005 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -33,6 +33,7 @@ static const int RESULT_ERROR_VULKAN = -4; enum CMD_LINE_OPT { CMD_LINE_OPT_VERBOSITY, + CMD_LINE_OPT_LINES, }; static enum class VERBOSITY @@ -97,6 +98,8 @@ static std::string g_FilePath; // Most significant 16 bits are major version, least significant 16 bits are minor version. static uint32_t g_FileVersion; +static RangeSequence g_LineRanges; + static bool ValidateFileVersion() { const uint32_t major = g_FileVersion >> 16; @@ -104,163 +107,6 @@ static bool ValidateFileVersion() return major == 1 && minor <= 2; } -struct StrRange -{ - const char* beg; - const char* end; - - StrRange() { } - StrRange(const char* beg, const char* end) : beg(beg), end(end) { } - explicit StrRange(const char* sz) : beg(sz), end(sz + strlen(sz)) { } - explicit StrRange(const std::string& s) : beg(s.data()), end(s.data() + s.length()) { } - - size_t length() const { return end - beg; } - void to_str(std::string& out) { out.assign(beg, end); } -}; - -static inline bool StrRangeEq(const StrRange& lhs, const char* rhsSz) -{ - const size_t rhsLen = strlen(rhsSz); - return rhsLen == lhs.length() && - memcmp(lhs.beg, rhsSz, rhsLen) == 0; -} - -static inline bool StrRangeToUint(const StrRange& s, uint32_t& out) -{ - char* end = (char*)s.end; - out = (uint32_t)strtoul(s.beg, &end, 10); - return end == s.end; -} -static inline bool StrRangeToUint(const StrRange& s, uint64_t& out) -{ - char* end = (char*)s.end; - out = (uint64_t)strtoull(s.beg, &end, 10); - return end == s.end; -} -static inline bool StrRangeToPtr(const StrRange& s, uint64_t& out) -{ - char* end = (char*)s.end; - out = (uint64_t)strtoull(s.beg, &end, 16); - return end == s.end; -} -static inline bool StrRangeToFloat(const StrRange& s, float& out) -{ - char* end = (char*)s.end; - out = strtof(s.beg, &end); - return end == s.end; -} -static inline bool StrRangeToBool(const StrRange& s, bool& out) -{ - if(s.end - s.beg == 1) - { - if(*s.beg == '1') - { - out = true; - } - else if(*s.beg == '0') - { - out = false; - } - else - { - return false; - } - } - else - { - return false; - } - - return true; -} - -//////////////////////////////////////////////////////////////////////////////// -// LineSplit class - -class LineSplit -{ -public: - LineSplit(const char* data, size_t numBytes) : - m_Data(data), - m_NumBytes(numBytes), - m_NextLineBeg(0), - m_NextLineIndex(0) - { - } - - bool GetNextLine(StrRange& out); - size_t GetNextLineIndex() const { return m_NextLineIndex; } - -private: - const char* const m_Data; - const size_t m_NumBytes; - size_t m_NextLineBeg; - size_t m_NextLineIndex; -}; - -bool LineSplit::GetNextLine(StrRange& out) -{ - if(m_NextLineBeg < m_NumBytes) - { - out.beg = m_Data + m_NextLineBeg; - size_t currLineEnd = m_NextLineBeg; - while(currLineEnd < m_NumBytes && m_Data[currLineEnd] != '\n') - ++currLineEnd; - out.end = m_Data + currLineEnd; - m_NextLineBeg = currLineEnd + 1; // Past '\n' - ++m_NextLineIndex; - return true; - } - else - return false; -} - - -//////////////////////////////////////////////////////////////////////////////// -// CsvSplit class - -class CsvSplit -{ -public: - static const size_t RANGE_COUNT_MAX = 32; - - void Set(const StrRange& line, size_t maxCount = RANGE_COUNT_MAX); - - const StrRange& GetLine() const { return m_Line; } - - size_t GetCount() const { return m_Count; } - StrRange GetRange(size_t index) const - { - return StrRange { - m_Line.beg + m_Ranges[index * 2], - m_Line.beg + m_Ranges[index * 2 + 1] }; - } - -private: - StrRange m_Line = { nullptr, nullptr }; - size_t m_Count = 0; - size_t m_Ranges[RANGE_COUNT_MAX * 2]; // Pairs of begin-end. -}; - -void CsvSplit::Set(const StrRange& line, size_t maxCount) -{ - assert(maxCount <= RANGE_COUNT_MAX); - m_Line = line; - const size_t strLen = line.length(); - size_t rangeIndex = 0; - size_t charIndex = 0; - while(charIndex < strLen && rangeIndex < maxCount) - { - m_Ranges[rangeIndex * 2] = charIndex; - while(charIndex < strLen && (rangeIndex + 1 == maxCount || m_Line.beg[charIndex] != ',')) - ++charIndex; - m_Ranges[rangeIndex * 2 + 1] = charIndex; - ++rangeIndex; - ++charIndex; // Past ',' - } - m_Count = rangeIndex; -} - static bool ParseFileVersion(const StrRange& s) { CsvSplit csvSplit; @@ -2042,11 +1888,15 @@ static void PrintCommandLineSyntax() " 0 - Minimum verbosity. Prints only warnings and errors.\n" " 1 - Default verbosity. Prints important messages and statistics.\n" " 2 - Maximum verbosity. Prints a lot of information.\n" + " --Lines - Replay only limited set of lines from file\n" + " Ranges is comma-separated list of ranges, e.g. \"-10,15,18-25,31-\".\n" ); } static int ProcessFile(const char* data, size_t numBytes) { + const bool useLineRanges = !g_LineRanges.IsEmpty(); + // Begin stats. if(g_Verbosity == VERBOSITY::MAXIMUM) { @@ -2076,18 +1926,36 @@ static int ProcessFile(const char* data, size_t numBytes) Player player; int result = player.Init(); + size_t executedLineCount = 0; if(result == 0) { if(g_Verbosity > VERBOSITY::MINIMUM) { - printf("Playing...\n"); + if(useLineRanges) + { + printf("Playing (limited range of lines)...\n"); + } + else + { + printf("Playing...\n"); + } } const time_point timeBeg = std::chrono::high_resolution_clock::now(); while(lineSplit.GetNextLine(line)) { - player.ExecuteLine(lineSplit.GetNextLineIndex(), line); + bool execute = true; + if(useLineRanges) + { + execute = g_LineRanges.Includes(lineSplit.GetNextLineIndex()); + } + + if(execute) + { + player.ExecuteLine(lineSplit.GetNextLineIndex(), line); + ++executedLineCount; + } } const duration playDuration = std::chrono::high_resolution_clock::now() - timeBeg; @@ -2099,6 +1967,7 @@ static int ProcessFile(const char* data, size_t numBytes) SecondsToFriendlyStr(ToFloatSeconds(playDuration), playDurationStr); printf("Done.\n"); + printf("Executed %zu file lines\n", executedLineCount); printf("Playback took: %s\n", playDurationStr.c_str()); } if(g_Verbosity == VERBOSITY::MAXIMUM) @@ -2154,6 +2023,7 @@ static int main2(int argc, char** argv) CmdLineParser cmdLineParser(argc, argv); cmdLineParser.RegisterOpt(CMD_LINE_OPT_VERBOSITY, 'v', true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_LINES, "Lines", true); CmdLineParser::RESULT res; while((res = cmdLineParser.ReadNext()) != CmdLineParser::RESULT_END) @@ -2178,6 +2048,15 @@ static int main2(int argc, char** argv) } } break; + case CMD_LINE_OPT_LINES: + { + if(!g_LineRanges.Parse(StrRange(cmdLineParser.GetParameter()))) + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + } + break; default: assert(0); } From 4b360aa9a85a41b5c9005221ac87409d40931360 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 13:31:04 +0200 Subject: [PATCH 39/51] VmaReplay: Added -i parameter for repeating playback a number of times (iterations). --- src/VmaReplay/VmaReplay.cpp | 63 ++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index dba1005..a186c77 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -33,6 +33,7 @@ static const int RESULT_ERROR_VULKAN = -4; enum CMD_LINE_OPT { CMD_LINE_OPT_VERBOSITY, + CMD_LINE_OPT_ITERATIONS, CMD_LINE_OPT_LINES, }; @@ -98,6 +99,7 @@ static std::string g_FilePath; // Most significant 16 bits are major version, least significant 16 bits are minor version. static uint32_t g_FileVersion; +static size_t g_IterationCount = 1; static RangeSequence g_LineRanges; static bool ValidateFileVersion() @@ -680,7 +682,7 @@ bool Player::IssueWarning() int Player::InitVulkan() { - if(g_Verbosity > VERBOSITY::MINIMUM) + if(g_Verbosity == VERBOSITY::MAXIMUM) { printf("Initializing Vulkan...\n"); } @@ -1888,20 +1890,18 @@ static void PrintCommandLineSyntax() " 0 - Minimum verbosity. Prints only warnings and errors.\n" " 1 - Default verbosity. Prints important messages and statistics.\n" " 2 - Maximum verbosity. Prints a lot of information.\n" + " -i - Repeat playback given number of times (iterations)\n" + " Default is 1. Vulkan is reinitialized with every iteration.\n" " --Lines - Replay only limited set of lines from file\n" " Ranges is comma-separated list of ranges, e.g. \"-10,15,18-25,31-\".\n" ); } -static int ProcessFile(const char* data, size_t numBytes) +static int ProcessFile(size_t iterationIndex, const char* data, size_t numBytes, duration& outDuration) { - const bool useLineRanges = !g_LineRanges.IsEmpty(); + outDuration = duration::max(); - // Begin stats. - if(g_Verbosity == VERBOSITY::MAXIMUM) - { - printf("File size: %zu B\n", numBytes); - } + const bool useLineRanges = !g_LineRanges.IsEmpty(); LineSplit lineSplit(data, numBytes); StrRange line; @@ -1933,11 +1933,11 @@ static int ProcessFile(const char* data, size_t numBytes) { if(useLineRanges) { - printf("Playing (limited range of lines)...\n"); + printf("Playing #%zu (limited range of lines)...\n", iterationIndex + 1); } else { - printf("Playing...\n"); + printf("Playing #%zu...\n", iterationIndex + 1); } } @@ -1959,6 +1959,7 @@ static int ProcessFile(const char* data, size_t numBytes) } const duration playDuration = std::chrono::high_resolution_clock::now() - timeBeg; + outDuration = playDuration; // End stats. if(g_Verbosity > VERBOSITY::MINIMUM) @@ -1967,12 +1968,12 @@ static int ProcessFile(const char* data, size_t numBytes) SecondsToFriendlyStr(ToFloatSeconds(playDuration), playDurationStr); printf("Done.\n"); - printf("Executed %zu file lines\n", executedLineCount); printf("Playback took: %s\n", playDurationStr.c_str()); } if(g_Verbosity == VERBOSITY::MAXIMUM) { printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); + printf("Executed %zu file lines\n", executedLineCount); } } @@ -1999,7 +2000,27 @@ static int ProcessFile() { std::vector fileContents(fileSize); fread(fileContents.data(), 1, fileSize, file); - ProcessFile(fileContents.data(), fileContents.size()); + + // Begin stats. + if(g_Verbosity == VERBOSITY::MAXIMUM) + { + printf("File size: %zu B\n", fileSize); + } + + duration durationSum = duration::zero(); + for(size_t i = 0; i < g_IterationCount; ++i) + { + duration currDuration; + ProcessFile(i, fileContents.data(), fileContents.size(), currDuration); + durationSum += currDuration; + } + + if(g_IterationCount > 1) + { + std::string playDurationStr; + SecondsToFriendlyStr(ToFloatSeconds(durationSum / g_IterationCount), playDurationStr); + printf("Average playback time from %zu iterations: %s\n", g_IterationCount, playDurationStr.c_str()); + } } else { @@ -2023,6 +2044,7 @@ static int main2(int argc, char** argv) CmdLineParser cmdLineParser(argc, argv); cmdLineParser.RegisterOpt(CMD_LINE_OPT_VERBOSITY, 'v', true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_ITERATIONS, 'i', true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_LINES, "Lines", true); CmdLineParser::RESULT res; @@ -2048,13 +2070,18 @@ static int main2(int argc, char** argv) } } break; - case CMD_LINE_OPT_LINES: + case CMD_LINE_OPT_ITERATIONS: + if(!StrRangeToUint(StrRange(cmdLineParser.GetParameter()), g_IterationCount)) { - if(!g_LineRanges.Parse(StrRange(cmdLineParser.GetParameter()))) - { - PrintCommandLineSyntax(); - return RESULT_ERROR_COMMAND_LINE; - } + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + break; + case CMD_LINE_OPT_LINES: + if(!g_LineRanges.Parse(StrRange(cmdLineParser.GetParameter()))) + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; } break; default: From eaa10d11fae1206d575e65eb43ea72e11073b5cc Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 13:53:46 +0200 Subject: [PATCH 40/51] Added function InitVulkanFeatures - better code for enabling Vulkan device features. --- src/VmaReplay/VmaReplay.cpp | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index a186c77..e88dde6 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -328,6 +328,24 @@ static bool IsLayerSupported(const VkLayerProperties* pProps, size_t propCount, static const size_t FIRST_PARAM_INDEX = 4; +static void InitVulkanFeatures( + VkPhysicalDeviceFeatures& outFeatures, + const VkPhysicalDeviceFeatures& supportedFeatures) +{ + ZeroMemory(&outFeatures, sizeof(outFeatures)); + + // Enable something what may interact with memory/buffer/image support. + + outFeatures.fullDrawIndexUint32 = supportedFeatures.fullDrawIndexUint32; + outFeatures.imageCubeArray = supportedFeatures.imageCubeArray; + outFeatures.geometryShader = supportedFeatures.geometryShader; + outFeatures.tessellationShader = supportedFeatures.tessellationShader; + outFeatures.multiDrawIndirect = supportedFeatures.multiDrawIndirect; + outFeatures.textureCompressionETC2 = supportedFeatures.textureCompressionETC2; + outFeatures.textureCompressionASTC_LDR = supportedFeatures.textureCompressionASTC_LDR; + outFeatures.textureCompressionBC = supportedFeatures.textureCompressionBC; +} + class Player { public: @@ -785,6 +803,9 @@ int Player::InitVulkan() return RESULT_ERROR_VULKAN; } + VkPhysicalDeviceFeatures supportedFeatures; + vkGetPhysicalDeviceFeatures(m_PhysicalDevice, &supportedFeatures); + // Create logical device const float queuePriority = 1.f; @@ -795,13 +816,8 @@ int Player::InitVulkan() deviceQueueCreateInfo.pQueuePriorities = &queuePriority; // Enable something what may interact with memory/buffer/image support. - VkPhysicalDeviceFeatures deviceFeatures = {}; - deviceFeatures.fullDrawIndexUint32 = VK_TRUE; - deviceFeatures.imageCubeArray = VK_TRUE; - deviceFeatures.geometryShader = VK_TRUE; - deviceFeatures.tessellationShader = VK_TRUE; - deviceFeatures.multiDrawIndirect = VK_TRUE; - deviceFeatures.textureCompressionBC = VK_TRUE; + VkPhysicalDeviceFeatures enabledFeatures; + InitVulkanFeatures(enabledFeatures, supportedFeatures); // Determine list of device extensions to enable. std::vector enabledDeviceExtensions; @@ -838,7 +854,7 @@ int Player::InitVulkan() deviceCreateInfo.ppEnabledExtensionNames = !enabledDeviceExtensions.empty() ? enabledDeviceExtensions.data() : nullptr; deviceCreateInfo.queueCreateInfoCount = 1; deviceCreateInfo.pQueueCreateInfos = &deviceQueueCreateInfo; - deviceCreateInfo.pEnabledFeatures = &deviceFeatures; + deviceCreateInfo.pEnabledFeatures = &enabledFeatures; res = vkCreateDevice(m_PhysicalDevice, &deviceCreateInfo, nullptr, &m_Device); if(res != VK_SUCCESS) From 7c1868ba32b5a69904ed8db2ee0c1a1a0db60587 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 14:01:04 +0200 Subject: [PATCH 41/51] VmaReplay: Added parameter --PhysicalDevice for choice of Vulkan physical device. --- src/VmaReplay/VmaReplay.cpp | 45 ++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index e88dde6..dfe84c3 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -35,6 +35,7 @@ enum CMD_LINE_OPT CMD_LINE_OPT_VERBOSITY, CMD_LINE_OPT_ITERATIONS, CMD_LINE_OPT_LINES, + CMD_LINE_OPT_PHYSICAL_DEVICE, }; static enum class VERBOSITY @@ -100,6 +101,7 @@ static std::string g_FilePath; static uint32_t g_FileVersion; static size_t g_IterationCount = 1; +static uint32_t g_PhysicalDeviceIndex = 0; static RangeSequence g_LineRanges; static bool ValidateFileVersion() @@ -354,6 +356,7 @@ public: ~Player(); void ExecuteLine(size_t lineNumber, const StrRange& line); + void PrintStats(); private: static const size_t MAX_WARNINGS_TO_SHOW = 64; @@ -416,7 +419,6 @@ private: int InitVulkan(); void FinalizeVulkan(); void RegisterDebugCallbacks(); - void PrintStats(); // If parmeter count doesn't match, issues warning and returns false. bool ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound); @@ -457,11 +459,6 @@ int Player::Init() Player::~Player() { - if(g_Verbosity > VERBOSITY::MINIMUM) - { - PrintStats(); - } - FinalizeVulkan(); if(g_Verbosity < VERBOSITY::MAXIMUM && m_WarningCount > MAX_WARNINGS_TO_SHOW) @@ -762,21 +759,27 @@ int Player::InitVulkan() // Find physical device - uint32_t deviceCount = 0; - res = vkEnumeratePhysicalDevices(m_VulkanInstance, &deviceCount, nullptr); + uint32_t physicalDeviceCount = 0; + res = vkEnumeratePhysicalDevices(m_VulkanInstance, &physicalDeviceCount, nullptr); assert(res == VK_SUCCESS); - if(deviceCount == 0) + if(physicalDeviceCount == 0) { printf("ERROR: No Vulkan physical devices found.\n"); return RESULT_ERROR_VULKAN; } - else if(deviceCount > 1) - printf("WARNING: %u Vulkan physical devices found. Choosing first one.\n", deviceCount); - std::vector physicalDevices(deviceCount); - res = vkEnumeratePhysicalDevices(m_VulkanInstance, &deviceCount, physicalDevices.data()); + std::vector physicalDevices(physicalDeviceCount); + res = vkEnumeratePhysicalDevices(m_VulkanInstance, &physicalDeviceCount, physicalDevices.data()); assert(res == VK_SUCCESS); + if(g_PhysicalDeviceIndex >= physicalDeviceCount) + { + printf("ERROR: Incorrect Vulkan physical device index %u. System has %u physical devices.\n", + g_PhysicalDeviceIndex, + physicalDeviceCount); + return RESULT_ERROR_VULKAN; + } + m_PhysicalDevice = physicalDevices[0]; // Find queue family index @@ -964,6 +967,11 @@ void Player::RegisterDebugCallbacks() void Player::PrintStats() { + if(g_Verbosity == VERBOSITY::MINIMUM) + { + return; + } + printf("Statistics:\n"); if(m_Stats.GetAllocationCreationCount() > 0) { @@ -1910,6 +1918,7 @@ static void PrintCommandLineSyntax() " Default is 1. Vulkan is reinitialized with every iteration.\n" " --Lines - Replay only limited set of lines from file\n" " Ranges is comma-separated list of ranges, e.g. \"-10,15,18-25,31-\".\n" + " --PhysicalDevice - Choice of Vulkan physical device. Default: 0.\n" ); } @@ -1991,6 +2000,8 @@ static int ProcessFile(size_t iterationIndex, const char* data, size_t numBytes, printf("File lines: %zu\n", lineSplit.GetNextLineIndex()); printf("Executed %zu file lines\n", executedLineCount); } + + player.PrintStats(); } return result; @@ -2062,6 +2073,7 @@ static int main2(int argc, char** argv) cmdLineParser.RegisterOpt(CMD_LINE_OPT_VERBOSITY, 'v', true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_ITERATIONS, 'i', true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_LINES, "Lines", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_PHYSICAL_DEVICE, "PhysicalDevice", true); CmdLineParser::RESULT res; while((res = cmdLineParser.ReadNext()) != CmdLineParser::RESULT_END) @@ -2100,6 +2112,13 @@ static int main2(int argc, char** argv) return RESULT_ERROR_COMMAND_LINE; } break; + case CMD_LINE_OPT_PHYSICAL_DEVICE: + if(!StrRangeToUint(StrRange(cmdLineParser.GetParameter()), g_PhysicalDeviceIndex)) + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + break; default: assert(0); } From 821e36705ffb191b15548eee60cf724ff6bf0d29 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 14:10:16 +0200 Subject: [PATCH 42/51] VmaReplay: Added parameter --UserData which allows to disable setting pUserData while playing. --- src/VmaReplay/VmaReplay.cpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index dfe84c3..cf3a1b3 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -36,6 +36,7 @@ enum CMD_LINE_OPT CMD_LINE_OPT_ITERATIONS, CMD_LINE_OPT_LINES, CMD_LINE_OPT_PHYSICAL_DEVICE, + CMD_LINE_OPT_USER_DATA, }; static enum class VERBOSITY @@ -103,6 +104,7 @@ static uint32_t g_FileVersion; static size_t g_IterationCount = 1; static uint32_t g_PhysicalDeviceIndex = 0; static RangeSequence g_LineRanges; +static bool g_UserDataEnabled = true; static bool ValidateFileVersion() { @@ -1088,6 +1090,12 @@ bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& c bool Player::PrepareUserData(size_t lineNumber, uint32_t allocCreateFlags, const StrRange& userDataColumn, const StrRange& wholeLine, void*& outUserData) { + if(!g_UserDataEnabled) + { + outUserData = nullptr; + return true; + } + // String if((allocCreateFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0) { @@ -1241,6 +1249,11 @@ void Player::ExecuteSetAllocationUserData(size_t lineNumber, const CsvSplit& csv { m_Stats.RegisterFunctionCall(VMA_FUNCTION::SetAllocationUserData); + if(!g_UserDataEnabled) + { + return; + } + if(ValidateFunctionParameterCount(lineNumber, csvSplit, 2, true)) { uint64_t origPtr = 0; @@ -1910,7 +1923,7 @@ static void PrintCommandLineSyntax() "Command line syntax:\n" " VmaReplay [Options] \n" "Available options:\n" - " -v - Verbosity level:\n" + " -v - Verbosity level:\n" " 0 - Minimum verbosity. Prints only warnings and errors.\n" " 1 - Default verbosity. Prints important messages and statistics.\n" " 2 - Maximum verbosity. Prints a lot of information.\n" @@ -1919,6 +1932,8 @@ static void PrintCommandLineSyntax() " --Lines - Replay only limited set of lines from file\n" " Ranges is comma-separated list of ranges, e.g. \"-10,15,18-25,31-\".\n" " --PhysicalDevice - Choice of Vulkan physical device. Default: 0.\n" + " --UserData - 0 to disable or 1 to enable setting pUserData during playback.\n" + " Default is 1. Affects both creation of buffers and images, as well as calls to vmaSetAllocationUserData.\n" ); } @@ -2074,6 +2089,7 @@ static int main2(int argc, char** argv) cmdLineParser.RegisterOpt(CMD_LINE_OPT_ITERATIONS, 'i', true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_LINES, "Lines", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_PHYSICAL_DEVICE, "PhysicalDevice", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_USER_DATA, "UserData", true); CmdLineParser::RESULT res; while((res = cmdLineParser.ReadNext()) != CmdLineParser::RESULT_END) @@ -2119,6 +2135,13 @@ static int main2(int argc, char** argv) return RESULT_ERROR_COMMAND_LINE; } break; + case CMD_LINE_OPT_USER_DATA: + if(!StrRangeToBool(StrRange(cmdLineParser.GetParameter()), g_UserDataEnabled)) + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + break; default: assert(0); } From f17013e6998b9e2c7537626b948976388c4a9465 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 14:12:38 +0200 Subject: [PATCH 43/51] Added constant CLEANUP_LEAKED_OBJECTS. --- src/VmaReplay/VmaReplay.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index cf3a1b3..c21ff47 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -97,6 +97,10 @@ static_assert( _countof(VMA_FUNCTION_NAMES) == (size_t)VMA_FUNCTION::Count, "VMA_FUNCTION_NAMES array doesn't match VMA_FUNCTION enum."); +// Set this to false to disable deleting leaked VmaAllocation, VmaPool objects +// and let VMA report asserts about them. +static const bool CLEANUP_LEAKED_OBJECTS = true; + static std::string g_FilePath; // Most significant 16 bits are major version, least significant 16 bits are minor version. static uint32_t g_FileVersion; @@ -895,9 +899,12 @@ void Player::FinalizeVulkan() { printf("WARNING: Allocations not destroyed: %zu.\n", m_Allocations.size()); - for(const auto it : m_Allocations) + if(CLEANUP_LEAKED_OBJECTS) { - Destroy(it.second); + for(const auto it : m_Allocations) + { + Destroy(it.second); + } } m_Allocations.clear(); @@ -907,8 +914,13 @@ void Player::FinalizeVulkan() { printf("WARNING: Custom pools not destroyed: %zu.\n", m_Pools.size()); - for(const auto it : m_Pools) - vmaDestroyPool(m_Allocator, it.second.pool); + if(CLEANUP_LEAKED_OBJECTS) + { + for(const auto it : m_Pools) + { + vmaDestroyPool(m_Allocator, it.second.pool); + } + } m_Pools.clear(); } From 33c1e8ddeff3087afcc27a812661f0cd2c6e9b6e Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 14:26:37 +0200 Subject: [PATCH 44/51] VmaReplay: added command line option --VK_KHR_dedicated_allocation. --- src/VmaReplay/VmaReplay.cpp | 68 +++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index c21ff47..ee310df 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -37,6 +37,7 @@ enum CMD_LINE_OPT CMD_LINE_OPT_LINES, CMD_LINE_OPT_PHYSICAL_DEVICE, CMD_LINE_OPT_USER_DATA, + CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, }; static enum class VERBOSITY @@ -47,6 +48,13 @@ static enum class VERBOSITY COUNT, } g_Verbosity = VERBOSITY::DEFAULT; +enum class VULKAN_EXTENSION_REQUEST +{ + DISABLED, + ENABLED, + DEFAULT +}; + enum class OBJECT_TYPE { BUFFER, IMAGE }; enum class VMA_FUNCTION @@ -109,6 +117,7 @@ static size_t g_IterationCount = 1; static uint32_t g_PhysicalDeviceIndex = 0; static RangeSequence g_LineRanges; static bool g_UserDataEnabled = true; +VULKAN_EXTENSION_REQUEST g_VK_KHR_dedicated_allocation_request = VULKAN_EXTENSION_REQUEST::DEFAULT; static bool ValidateFileVersion() { @@ -274,8 +283,6 @@ static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_LUNARG_standard_valid static bool g_MemoryAliasingWarningEnabled = true; static bool g_EnableValidationLayer = true; -static bool VK_KHR_get_memory_requirements2_enabled = false; -static bool VK_KHR_dedicated_allocation_enabled = false; static VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( VkDebugReportFlagsEXT flags, @@ -828,6 +835,9 @@ int Player::InitVulkan() VkPhysicalDeviceFeatures enabledFeatures; InitVulkanFeatures(enabledFeatures, supportedFeatures); + bool VK_KHR_get_memory_requirements2_available = false; + bool VK_KHR_dedicated_allocation_available = false; + // Determine list of device extensions to enable. std::vector enabledDeviceExtensions; //enabledDeviceExtensions.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME); @@ -846,18 +856,43 @@ int Player::InitVulkan() { if(strcmp(properties[i].extensionName, VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME) == 0) { - enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); - VK_KHR_get_memory_requirements2_enabled = true; + VK_KHR_get_memory_requirements2_available = true; } else if(strcmp(properties[i].extensionName, VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME) == 0) { - enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); - VK_KHR_dedicated_allocation_enabled = true; + VK_KHR_dedicated_allocation_available = true; } } } } + const bool dedicatedAllocationAvailable = + VK_KHR_get_memory_requirements2_available && VK_KHR_dedicated_allocation_available; + + bool dedicatedAllocationEnabled = false; + switch(g_VK_KHR_dedicated_allocation_request) + { + case VULKAN_EXTENSION_REQUEST::DISABLED: + break; + case VULKAN_EXTENSION_REQUEST::DEFAULT: + dedicatedAllocationEnabled = dedicatedAllocationAvailable; + break; + case VULKAN_EXTENSION_REQUEST::ENABLED: + dedicatedAllocationEnabled = dedicatedAllocationAvailable; + if(!dedicatedAllocationAvailable) + { + printf("WARNING: VK_KHR_dedicated_allocation extension cannot be enabled.\n"); + } + break; + default: assert(0); + } + + if(dedicatedAllocationEnabled) + { + enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); + enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); + } + VkDeviceCreateInfo deviceCreateInfo = { VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO }; deviceCreateInfo.enabledExtensionCount = (uint32_t)enabledDeviceExtensions.size(); deviceCreateInfo.ppEnabledExtensionNames = !enabledDeviceExtensions.empty() ? enabledDeviceExtensions.data() : nullptr; @@ -878,7 +913,7 @@ int Player::InitVulkan() allocatorInfo.physicalDevice = m_PhysicalDevice; allocatorInfo.device = m_Device; - if(VK_KHR_dedicated_allocation_enabled) + if(dedicatedAllocationEnabled) { allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; } @@ -1946,6 +1981,8 @@ static void PrintCommandLineSyntax() " --PhysicalDevice - Choice of Vulkan physical device. Default: 0.\n" " --UserData - 0 to disable or 1 to enable setting pUserData during playback.\n" " Default is 1. Affects both creation of buffers and images, as well as calls to vmaSetAllocationUserData.\n" + " --VK_KHR_dedicated_allocation - 0 to disable or 1 to enable this extension.\n" + " By defalut the extension is silently enabled if available.\n" ); } @@ -2102,6 +2139,7 @@ static int main2(int argc, char** argv) cmdLineParser.RegisterOpt(CMD_LINE_OPT_LINES, "Lines", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_PHYSICAL_DEVICE, "PhysicalDevice", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_USER_DATA, "UserData", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, "VK_KHR_dedicated_allocation", true); CmdLineParser::RESULT res; while((res = cmdLineParser.ReadNext()) != CmdLineParser::RESULT_END) @@ -2154,6 +2192,22 @@ static int main2(int argc, char** argv) return RESULT_ERROR_COMMAND_LINE; } break; + case CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION: + { + bool newValue; + if(StrRangeToBool(StrRange(cmdLineParser.GetParameter()), newValue)) + { + g_VK_KHR_dedicated_allocation_request = newValue ? + VULKAN_EXTENSION_REQUEST::ENABLED : + VULKAN_EXTENSION_REQUEST::DISABLED; + } + else + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + } + break; default: assert(0); } From 1dcc72d7ddfa5e76a8c42210cb4f6ea6e07d6b29 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 14:33:12 +0200 Subject: [PATCH 45/51] VmaReplay: Added parameter --VK_LAYER_LUNARG_standard_validation to enable/disable Vulkan validation layers. --- src/VmaReplay/VmaReplay.cpp | 50 +++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index ee310df..f9370cf 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -38,6 +38,7 @@ enum CMD_LINE_OPT CMD_LINE_OPT_PHYSICAL_DEVICE, CMD_LINE_OPT_USER_DATA, CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, + CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION, }; static enum class VERBOSITY @@ -118,6 +119,7 @@ static uint32_t g_PhysicalDeviceIndex = 0; static RangeSequence g_LineRanges; static bool g_UserDataEnabled = true; VULKAN_EXTENSION_REQUEST g_VK_KHR_dedicated_allocation_request = VULKAN_EXTENSION_REQUEST::DEFAULT; +VULKAN_EXTENSION_REQUEST g_VK_LAYER_LUNARG_standard_validation = VULKAN_EXTENSION_REQUEST::DEFAULT; static bool ValidateFileVersion() { @@ -282,7 +284,6 @@ void Statistics::RegisterCreateAllocation() static const char* const VALIDATION_LAYER_NAME = "VK_LAYER_LUNARG_standard_validation"; static bool g_MemoryAliasingWarningEnabled = true; -static bool g_EnableValidationLayer = true; static VKAPI_ATTR VkBool32 VKAPI_CALL MyDebugReportCallback( VkDebugReportFlagsEXT flags, @@ -726,13 +727,25 @@ int Player::InitVulkan() assert(res == VK_SUCCESS); } - if(g_EnableValidationLayer == true) + const bool validationLayersAvailable = + IsLayerSupported(instanceLayerProps.data(), instanceLayerProps.size(), VALIDATION_LAYER_NAME); + + bool validationLayersEnabled = false; + switch(g_VK_LAYER_LUNARG_standard_validation) { - if(IsLayerSupported(instanceLayerProps.data(), instanceLayerProps.size(), VALIDATION_LAYER_NAME) == false) + case VULKAN_EXTENSION_REQUEST::DISABLED: + break; + case VULKAN_EXTENSION_REQUEST::DEFAULT: + validationLayersEnabled = validationLayersAvailable; + break; + case VULKAN_EXTENSION_REQUEST::ENABLED: + validationLayersEnabled = validationLayersAvailable; + if(!validationLayersAvailable) { - printf("WARNING: Layer \"%s\" not supported.\n", VALIDATION_LAYER_NAME); - g_EnableValidationLayer = false; + printf("WARNING: %s layer cannot be enabled.\n", VALIDATION_LAYER_NAME); } + break; + default: assert(0); } std::vector instanceExtensions; @@ -740,7 +753,7 @@ int Player::InitVulkan() //instanceExtensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME); std::vector instanceLayers; - if(g_EnableValidationLayer) + if(validationLayersEnabled) { instanceLayers.push_back(VALIDATION_LAYER_NAME); instanceExtensions.push_back("VK_EXT_debug_report"); @@ -767,8 +780,10 @@ int Player::InitVulkan() return RESULT_ERROR_VULKAN; } - if(g_EnableValidationLayer) + if(validationLayersEnabled) + { RegisterDebugCallbacks(); + } // Find physical device @@ -1981,8 +1996,10 @@ static void PrintCommandLineSyntax() " --PhysicalDevice - Choice of Vulkan physical device. Default: 0.\n" " --UserData - 0 to disable or 1 to enable setting pUserData during playback.\n" " Default is 1. Affects both creation of buffers and images, as well as calls to vmaSetAllocationUserData.\n" + " --VK_LAYER_LUNARG_standard_validation - 0 to disable or 1 to enable validation layers.\n" + " By default the layers are silently enabled if available.\n" " --VK_KHR_dedicated_allocation - 0 to disable or 1 to enable this extension.\n" - " By defalut the extension is silently enabled if available.\n" + " By default the extension is silently enabled if available.\n" ); } @@ -2140,6 +2157,7 @@ static int main2(int argc, char** argv) cmdLineParser.RegisterOpt(CMD_LINE_OPT_PHYSICAL_DEVICE, "PhysicalDevice", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_USER_DATA, "UserData", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, "VK_KHR_dedicated_allocation", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION, VALIDATION_LAYER_NAME, true); CmdLineParser::RESULT res; while((res = cmdLineParser.ReadNext()) != CmdLineParser::RESULT_END) @@ -2208,6 +2226,22 @@ static int main2(int argc, char** argv) } } break; + case CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION: + { + bool newValue; + if(StrRangeToBool(StrRange(cmdLineParser.GetParameter()), newValue)) + { + g_VK_LAYER_LUNARG_standard_validation = newValue ? + VULKAN_EXTENSION_REQUEST::ENABLED : + VULKAN_EXTENSION_REQUEST::DISABLED; + } + else + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + } + break; default: assert(0); } From a2d6395ec474d783492730f12a40bd5b1fe9412c Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 15:05:21 +0200 Subject: [PATCH 46/51] VmaReplay: Added parameter --MemStats and memory statistics. --- src/VmaReplay/VmaReplay.cpp | 167 +++++++++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 2 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index f9370cf..c714d6f 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -39,6 +39,7 @@ enum CMD_LINE_OPT CMD_LINE_OPT_USER_DATA, CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION, + CMD_LINE_OPT_MEM_STATS, }; static enum class VERBOSITY @@ -118,6 +119,7 @@ static size_t g_IterationCount = 1; static uint32_t g_PhysicalDeviceIndex = 0; static RangeSequence g_LineRanges; static bool g_UserDataEnabled = true; +static bool g_MemStatsEnabled = true; VULKAN_EXTENSION_REQUEST g_VK_KHR_dedicated_allocation_request = VULKAN_EXTENSION_REQUEST::DEFAULT; VULKAN_EXTENSION_REQUEST g_VK_LAYER_LUNARG_standard_validation = VULKAN_EXTENSION_REQUEST::DEFAULT; @@ -155,7 +157,9 @@ public: static uint32_t BufferUsageToClass(uint32_t usage); static uint32_t ImageUsageToClass(uint32_t usage); - Statistics() { } + Statistics(); + void Init(uint32_t memHeapCount, uint32_t memTypeCount); + void PrintMemStats() const; const size_t* GetFunctionCallCount() const { return m_FunctionCallCount; } size_t GetImageCreationCount(uint32_t imgClass) const { return m_ImageCreationCount[imgClass]; } @@ -170,13 +174,38 @@ public: void RegisterCreatePool(); void RegisterCreateAllocation(); + void UpdateMemStats(const VmaStats& currStats); + private: + uint32_t m_MemHeapCount = 0; + uint32_t m_MemTypeCount = 0; + size_t m_FunctionCallCount[(size_t)VMA_FUNCTION::Count] = {}; size_t m_ImageCreationCount[4] = { }; size_t m_LinearImageCreationCount = 0; size_t m_BufferCreationCount[4] = { }; size_t m_AllocationCreationCount = 0; // Also includes buffers and images, and lost allocations. size_t m_PoolCreationCount = 0; + + // Structure similar to VmaStatInfo, but not the same. + struct MemStatInfo + { + uint32_t blockCount; + uint32_t allocationCount; + uint32_t unusedRangeCount; + VkDeviceSize usedBytes; + VkDeviceSize unusedBytes; + VkDeviceSize totalBytes; + }; + struct MemStats + { + MemStatInfo memoryType[VK_MAX_MEMORY_TYPES]; + MemStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]; + MemStatInfo total; + } m_PeakMemStats; + + void UpdateMemStatInfo(MemStatInfo& inoutPeakInfo, const VmaStatInfo& currInfo); + static void PrintMemStatInfo(const MemStatInfo& info); }; uint32_t Statistics::BufferUsageToClass(uint32_t usage) @@ -242,6 +271,45 @@ uint32_t Statistics::ImageUsageToClass(uint32_t usage) } } +Statistics::Statistics() +{ + ZeroMemory(&m_PeakMemStats, sizeof(m_PeakMemStats)); +} + +void Statistics::Init(uint32_t memHeapCount, uint32_t memTypeCount) +{ + m_MemHeapCount = memHeapCount; + m_MemTypeCount = memTypeCount; +} + +void Statistics::PrintMemStats() const +{ + printf("Memory statistics:\n"); + + printf(" Total:\n"); + PrintMemStatInfo(m_PeakMemStats.total); + + for(uint32_t i = 0; i < m_MemHeapCount; ++i) + { + const MemStatInfo& info = m_PeakMemStats.memoryHeap[i]; + if(info.blockCount > 0 || info.totalBytes > 0) + { + printf(" Heap %u:\n", i); + PrintMemStatInfo(info); + } + } + + for(uint32_t i = 0; i < m_MemTypeCount; ++i) + { + const MemStatInfo& info = m_PeakMemStats.memoryType[i]; + if(info.blockCount > 0 || info.totalBytes > 0) + { + printf(" Type %u:\n", i); + PrintMemStatInfo(info); + } + } +} + void Statistics::RegisterFunctionCall(VMA_FUNCTION func) { ++m_FunctionCallCount[(size_t)func]; @@ -278,6 +346,51 @@ void Statistics::RegisterCreateAllocation() ++m_AllocationCreationCount; } +void Statistics::UpdateMemStats(const VmaStats& currStats) +{ + UpdateMemStatInfo(m_PeakMemStats.total, currStats.total); + + for(uint32_t i = 0; i < m_MemHeapCount; ++i) + { + UpdateMemStatInfo(m_PeakMemStats.memoryHeap[i], currStats.memoryHeap[i]); + } + + for(uint32_t i = 0; i < m_MemTypeCount; ++i) + { + UpdateMemStatInfo(m_PeakMemStats.memoryType[i], currStats.memoryType[i]); + } +} + +void Statistics::UpdateMemStatInfo(MemStatInfo& inoutPeakInfo, const VmaStatInfo& currInfo) +{ +#define SET_PEAK(inoutDst, src) \ + if((src) > (inoutDst)) \ + { \ + (inoutDst) = (src); \ + } + + SET_PEAK(inoutPeakInfo.blockCount, currInfo.blockCount); + SET_PEAK(inoutPeakInfo.allocationCount, currInfo.allocationCount); + SET_PEAK(inoutPeakInfo.unusedRangeCount, currInfo.unusedRangeCount); + SET_PEAK(inoutPeakInfo.usedBytes, currInfo.usedBytes); + SET_PEAK(inoutPeakInfo.unusedBytes, currInfo.unusedBytes); + SET_PEAK(inoutPeakInfo.totalBytes, currInfo.usedBytes + currInfo.unusedBytes); + +#undef SET_PEAK +} + +void Statistics::PrintMemStatInfo(const MemStatInfo& info) +{ + printf(" Peak blocks %u, allocations %u, unused ranges %u\n", + info.blockCount, + info.allocationCount, + info.unusedRangeCount); + printf(" Peak total bytes: %llu, used bytes %llu, unused bytes %llu\n", + info.totalBytes, + info.usedBytes, + info.unusedBytes); +} + //////////////////////////////////////////////////////////////////////////////// // class Player @@ -383,6 +496,7 @@ private: uint32_t m_GraphicsQueueFamilyIndex = UINT_MAX; VkDevice m_Device = VK_NULL_HANDLE; VmaAllocator m_Allocator = VK_NULL_HANDLE; + const VkPhysicalDeviceMemoryProperties* m_MemProps = nullptr; PFN_vkCreateDebugReportCallbackEXT m_pvkCreateDebugReportCallbackEXT; PFN_vkDebugReportMessageEXT m_pvkDebugReportMessageEXT; @@ -440,6 +554,8 @@ private: // If failed, prints warning, returns false, and sets allocCreateInfo.pUserData to null. bool PrepareUserData(size_t lineNumber, uint32_t allocCreateFlags, const StrRange& userDataColumn, const StrRange& wholeLine, void*& outUserData); + void UpdateMemStats(); + void ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit); void ExecuteSetAllocationUserData(size_t lineNumber, const CsvSplit& csvSplit); @@ -468,7 +584,15 @@ Player::Player() int Player::Init() { - return InitVulkan(); + int result = InitVulkan(); + + if(result == 0) + { + m_Stats.Init(m_MemProps->memoryHeapCount, m_MemProps->memoryTypeCount); + UpdateMemStats(); + } + + return result; } Player::~Player() @@ -940,6 +1064,8 @@ int Player::InitVulkan() return RESULT_ERROR_VULKAN; } + vmaGetMemoryProperties(m_Allocator, &m_MemProps); + return 0; } @@ -1129,6 +1255,11 @@ void Player::PrintStats() } } } + + if(g_MemStatsEnabled) + { + m_Stats.PrintMemStats(); + } } bool Player::ValidateFunctionParameterCount(size_t lineNumber, const CsvSplit& csvSplit, size_t expectedParamCount, bool lastUnbound) @@ -1187,6 +1318,18 @@ bool Player::PrepareUserData(size_t lineNumber, uint32_t allocCreateFlags, const return false; } +void Player::UpdateMemStats() +{ + if(!g_MemStatsEnabled) + { + return; + } + + VmaStats stats; + vmaCalculateStats(m_Allocator, &stats); + m_Stats.UpdateMemStats(stats); +} + void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) { m_Stats.RegisterFunctionCall(VMA_FUNCTION::CreatePool); @@ -1259,6 +1402,8 @@ void Player::ExecuteCreatePool(size_t lineNumber, const CsvSplit& csvSplit) } } } + + UpdateMemStats(); } else { @@ -1286,6 +1431,7 @@ void Player::ExecuteDestroyPool(size_t lineNumber, const CsvSplit& csvSplit) if(it != m_Pools.end()) { vmaDestroyPool(m_Allocator, it->second.pool); + UpdateMemStats(); m_Pools.erase(it); } else @@ -1395,6 +1541,7 @@ void Player::ExecuteCreateBuffer(size_t lineNumber, const CsvSplit& csvSplit) Allocation allocDesc = { }; allocDesc.allocationFlags = allocCreateInfo.flags; VkResult res = vmaCreateBuffer(m_Allocator, &bufCreateInfo, &allocCreateInfo, &allocDesc.buffer, &allocDesc.allocation, nullptr); + UpdateMemStats(); AddAllocation(lineNumber, origPtr, res, "vmaCreateBuffer", std::move(allocDesc)); } else @@ -1421,6 +1568,7 @@ void Player::DestroyAllocation(size_t lineNumber, const CsvSplit& csvSplit) if(it != m_Allocations.end()) { Destroy(it->second); + UpdateMemStats(); m_Allocations.erase(it); } else @@ -1491,6 +1639,7 @@ void Player::ExecuteCreateImage(size_t lineNumber, const CsvSplit& csvSplit) Allocation allocDesc = {}; allocDesc.allocationFlags = allocCreateInfo.flags; VkResult res = vmaCreateImage(m_Allocator, &imageCreateInfo, &allocCreateInfo, &allocDesc.image, &allocDesc.allocation, nullptr); + UpdateMemStats(); AddAllocation(lineNumber, origPtr, res, "vmaCreateImage", std::move(allocDesc)); } else @@ -1515,6 +1664,7 @@ void Player::ExecuteCreateLostAllocation(size_t lineNumber, const CsvSplit& csvS { Allocation allocDesc = {}; vmaCreateLostAllocation(m_Allocator, &allocDesc.allocation); + UpdateMemStats(); m_Stats.RegisterCreateAllocation(); AddAllocation(lineNumber, origPtr, VK_SUCCESS, "vmaCreateLostAllocation", std::move(allocDesc)); @@ -1563,6 +1713,7 @@ void Player::ExecuteAllocateMemory(size_t lineNumber, const CsvSplit& csvSplit) allocCreateInfo.pUserData); } + UpdateMemStats(); m_Stats.RegisterCreateAllocation(); Allocation allocDesc = {}; @@ -1641,6 +1792,7 @@ void Player::ExecuteAllocateMemoryForBufferOrImage(size_t lineNumber, const CsvS m_AllocateForBufferImageWarningIssued = true; } + UpdateMemStats(); m_Stats.RegisterCreateAllocation(); Allocation allocDesc = {}; @@ -1956,6 +2108,7 @@ void Player::ExecuteMakePoolAllocationsLost(size_t lineNumber, const CsvSplit& c if(it != m_Pools.end()) { vmaMakePoolAllocationsLost(m_Allocator, it->second.pool, nullptr); + UpdateMemStats(); } else { @@ -1991,6 +2144,8 @@ static void PrintCommandLineSyntax() " 2 - Maximum verbosity. Prints a lot of information.\n" " -i - Repeat playback given number of times (iterations)\n" " Default is 1. Vulkan is reinitialized with every iteration.\n" + " --MemStats - 0 to disable or 1 to enable memory statistics.\n" + " Default is 1. Enabling it may negatively impact playback performance.\n" " --Lines - Replay only limited set of lines from file\n" " Ranges is comma-separated list of ranges, e.g. \"-10,15,18-25,31-\".\n" " --PhysicalDevice - Choice of Vulkan physical device. Default: 0.\n" @@ -2158,6 +2313,7 @@ static int main2(int argc, char** argv) cmdLineParser.RegisterOpt(CMD_LINE_OPT_USER_DATA, "UserData", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, "VK_KHR_dedicated_allocation", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION, VALIDATION_LAYER_NAME, true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_MEM_STATS, "MemStats", true); CmdLineParser::RESULT res; while((res = cmdLineParser.ReadNext()) != CmdLineParser::RESULT_END) @@ -2242,6 +2398,13 @@ static int main2(int argc, char** argv) } } break; + case CMD_LINE_OPT_MEM_STATS: + if(!StrRangeToBool(StrRange(cmdLineParser.GetParameter()), g_MemStatsEnabled)) + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + break; default: assert(0); } From ac3c67fd5dec2cceb71748f8e6094466a3566e7b Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 16:18:42 +0200 Subject: [PATCH 47/51] Added parameters: --DumpStatsAfterLine, --DumpDetailedStatsAfterLine, to dump VMA stats to JSON file. Set --MemStats default to 0. --- src/VmaReplay/VmaReplay.cpp | 100 ++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index c714d6f..a71e5a6 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -40,6 +40,8 @@ enum CMD_LINE_OPT CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION, CMD_LINE_OPT_MEM_STATS, + CMD_LINE_OPT_DUMP_STATS_AFTER_LINE, + CMD_LINE_OPT_DUMP_DETAILED_STATS_AFTER_LINE, }; static enum class VERBOSITY @@ -119,10 +121,21 @@ static size_t g_IterationCount = 1; static uint32_t g_PhysicalDeviceIndex = 0; static RangeSequence g_LineRanges; static bool g_UserDataEnabled = true; -static bool g_MemStatsEnabled = true; +static bool g_MemStatsEnabled = false; VULKAN_EXTENSION_REQUEST g_VK_KHR_dedicated_allocation_request = VULKAN_EXTENSION_REQUEST::DEFAULT; VULKAN_EXTENSION_REQUEST g_VK_LAYER_LUNARG_standard_validation = VULKAN_EXTENSION_REQUEST::DEFAULT; +struct StatsAfterLineEntry +{ + size_t line; + bool detailed; + + bool operator<(const StatsAfterLineEntry& rhs) const { return line < rhs.line; } + bool operator==(const StatsAfterLineEntry& rhs) const { return line == rhs.line; } +}; +static std::vector g_DumpStatsAfterLine; +static size_t g_DumpStatsAfterLineNextIndex = 0; + static bool ValidateFileVersion() { const uint32_t major = g_FileVersion >> 16; @@ -483,6 +496,8 @@ public: ~Player(); void ExecuteLine(size_t lineNumber, const StrRange& line); + void DumpStats(const char* fileNameFormat, size_t lineNumber, bool detailed); + void PrintStats(); private: @@ -727,6 +742,29 @@ void Player::ExecuteLine(size_t lineNumber, const StrRange& line) } } +void Player::DumpStats(const char* fileNameFormat, size_t lineNumber, bool detailed) +{ + char* pStatsString = nullptr; + vmaBuildStatsString(m_Allocator, &pStatsString, detailed ? VK_TRUE : VK_FALSE); + + char fileName[MAX_PATH]; + sprintf_s(fileName, fileNameFormat, lineNumber); + + FILE* file = nullptr; + errno_t err = fopen_s(&file, fileName, "wb"); + if(err == 0) + { + fwrite(pStatsString, 1, strlen(pStatsString), file); + fclose(file); + } + else + { + printf("ERROR: Failed to write file: %s\n", fileName); + } + + vmaFreeStatsString(m_Allocator, pStatsString); +} + void Player::Destroy(const Allocation& alloc) { if(alloc.buffer) @@ -2145,7 +2183,11 @@ static void PrintCommandLineSyntax() " -i - Repeat playback given number of times (iterations)\n" " Default is 1. Vulkan is reinitialized with every iteration.\n" " --MemStats - 0 to disable or 1 to enable memory statistics.\n" - " Default is 1. Enabling it may negatively impact playback performance.\n" + " Default is 0. Enabling it may negatively impact playback performance.\n" + " --DumpStatsAfterLine - Dump VMA statistics to JSON file after specified source file line finishes execution.\n" + " File is written to current directory with name: VmaReplay_Line####.json.\n" + " This parameter can be repeated.\n" + " --DumpDetailedStatsAfterLine - Like command above, but includes detailed map.\n" " --Lines - Replay only limited set of lines from file\n" " Ranges is comma-separated list of ranges, e.g. \"-10,15,18-25,31-\".\n" " --PhysicalDevice - Choice of Vulkan physical device. Default: 0.\n" @@ -2163,6 +2205,7 @@ static int ProcessFile(size_t iterationIndex, const char* data, size_t numBytes, outDuration = duration::max(); const bool useLineRanges = !g_LineRanges.IsEmpty(); + const bool useDumpStatsAfterLine = !g_DumpStatsAfterLine.empty(); LineSplit lineSplit(data, numBytes); StrRange line; @@ -2206,17 +2249,39 @@ static int ProcessFile(size_t iterationIndex, const char* data, size_t numBytes, while(lineSplit.GetNextLine(line)) { + const size_t currLineNumber = lineSplit.GetNextLineIndex(); + bool execute = true; if(useLineRanges) { - execute = g_LineRanges.Includes(lineSplit.GetNextLineIndex()); + execute = g_LineRanges.Includes(currLineNumber); } if(execute) { - player.ExecuteLine(lineSplit.GetNextLineIndex(), line); + player.ExecuteLine(currLineNumber, line); ++executedLineCount; } + + while(useDumpStatsAfterLine && + g_DumpStatsAfterLineNextIndex < g_DumpStatsAfterLine.size() && + currLineNumber >= g_DumpStatsAfterLine[g_DumpStatsAfterLineNextIndex].line) + { + const size_t requestedLine = g_DumpStatsAfterLine[g_DumpStatsAfterLineNextIndex].line; + const bool detailed = g_DumpStatsAfterLine[g_DumpStatsAfterLineNextIndex].detailed; + + if(g_Verbosity == VERBOSITY::MAXIMUM) + { + printf("Dumping %sstats after line %zu actual line %zu...\n", + detailed ? "detailed " : "", + requestedLine, + currLineNumber); + } + + player.DumpStats("VmaReplay_Line%04zu.json", requestedLine, detailed); + + ++g_DumpStatsAfterLineNextIndex; + } } const duration playDuration = std::chrono::high_resolution_clock::now() - timeBeg; @@ -2314,6 +2379,8 @@ static int main2(int argc, char** argv) cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_KHR_DEDICATED_ALLOCATION, "VK_KHR_dedicated_allocation", true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_VK_LAYER_LUNARG_STANDARD_VALIDATION, VALIDATION_LAYER_NAME, true); cmdLineParser.RegisterOpt(CMD_LINE_OPT_MEM_STATS, "MemStats", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_DUMP_STATS_AFTER_LINE, "DumpStatsAfterLine", true); + cmdLineParser.RegisterOpt(CMD_LINE_OPT_DUMP_DETAILED_STATS_AFTER_LINE, "DumpDetailedStatsAfterLine", true); CmdLineParser::RESULT res; while((res = cmdLineParser.ReadNext()) != CmdLineParser::RESULT_END) @@ -2405,6 +2472,23 @@ static int main2(int argc, char** argv) return RESULT_ERROR_COMMAND_LINE; } break; + case CMD_LINE_OPT_DUMP_STATS_AFTER_LINE: + case CMD_LINE_OPT_DUMP_DETAILED_STATS_AFTER_LINE: + { + size_t line; + if(StrRangeToUint(StrRange(cmdLineParser.GetParameter()), line)) + { + const bool detailed = + cmdLineParser.GetOptId() == CMD_LINE_OPT_DUMP_DETAILED_STATS_AFTER_LINE; + g_DumpStatsAfterLine.push_back({line, detailed}); + } + else + { + PrintCommandLineSyntax(); + return RESULT_ERROR_COMMAND_LINE; + } + } + break; default: assert(0); } @@ -2429,12 +2513,20 @@ static int main2(int argc, char** argv) } } + // Postprocess command line parameters. + if(g_FilePath.empty()) { PrintCommandLineSyntax(); return RESULT_ERROR_COMMAND_LINE; } + // Sort g_DumpStatsAfterLine and make unique. + std::sort(g_DumpStatsAfterLine.begin(), g_DumpStatsAfterLine.end()); + g_DumpStatsAfterLine.erase( + std::unique(g_DumpStatsAfterLine.begin(), g_DumpStatsAfterLine.end()), + g_DumpStatsAfterLine.end()); + return ProcessFile(); } From cb4e8da23b71037b0ed8c2207aa94f3be217c090 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 21 Aug 2018 17:56:25 +0200 Subject: [PATCH 48/51] Updated file format to 1,3. Added configuration section. --- docs/Recording file format.md | 85 ++++++- src/VmaReplay/Common.h | 2 +- src/VmaReplay/VmaReplay.cpp | 420 +++++++++++++++++++++++++++++++++- src/vk_mem_alloc.h | 56 ++++- 4 files changed, 549 insertions(+), 14 deletions(-) diff --git a/docs/Recording file format.md b/docs/Recording file format.md index d6c2878..b01eaf4 100644 --- a/docs/Recording file format.md +++ b/docs/Recording file format.md @@ -23,7 +23,49 @@ Formats with only minor version incremented are backward compatible. VmaReplay application supports all older versions. Current version is: - 1,2 + 1,3 + +# Configuration + +Header is followed by mandatory configuration section (min format version 1.3). It starts with line: + + Config,Begin + +And ends with line: + + Config,End + +Between them there can be zero or more lines with configuration options. They store values of various variables from the current environment from the time of recording, like properties and limits of Vulkan physical device, available memory heaps and types, enabled Vulkan extensions, as well macros that configure VMA internals. Supported configuration options are: + + PhysicalDevice,apiVersion, + PhysicalDevice,driverVersion, + PhysicalDevice,vendorID, + PhysicalDevice,deviceID, + PhysicalDevice,deviceType, + PhysicalDevice,deviceName, + + PhysicalDeviceLimits,maxMemoryAllocationCount, + PhysicalDeviceLimits,bufferImageGranularity, + PhysicalDeviceLimits,nonCoherentAtomSize, + + PhysicalDeviceMemory,HeapCount, + PhysicalDeviceMemory,Heap,,size, + PhysicalDeviceMemory,Heap,,flags, + PhysicalDeviceMemory,TypeCount, + PhysicalDeviceMemory,Type,,heapIndex, + PhysicalDeviceMemory,Type,,propertyFlags, + + Extension,VK_KHR_dedicated_allocation, + + Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY, + Macro,VMA_DEBUG_ALIGNMENT, + Macro,VMA_DEBUG_MARGIN, + Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS, + Macro,VMA_DEBUG_DETECT_CORRUPTION, + Macro,VMA_DEBUG_GLOBAL_MUTEX, + Macro,VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY, + Macro,VMA_SMALL_HEAP_MAX_SIZE, + Macro,VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE, # Function calls @@ -162,7 +204,7 @@ No parameters. - pool : pointer -## Data types +# Data types **bool** @@ -186,7 +228,44 @@ It should not contain end-of-line characters - results are then undefined. # Example file Vulkan Memory Allocator,Calls recording - 1,2 + 1,3 + Config,Begin + PhysicalDevice,apiVersion,4198477 + PhysicalDevice,driverVersion,8388653 + PhysicalDevice,vendorID,4098 + PhysicalDevice,deviceID,26751 + PhysicalDevice,deviceType,2 + PhysicalDevice,deviceName,Radeon RX Vega + PhysicalDeviceLimits,maxMemoryAllocationCount,4096 + PhysicalDeviceLimits,bufferImageGranularity,1 + PhysicalDeviceLimits,nonCoherentAtomSize,128 + PhysicalDeviceMemory,HeapCount,3 + PhysicalDeviceMemory,Heap,0,size,8304721920 + PhysicalDeviceMemory,Heap,0,flags,3 + PhysicalDeviceMemory,Heap,1,size,8286175232 + PhysicalDeviceMemory,Heap,1,flags,0 + PhysicalDeviceMemory,Heap,2,size,268435456 + PhysicalDeviceMemory,Heap,2,flags,3 + PhysicalDeviceMemory,TypeCount,4 + PhysicalDeviceMemory,Type,0,heapIndex,0 + PhysicalDeviceMemory,Type,0,propertyFlags,1 + PhysicalDeviceMemory,Type,1,heapIndex,1 + PhysicalDeviceMemory,Type,1,propertyFlags,6 + PhysicalDeviceMemory,Type,2,heapIndex,2 + PhysicalDeviceMemory,Type,2,propertyFlags,7 + PhysicalDeviceMemory,Type,3,heapIndex,1 + PhysicalDeviceMemory,Type,3,propertyFlags,14 + Extension,VK_KHR_dedicated_allocation,1 + Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,0 + Macro,VMA_DEBUG_ALIGNMENT,1 + Macro,VMA_DEBUG_MARGIN,0 + Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS,0 + Macro,VMA_DEBUG_DETECT_CORRUPTION,0 + Macro,VMA_DEBUG_GLOBAL_MUTEX,0 + Macro,VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY,1 + Macro,VMA_SMALL_HEAP_MAX_SIZE,1073741824 + Macro,VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE,268435456 + Config,End 12552,0.000,0,vmaCreateAllocator 12552,0.010,0,vmaCreateImage,0,1,37,128,128,1,1,1,1,1,1,0,8,36,2,0,0,0,0000000000000000,000001D85B8B1A80, 12552,0.010,0,vmaSetAllocationUserData,000001D85B8B1A80,Ala ma kota diff --git a/src/VmaReplay/Common.h b/src/VmaReplay/Common.h index e4b6251..28b5bbc 100644 --- a/src/VmaReplay/Common.h +++ b/src/VmaReplay/Common.h @@ -53,7 +53,7 @@ struct StrRange explicit StrRange(const std::string& s) : beg(s.data()), end(s.data() + s.length()) { } size_t length() const { return end - beg; } - void to_str(std::string& out) { out.assign(beg, end); } + void to_str(std::string& out) const { out.assign(beg, end); } }; inline bool StrRangeEq(const StrRange& lhs, const char* rhsSz) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index a71e5a6..feb5005 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -117,6 +117,10 @@ static std::string g_FilePath; // Most significant 16 bits are major version, least significant 16 bits are minor version. static uint32_t g_FileVersion; +inline uint32_t MakeVersion(uint32_t major, uint32_t minor) { return (major << 16) | minor; } +inline uint32_t GetVersionMajor(uint32_t version) { return version >> 16; } +inline uint32_t GetVersionMinor(uint32_t version) { return version & 0xFFFF; } + static size_t g_IterationCount = 1; static uint32_t g_PhysicalDeviceIndex = 0; static RangeSequence g_LineRanges; @@ -138,9 +142,13 @@ static size_t g_DumpStatsAfterLineNextIndex = 0; static bool ValidateFileVersion() { - const uint32_t major = g_FileVersion >> 16; - const uint32_t minor = g_FileVersion & 0xFFFF; - return major == 1 && minor <= 2; + if(GetVersionMajor(g_FileVersion) == 1 && + GetVersionMinor(g_FileVersion) <= 3) + { + return true; + } + + return false; } static bool ParseFileVersion(const StrRange& s) @@ -404,6 +412,373 @@ void Statistics::PrintMemStatInfo(const MemStatInfo& info) info.unusedBytes); } +//////////////////////////////////////////////////////////////////////////////// +// class ConfigurationParser + +class ConfigurationParser +{ +public: + ConfigurationParser(); + + bool Parse(LineSplit& lineSplit); + + void Compare( + const VkPhysicalDeviceProperties& devProps, + const VkPhysicalDeviceMemoryProperties& memProps, + bool dedicatedAllocationExtensionEnabled); + +private: + enum class OPTION + { + PhysicalDevice_apiVersion, + PhysicalDevice_driverVersion, + PhysicalDevice_vendorID, + PhysicalDevice_deviceID, + PhysicalDevice_deviceType, + PhysicalDevice_deviceName, + PhysicalDeviceLimits_maxMemoryAllocationCount, + PhysicalDeviceLimits_bufferImageGranularity, + PhysicalDeviceLimits_nonCoherentAtomSize, + Extension_VK_KHR_dedicated_allocation, + Macro_VMA_DEBUG_ALWAYS_DEDICATED_MEMORY, + Macro_VMA_DEBUG_ALIGNMENT, + Macro_VMA_DEBUG_MARGIN, + Macro_VMA_DEBUG_INITIALIZE_ALLOCATIONS, + Macro_VMA_DEBUG_DETECT_CORRUPTION, + Macro_VMA_DEBUG_GLOBAL_MUTEX, + Macro_VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY, + Macro_VMA_SMALL_HEAP_MAX_SIZE, + Macro_VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE, + Count + }; + + std::vector m_OptionSet; + std::vector m_OptionValue; + VkPhysicalDeviceMemoryProperties m_MemProps; + + bool m_WarningHeaderPrinted = false; + + void SetOption( + size_t lineNumber, + OPTION option, + const StrRange& str); + void EnsureWarningHeader(); + void CompareOption(VERBOSITY minVerbosity, const char* name, + OPTION option, uint32_t currValue); + void CompareOption(VERBOSITY minVerbosity, const char* name, + OPTION option, uint64_t currValue); + void CompareOption(VERBOSITY minVerbosity, const char* name, + OPTION option, bool currValue); + void CompareOption(VERBOSITY minVerbosity, const char* name, + OPTION option, const char* currValue); +}; + +ConfigurationParser::ConfigurationParser() : + m_OptionSet((size_t)OPTION::Count), + m_OptionValue((size_t)OPTION::Count) +{ + ZeroMemory(&m_MemProps, sizeof(m_MemProps)); +} + +bool ConfigurationParser::Parse(LineSplit& lineSplit) +{ + for(auto& it : m_OptionSet) + { + it = false; + } + for(auto& it : m_OptionValue) + { + it.clear(); + } + + StrRange line; + + if(!lineSplit.GetNextLine(line) && !StrRangeEq(line, "Config,Begin")) + { + return false; + } + + CsvSplit csvSplit; + while(lineSplit.GetNextLine(line)) + { + if(StrRangeEq(line, "Config,End")) + { + break; + } + + const size_t currLineNumber = lineSplit.GetNextLineIndex(); + + csvSplit.Set(line); + if(csvSplit.GetCount() == 0) + { + return false; + } + + const StrRange optionName = csvSplit.GetRange(0); + if(StrRangeEq(optionName, "PhysicalDevice")) + { + if(csvSplit.GetCount() >= 3) + { + const StrRange subOptionName = csvSplit.GetRange(1); + if(StrRangeEq(subOptionName, "apiVersion")) + SetOption(currLineNumber, OPTION::PhysicalDevice_apiVersion, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "driverVersion")) + SetOption(currLineNumber, OPTION::PhysicalDevice_driverVersion, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "vendorID")) + SetOption(currLineNumber, OPTION::PhysicalDevice_vendorID, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "deviceID")) + SetOption(currLineNumber, OPTION::PhysicalDevice_deviceID, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "deviceType")) + SetOption(currLineNumber, OPTION::PhysicalDevice_deviceType, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "deviceName")) + SetOption(currLineNumber, OPTION::PhysicalDevice_deviceName, StrRange(csvSplit.GetRange(2).beg, line.end)); + else + printf("Line %zu: Unrecognized configuration option.\n", currLineNumber); + } + else + printf("Line %zu: Too few columns.\n", currLineNumber); + } + else if(StrRangeEq(optionName, "PhysicalDeviceLimits")) + { + if(csvSplit.GetCount() >= 3) + { + const StrRange subOptionName = csvSplit.GetRange(1); + if(StrRangeEq(subOptionName, "maxMemoryAllocationCount")) + SetOption(currLineNumber, OPTION::PhysicalDeviceLimits_maxMemoryAllocationCount, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "bufferImageGranularity")) + SetOption(currLineNumber, OPTION::PhysicalDeviceLimits_bufferImageGranularity, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "nonCoherentAtomSize")) + SetOption(currLineNumber, OPTION::PhysicalDeviceLimits_nonCoherentAtomSize, csvSplit.GetRange(2)); + else + printf("Line %zu: Unrecognized configuration option.\n", currLineNumber); + } + else + printf("Line %zu: Too few columns.\n", currLineNumber); + } + else if(StrRangeEq(optionName, "Extension")) + { + if(csvSplit.GetCount() >= 3) + { + const StrRange subOptionName = csvSplit.GetRange(1); + if(StrRangeEq(subOptionName, "VK_KHR_dedicated_allocation")) + SetOption(currLineNumber, OPTION::Extension_VK_KHR_dedicated_allocation, csvSplit.GetRange(2)); + else + printf("Line %zu: Unrecognized configuration option.\n", currLineNumber); + } + else + printf("Line %zu: Too few columns.\n", currLineNumber); + } + else if(StrRangeEq(optionName, "Macro")) + { + if(csvSplit.GetCount() >= 3) + { + const StrRange subOptionName = csvSplit.GetRange(1); + if(StrRangeEq(subOptionName, "VMA_DEBUG_ALWAYS_DEDICATED_MEMORY")) + SetOption(currLineNumber, OPTION::Macro_VMA_DEBUG_ALWAYS_DEDICATED_MEMORY, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VMA_DEBUG_ALIGNMENT")) + SetOption(currLineNumber, OPTION::Macro_VMA_DEBUG_ALIGNMENT, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VMA_DEBUG_MARGIN")) + SetOption(currLineNumber, OPTION::Macro_VMA_DEBUG_MARGIN, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VMA_DEBUG_INITIALIZE_ALLOCATIONS")) + SetOption(currLineNumber, OPTION::Macro_VMA_DEBUG_INITIALIZE_ALLOCATIONS, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VMA_DEBUG_DETECT_CORRUPTION")) + SetOption(currLineNumber, OPTION::Macro_VMA_DEBUG_DETECT_CORRUPTION, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VMA_DEBUG_GLOBAL_MUTEX")) + SetOption(currLineNumber, OPTION::Macro_VMA_DEBUG_GLOBAL_MUTEX, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY")) + SetOption(currLineNumber, OPTION::Macro_VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VMA_SMALL_HEAP_MAX_SIZE")) + SetOption(currLineNumber, OPTION::Macro_VMA_SMALL_HEAP_MAX_SIZE, csvSplit.GetRange(2)); + else if(StrRangeEq(subOptionName, "VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE")) + SetOption(currLineNumber, OPTION::Macro_VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE, csvSplit.GetRange(2)); + else + printf("Line %zu: Unrecognized configuration option.\n", currLineNumber); + } + else + printf("Line %zu: Too few columns.\n", currLineNumber); + } + else if(StrRangeEq(optionName, "PhysicalDeviceMemory")) + { + uint32_t value = 0; + if(csvSplit.GetCount() == 3 && StrRangeEq(csvSplit.GetRange(1), "HeapCount") && + StrRangeToUint(csvSplit.GetRange(2), value)) + { + m_MemProps.memoryHeapCount = value; + } + else if(csvSplit.GetCount() == 3 && StrRangeEq(csvSplit.GetRange(1), "TypeCount") && + StrRangeToUint(csvSplit.GetRange(2), value)) + { + m_MemProps.memoryTypeCount = value; + } + else if(csvSplit.GetCount() == 5 && StrRangeEq(csvSplit.GetRange(1), "Heap") && + StrRangeToUint(csvSplit.GetRange(2), value) && + value < m_MemProps.memoryHeapCount) + { + if(StrRangeEq(csvSplit.GetRange(3), "size") && + StrRangeToUint(csvSplit.GetRange(4), m_MemProps.memoryHeaps[value].size)) + { + // Parsed. + } + else if(StrRangeEq(csvSplit.GetRange(3), "flags") && + StrRangeToUint(csvSplit.GetRange(4), m_MemProps.memoryHeaps[value].flags)) + { + // Parsed. + } + else + printf("Line %zu: Invalid configuration option.\n", currLineNumber); + } + else if(csvSplit.GetCount() == 5 && StrRangeEq(csvSplit.GetRange(1), "Type") && + StrRangeToUint(csvSplit.GetRange(2), value) && + value < m_MemProps.memoryTypeCount) + { + if(StrRangeEq(csvSplit.GetRange(3), "heapIndex") && + StrRangeToUint(csvSplit.GetRange(4), m_MemProps.memoryTypes[value].heapIndex)) + { + // Parsed. + } + else if(StrRangeEq(csvSplit.GetRange(3), "propertyFlags") && + StrRangeToUint(csvSplit.GetRange(4), m_MemProps.memoryTypes[value].propertyFlags)) + { + // Parsed. + } + else + printf("Line %zu: Invalid configuration option.\n", currLineNumber); + } + else + printf("Line %zu: Invalid configuration option.\n", currLineNumber); + } + else + printf("Line %zu: Unrecognized configuration option.\n", currLineNumber); + } + + return true; +} + +void ConfigurationParser::Compare( + const VkPhysicalDeviceProperties& devProps, + const VkPhysicalDeviceMemoryProperties& memProps, + bool dedicatedAllocationExtensionEnabled) +{ + CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice apiVersion", + OPTION::PhysicalDevice_apiVersion, devProps.apiVersion); + CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice driverVersion", + OPTION::PhysicalDevice_driverVersion, devProps.driverVersion); + CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice vendorID", + OPTION::PhysicalDevice_vendorID, devProps.vendorID); + CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice deviceID", + OPTION::PhysicalDevice_deviceID, devProps.deviceID); + CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice deviceType", + OPTION::PhysicalDevice_deviceType, (uint32_t)devProps.deviceType); + CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice deviceName", + OPTION::PhysicalDevice_deviceName, devProps.deviceName); + + CompareOption(VERBOSITY::DEFAULT, "PhysicalDeviceLimits maxMemoryAllocationCount", + OPTION::PhysicalDeviceLimits_maxMemoryAllocationCount, devProps.limits.maxMemoryAllocationCount); + CompareOption(VERBOSITY::DEFAULT, "PhysicalDeviceLimits bufferImageGranularity", + OPTION::PhysicalDeviceLimits_bufferImageGranularity, devProps.limits.bufferImageGranularity); + CompareOption(VERBOSITY::DEFAULT, "PhysicalDeviceLimits nonCoherentAtomSize", + OPTION::PhysicalDeviceLimits_nonCoherentAtomSize, devProps.limits.nonCoherentAtomSize); + CompareOption(VERBOSITY::DEFAULT, "Extension VK_KHR_dedicated_allocation", + OPTION::Extension_VK_KHR_dedicated_allocation, dedicatedAllocationExtensionEnabled); +} + +void ConfigurationParser::SetOption( + size_t lineNumber, + OPTION option, + const StrRange& str) +{ + if(m_OptionSet[(size_t)option]) + { + printf("Line %zu: Option already specified.\n" ,lineNumber); + } + + m_OptionSet[(size_t)option] = true; + + std::string val; + str.to_str(val); + m_OptionValue[(size_t)option] = std::move(val); +} + +void ConfigurationParser::EnsureWarningHeader() +{ + if(!m_WarningHeaderPrinted) + { + printf("WARNING: Following configuration parameters don't match:\n"); + m_WarningHeaderPrinted = true; + } +} + +void ConfigurationParser::CompareOption(VERBOSITY minVerbosity, const char* name, + OPTION option, uint32_t currValue) +{ + if(m_OptionSet[(size_t)option] && + g_Verbosity >= minVerbosity) + { + uint32_t origValue; + if(StrRangeToUint(StrRange(m_OptionValue[(size_t)option]), origValue)) + { + if(origValue != currValue) + { + EnsureWarningHeader(); + printf(" %s: original %u, current %u\n", name, origValue, currValue); + } + } + } +} + +void ConfigurationParser::CompareOption(VERBOSITY minVerbosity, const char* name, + OPTION option, uint64_t currValue) +{ + if(m_OptionSet[(size_t)option] && + g_Verbosity >= minVerbosity) + { + uint64_t origValue; + if(StrRangeToUint(StrRange(m_OptionValue[(size_t)option]), origValue)) + { + if(origValue != currValue) + { + EnsureWarningHeader(); + printf(" %s: original %llu, current %llu\n", name, origValue, currValue); + } + } + } +} + +void ConfigurationParser::CompareOption(VERBOSITY minVerbosity, const char* name, + OPTION option, bool currValue) +{ + if(m_OptionSet[(size_t)option] && + g_Verbosity >= minVerbosity) + { + bool origValue; + if(StrRangeToBool(StrRange(m_OptionValue[(size_t)option]), origValue)) + { + if(origValue != currValue) + { + EnsureWarningHeader(); + printf(" %s: original %u, current %u\n", name, + origValue ? 1 : 0, + currValue ? 1 : 0); + } + } + } +} + +void ConfigurationParser::CompareOption(VERBOSITY minVerbosity, const char* name, + OPTION option, const char* currValue) +{ + if(m_OptionSet[(size_t)option] && + g_Verbosity >= minVerbosity) + { + const std::string& origValue = m_OptionValue[(size_t)option]; + if(origValue != currValue) + { + EnsureWarningHeader(); + printf(" %s: original \"%s\", current \"%s\"\n", name, origValue.c_str(), currValue); + } + } +} + //////////////////////////////////////////////////////////////////////////////// // class Player @@ -495,6 +870,7 @@ public: int Init(); ~Player(); + void ApplyConfig(ConfigurationParser& configParser); void ExecuteLine(size_t lineNumber, const StrRange& line); void DumpStats(const char* fileNameFormat, size_t lineNumber, bool detailed); @@ -511,6 +887,8 @@ private: uint32_t m_GraphicsQueueFamilyIndex = UINT_MAX; VkDevice m_Device = VK_NULL_HANDLE; VmaAllocator m_Allocator = VK_NULL_HANDLE; + bool m_DedicatedAllocationEnabled = false; + const VkPhysicalDeviceProperties* m_DevProps = nullptr; const VkPhysicalDeviceMemoryProperties* m_MemProps = nullptr; PFN_vkCreateDebugReportCallbackEXT m_pvkCreateDebugReportCallbackEXT; @@ -618,6 +996,11 @@ Player::~Player() printf("WARNING: %zu more warnings not shown.\n", m_WarningCount - MAX_WARNINGS_TO_SHOW); } +void Player::ApplyConfig(ConfigurationParser& configParser) +{ + configParser.Compare(*m_DevProps, *m_MemProps, m_DedicatedAllocationEnabled); +} + void Player::ExecuteLine(size_t lineNumber, const StrRange& line) { CsvSplit csvSplit; @@ -1046,16 +1429,15 @@ int Player::InitVulkan() const bool dedicatedAllocationAvailable = VK_KHR_get_memory_requirements2_available && VK_KHR_dedicated_allocation_available; - bool dedicatedAllocationEnabled = false; switch(g_VK_KHR_dedicated_allocation_request) { case VULKAN_EXTENSION_REQUEST::DISABLED: break; case VULKAN_EXTENSION_REQUEST::DEFAULT: - dedicatedAllocationEnabled = dedicatedAllocationAvailable; + m_DedicatedAllocationEnabled = dedicatedAllocationAvailable; break; case VULKAN_EXTENSION_REQUEST::ENABLED: - dedicatedAllocationEnabled = dedicatedAllocationAvailable; + m_DedicatedAllocationEnabled = dedicatedAllocationAvailable; if(!dedicatedAllocationAvailable) { printf("WARNING: VK_KHR_dedicated_allocation extension cannot be enabled.\n"); @@ -1064,7 +1446,7 @@ int Player::InitVulkan() default: assert(0); } - if(dedicatedAllocationEnabled) + if(m_DedicatedAllocationEnabled) { enabledDeviceExtensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME); enabledDeviceExtensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); @@ -1090,7 +1472,7 @@ int Player::InitVulkan() allocatorInfo.physicalDevice = m_PhysicalDevice; allocatorInfo.device = m_Device; - if(dedicatedAllocationEnabled) + if(m_DedicatedAllocationEnabled) { allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT; } @@ -1102,6 +1484,7 @@ int Player::InitVulkan() return RESULT_ERROR_VULKAN; } + vmaGetPhysicalDeviceProperties(m_Allocator, &m_DevProps); vmaGetMemoryProperties(m_Allocator, &m_MemProps); return 0; @@ -2225,11 +2608,30 @@ static int ProcessFile(size_t iterationIndex, const char* data, size_t numBytes, if(g_Verbosity == VERBOSITY::MAXIMUM) { - printf("Format version: %u,%u\n", g_FileVersion >> 16, g_FileVersion & 0xFFFF); + printf("Format version: %u,%u\n", + GetVersionMajor(g_FileVersion), + GetVersionMinor(g_FileVersion)); + } + + // Parse configuration + const bool configEnabled = g_FileVersion >= MakeVersion(1, 3); + ConfigurationParser configParser; + if(configEnabled) + { + if(!configParser.Parse(lineSplit)) + { + return RESULT_ERROR_FORMAT; + } } Player player; int result = player.Init(); + + if(configEnabled) + { + player.ApplyConfig(configParser); + } + size_t executedLineCount = 0; if(result == 0) { diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 479b4b4..a908c90 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -4799,6 +4799,10 @@ class VmaRecorder public: VmaRecorder(); VkResult Init(const VmaRecordSettings& settings, bool useMutex); + void WriteConfiguration( + const VkPhysicalDeviceProperties& devProps, + const VkPhysicalDeviceMemoryProperties& memProps, + bool dedicatedAllocationExtensionEnabled); ~VmaRecorder(); void RecordCreateAllocator(uint32_t frameIndex); @@ -8163,7 +8167,7 @@ VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex) // Write header. fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording"); - fprintf(m_File, "%s\n", "1,2"); + fprintf(m_File, "%s\n", "1,3"); return VK_SUCCESS; } @@ -8541,6 +8545,52 @@ VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, } } +void VmaRecorder::WriteConfiguration( + const VkPhysicalDeviceProperties& devProps, + const VkPhysicalDeviceMemoryProperties& memProps, + bool dedicatedAllocationExtensionEnabled) +{ + fprintf(m_File, "Config,Begin\n"); + + fprintf(m_File, "PhysicalDevice,apiVersion,%u\n", devProps.apiVersion); + fprintf(m_File, "PhysicalDevice,driverVersion,%u\n", devProps.driverVersion); + fprintf(m_File, "PhysicalDevice,vendorID,%u\n", devProps.vendorID); + fprintf(m_File, "PhysicalDevice,deviceID,%u\n", devProps.deviceID); + fprintf(m_File, "PhysicalDevice,deviceType,%u\n", devProps.deviceType); + fprintf(m_File, "PhysicalDevice,deviceName,%s\n", devProps.deviceName); + + fprintf(m_File, "PhysicalDeviceLimits,maxMemoryAllocationCount,%u\n", devProps.limits.maxMemoryAllocationCount); + fprintf(m_File, "PhysicalDeviceLimits,bufferImageGranularity,%llu\n", devProps.limits.bufferImageGranularity); + fprintf(m_File, "PhysicalDeviceLimits,nonCoherentAtomSize,%llu\n", devProps.limits.nonCoherentAtomSize); + + fprintf(m_File, "PhysicalDeviceMemory,HeapCount,%u\n", memProps.memoryHeapCount); + for(uint32_t i = 0; i < memProps.memoryHeapCount; ++i) + { + fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,size,%llu\n", i, memProps.memoryHeaps[i].size); + fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,flags,%u\n", i, memProps.memoryHeaps[i].flags); + } + fprintf(m_File, "PhysicalDeviceMemory,TypeCount,%u\n", memProps.memoryTypeCount); + for(uint32_t i = 0; i < memProps.memoryTypeCount; ++i) + { + fprintf(m_File, "PhysicalDeviceMemory,Type,%u,heapIndex,%u\n", i, memProps.memoryTypes[i].heapIndex); + fprintf(m_File, "PhysicalDeviceMemory,Type,%u,propertyFlags,%u\n", i, memProps.memoryTypes[i].propertyFlags); + } + + fprintf(m_File, "Extension,VK_KHR_dedicated_allocation,%u\n", dedicatedAllocationExtensionEnabled ? 1 : 0); + + fprintf(m_File, "Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,%u\n", VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ? 1 : 0); + fprintf(m_File, "Macro,VMA_DEBUG_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_DEBUG_ALIGNMENT); + fprintf(m_File, "Macro,VMA_DEBUG_MARGIN,%llu\n", (VkDeviceSize)VMA_DEBUG_MARGIN); + fprintf(m_File, "Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS,%u\n", VMA_DEBUG_INITIALIZE_ALLOCATIONS ? 1 : 0); + fprintf(m_File, "Macro,VMA_DEBUG_DETECT_CORRUPTION,%u\n", VMA_DEBUG_DETECT_CORRUPTION ? 1 : 0); + fprintf(m_File, "Macro,VMA_DEBUG_GLOBAL_MUTEX,%u\n", VMA_DEBUG_GLOBAL_MUTEX ? 1 : 0); + fprintf(m_File, "Macro,VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY,%llu\n", (VkDeviceSize)VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY); + fprintf(m_File, "Macro,VMA_SMALL_HEAP_MAX_SIZE,%llu\n", (VkDeviceSize)VMA_SMALL_HEAP_MAX_SIZE); + fprintf(m_File, "Macro,VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE,%llu\n", (VkDeviceSize)VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE); + + fprintf(m_File, "Config,End\n"); +} + void VmaRecorder::GetBasicParams(CallParams& outParams) { outParams.threadId = GetCurrentThreadId(); @@ -8670,6 +8720,10 @@ VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo) { return res; } + m_pRecorder->WriteConfiguration( + m_PhysicalDeviceProperties, + m_MemProps, + m_UseKhrDedicatedAllocation); m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex()); #else VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1."); From 3b96fb8130cf33b99bdc38f5fb1208e8f695f131 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Wed, 22 Aug 2018 11:44:44 +0200 Subject: [PATCH 49/51] VmaReplay: Added comparison of memory heaps and types and appropriate warnings. Fixed major bug in SecondsToFriendlyStr. --- src/VmaReplay/Common.cpp | 6 +-- src/VmaReplay/VmaReplay.cpp | 82 +++++++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/src/VmaReplay/Common.cpp b/src/VmaReplay/Common.cpp index 0350343..c324245 100644 --- a/src/VmaReplay/Common.cpp +++ b/src/VmaReplay/Common.cpp @@ -624,19 +624,19 @@ void SecondsToFriendlyStr(float seconds, std::string& out) // #.### ns if(seconds < 1e-6) { - sprintf_s(s, "%.3f ns", seconds * 1e-9); + sprintf_s(s, "%.3f ns", seconds * 1e9); out += s; } // #.### us else if(seconds < 1e-3) { - sprintf_s(s, "%.3f us", seconds * 1e-6); + sprintf_s(s, "%.3f us", seconds * 1e6); out += s; } // #.### ms else if(seconds < 1.f) { - sprintf_s(s, "%.3f ms", seconds * 1e-3); + sprintf_s(s, "%.3f ms", seconds * 1e3); out += s; } // #.### s diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index feb5005..8450f08 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -406,7 +406,7 @@ void Statistics::PrintMemStatInfo(const MemStatInfo& info) info.blockCount, info.allocationCount, info.unusedRangeCount); - printf(" Peak total bytes: %llu, used bytes %llu, unused bytes %llu\n", + printf(" Peak total bytes %llu, used bytes %llu, unused bytes %llu\n", info.totalBytes, info.usedBytes, info.unusedBytes); @@ -423,9 +423,9 @@ public: bool Parse(LineSplit& lineSplit); void Compare( - const VkPhysicalDeviceProperties& devProps, - const VkPhysicalDeviceMemoryProperties& memProps, - bool dedicatedAllocationExtensionEnabled); + const VkPhysicalDeviceProperties& currDevProps, + const VkPhysicalDeviceMemoryProperties& currMemProps, + bool currDedicatedAllocationExtensionEnabled); private: enum class OPTION @@ -471,6 +471,8 @@ private: OPTION option, bool currValue); void CompareOption(VERBOSITY minVerbosity, const char* name, OPTION option, const char* currValue); + void CompareMemProps( + const VkPhysicalDeviceMemoryProperties& currMemProps); }; ConfigurationParser::ConfigurationParser() : @@ -655,31 +657,33 @@ bool ConfigurationParser::Parse(LineSplit& lineSplit) } void ConfigurationParser::Compare( - const VkPhysicalDeviceProperties& devProps, - const VkPhysicalDeviceMemoryProperties& memProps, - bool dedicatedAllocationExtensionEnabled) + const VkPhysicalDeviceProperties& currDevProps, + const VkPhysicalDeviceMemoryProperties& currMemProps, + bool currDedicatedAllocationExtensionEnabled) { CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice apiVersion", - OPTION::PhysicalDevice_apiVersion, devProps.apiVersion); + OPTION::PhysicalDevice_apiVersion, currDevProps.apiVersion); CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice driverVersion", - OPTION::PhysicalDevice_driverVersion, devProps.driverVersion); + OPTION::PhysicalDevice_driverVersion, currDevProps.driverVersion); CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice vendorID", - OPTION::PhysicalDevice_vendorID, devProps.vendorID); + OPTION::PhysicalDevice_vendorID, currDevProps.vendorID); CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice deviceID", - OPTION::PhysicalDevice_deviceID, devProps.deviceID); + OPTION::PhysicalDevice_deviceID, currDevProps.deviceID); CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice deviceType", - OPTION::PhysicalDevice_deviceType, (uint32_t)devProps.deviceType); + OPTION::PhysicalDevice_deviceType, (uint32_t)currDevProps.deviceType); CompareOption(VERBOSITY::MAXIMUM, "PhysicalDevice deviceName", - OPTION::PhysicalDevice_deviceName, devProps.deviceName); + OPTION::PhysicalDevice_deviceName, currDevProps.deviceName); CompareOption(VERBOSITY::DEFAULT, "PhysicalDeviceLimits maxMemoryAllocationCount", - OPTION::PhysicalDeviceLimits_maxMemoryAllocationCount, devProps.limits.maxMemoryAllocationCount); + OPTION::PhysicalDeviceLimits_maxMemoryAllocationCount, currDevProps.limits.maxMemoryAllocationCount); CompareOption(VERBOSITY::DEFAULT, "PhysicalDeviceLimits bufferImageGranularity", - OPTION::PhysicalDeviceLimits_bufferImageGranularity, devProps.limits.bufferImageGranularity); + OPTION::PhysicalDeviceLimits_bufferImageGranularity, currDevProps.limits.bufferImageGranularity); CompareOption(VERBOSITY::DEFAULT, "PhysicalDeviceLimits nonCoherentAtomSize", - OPTION::PhysicalDeviceLimits_nonCoherentAtomSize, devProps.limits.nonCoherentAtomSize); + OPTION::PhysicalDeviceLimits_nonCoherentAtomSize, currDevProps.limits.nonCoherentAtomSize); CompareOption(VERBOSITY::DEFAULT, "Extension VK_KHR_dedicated_allocation", - OPTION::Extension_VK_KHR_dedicated_allocation, dedicatedAllocationExtensionEnabled); + OPTION::Extension_VK_KHR_dedicated_allocation, currDedicatedAllocationExtensionEnabled); + + CompareMemProps(currMemProps); } void ConfigurationParser::SetOption( @@ -779,6 +783,50 @@ void ConfigurationParser::CompareOption(VERBOSITY minVerbosity, const char* name } } +void ConfigurationParser::CompareMemProps( + const VkPhysicalDeviceMemoryProperties& currMemProps) +{ + if(g_Verbosity < VERBOSITY::DEFAULT) + { + return; + } + + bool memoryMatch = + currMemProps.memoryHeapCount == m_MemProps.memoryHeapCount && + currMemProps.memoryTypeCount == m_MemProps.memoryTypeCount; + + for(uint32_t i = 0; memoryMatch && i < currMemProps.memoryHeapCount; ++i) + { + memoryMatch = + currMemProps.memoryHeaps[i].flags == m_MemProps.memoryHeaps[i].flags; + } + for(uint32_t i = 0; memoryMatch && i < currMemProps.memoryTypeCount; ++i) + { + memoryMatch = + currMemProps.memoryTypes[i].heapIndex == m_MemProps.memoryTypes[i].heapIndex && + currMemProps.memoryTypes[i].propertyFlags == m_MemProps.memoryTypes[i].propertyFlags; + } + + if(memoryMatch && g_Verbosity == VERBOSITY::MAXIMUM) + { + bool memorySizeMatch = true; + for(uint32_t i = 0; memorySizeMatch && i < currMemProps.memoryHeapCount; ++i) + { + memorySizeMatch = + currMemProps.memoryHeaps[i].size == m_MemProps.memoryHeaps[i].size; + } + + if(!memorySizeMatch) + { + printf("WARNING: Sizes of original memory heaps are different from current ones.\n"); + } + } + else + { + printf("WARNING: Layout of original memory heaps and types is different from current one.\n"); + } +} + //////////////////////////////////////////////////////////////////////////////// // class Player From 83e5ea42f7aa9af77ff0f7d664308c974fde25f1 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Wed, 22 Aug 2018 11:46:26 +0200 Subject: [PATCH 50/51] VmaReplay: Added usage of VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT, as the app is currently single-threaded only anyway. --- src/VmaReplay/VmaReplay.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/VmaReplay/VmaReplay.cpp b/src/VmaReplay/VmaReplay.cpp index 8450f08..9635b03 100644 --- a/src/VmaReplay/VmaReplay.cpp +++ b/src/VmaReplay/VmaReplay.cpp @@ -1519,6 +1519,7 @@ int Player::InitVulkan() VmaAllocatorCreateInfo allocatorInfo = {}; allocatorInfo.physicalDevice = m_PhysicalDevice; allocatorInfo.device = m_Device; + allocatorInfo.flags = VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT; if(m_DedicatedAllocationEnabled) { From 62c009011256f73f7f7298227dfe18321e09c73a Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Wed, 22 Aug 2018 11:54:36 +0200 Subject: [PATCH 51/51] Recompiled binaries. Bumped version number to Version 2.1.0-alpha.4. --- bin/VmaReplay_Release_vs2015.exe | Bin 100864 -> 154624 bytes bin/VulkanSample_Release_vs2015.exe | Bin 179200 -> 185856 bytes docs/html/vk__mem__alloc_8h_source.html | 2 +- src/vk_mem_alloc.h | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/VmaReplay_Release_vs2015.exe b/bin/VmaReplay_Release_vs2015.exe index 05a6da46851e2ea0173f02569a0ac32c51fb736e..1e08a58718a3c12f39acb9fb603d2d03c993e275 100644 GIT binary patch literal 154624 zcmdqKd3+Q_`UgBA2}B@tP(~x7j2bl>l}J1ihPX43fgYJi6p^SLSr3eNHcT)cU~n>+ zu^nX<5N~!@SG?JERn{{hKmsU4R7CN>W3^*E;;|U;nD_gAs(Ug?262Dy``4QfOm|n; z^VCyMJ@wR6Pd!y#dhQ}es>9((!~a;!;aG($|2*=$m!A}eW5~$ohdBPV&ufRSN(sDn z=vkGQ%yCs!Uv+WyAFps-@W(5!yejBA{~}j)&6Tc8u5^`5p5nUVstYeVJ~MM*j)nTM zxi75w=76{EwEus0)r31g!tehx=}>$QD#tVAN2iN91w#R411f<>xAVqcigO5v>|SlrLi z$(u#}96MN0ML)-T!47u5#w z?!M=(da~|TJ!}vcs5!p+!aoN8=x~g=9;v|Tn2&2Ct_go0h|lpVn`8L^yeI=t@Xv87 zf4cuXfIPlRWt2Liu8#fj-qRn-hy1Ip{C~Cc|5@@!)4fP&Mp5#oN2cevyk`TAD`0%58}I1GP(8dQc!h5G zUHS#%vvhZB^^nnxx-mUR2ca(S%x2vfwxMo}qh@45qu454&DM*&t~pB!8uiHKIr;w3{Ol}$WI|STy5H#Ek?J?r zt#dd$qrcJ(uS<7(v$fDDbP_!h%qi3(*XMY&k@ZsXfH6M9t35l`;i-#F*TPSe7sixq zknsvTmF`|WXE@#ucGT^fUUQrtYrz{NR(I&(Sa6tb{ODadopk;SIzkV9ol^6So^qY5 z{XxLi?K-Rm?D^5Nvn8eGX#MrCgZs^1WMvO`XOB)y(No5|JoR6Bw8d*Xu3*ODvu*mU z{8Quky(kDJ=;4?36xidPzCb59(~ngkILUM{vS#8Ya_&h{H~v;5I{8M7DpLqCfQ>`xiks)ugK#-na% z9e~}e8?&+k#yZ{oK|mYVTneu2V}Ac6*;6wBiPin)mg+{So=|w+YkoBOkL`M>CBG*D z0r&3e+aZ0B^D^?NTsML_6(aGoyfeKsywhhjPl~49pf~ME^QW%u*sAIuFy5P{FZgm1 zi1)XQUkRqi{E_prbR&JY8~wCkjUK7bVb%Obqw%h8>_V;;R0Fs!$HlR(Bq!J553P^+ zSa_R1I%$zPU?_zFFZ4)Bjtf1sOm8X4sZ!dbB&X7XE04oP&qCg4AgA1X6@wp8)~4_= z*DM{3kx!3K_UOn`X3iWakIYzrH@b04P92cx1+SWO?uV8#?#)@j=LKt6p;^!pdZ;O* zeE{-W%}#xhefBib7*(j{FuRr|ITdx+lmo>(TKzP1wSaL>ZNOMS^#0IKS=If~b1NW_ zQE$Zr_CLyiH{W<$&-3Q$Ov`jI zNsnZAw^e`A-o=8VCvGhb@6_r$@Zc?+pRUy}W1&$!MK79ERdc)^eVVn=jg7jo)g1Iw zELM-A3Q&}%;kt6K(LWO6s8!I|eg_6CuqDMryP4P%UH| zVhyqi6G4B0nW0BQb)d0hvmX6Z1*=iAo7Jc}#zFvY5?lu!8^tP)1d$h5wf5(U!kAJ8 z&92p#rjwM@v;Co6(wCrKbz@y!;c;5{O7cInJ4IVK7R>CGp*96Ma>!&stwZe`!s;7c z@dQ2871Qc(Mv;N2{;ROfd_IGD+Q5EuGk7oT2^gK`KLLw&3!dbyo1g0l9v^`AvzhH8 z@&yF!=Iy@VD1lw=51^M7#8lPv1hMXjUDZ)QJ*+YGryY>pGAp_ zl;m2JJf@iYth<9|iir#eUk2XHy;Y zU^5hUSqf!;e13`1A`)#2e;zUijPr7#0@6wfhbpo6L-{9&eZbuau}?wH$*QP2C`ddllSTxcIQ3l>Go2*gfObsw9vRW1vOSpM)a*_ z5!8zaP~^Q4$@GcEM=jycju6Ovo{zIBD|dDUJ@O9=eun zJ6C#JKX#RthHREFZOixCx$9TqaNBAo4rVnHP?;JhP~hZ0@5YBHp=-V@r@_G7jXxC??0#TsWL5; zJF+d5bqOfD5nhT^ZPZDZ64wOcx?JJ9D;ciEHm(yBaA_l9B-`{Iq|l8dx>KZ1plhmy z%RI}5|ABqYs$LKH?RfFMmX-e9e^OvOfa}%!f3HpNB>OXUY83v}=|a9Pt>q9)qBRu-T~(BV~JzWIa0KJiY0YG(B}> z#{u1WCMM*`Gg4taW<>hy(J2{v3ucOF23)h)9P7{*-bHuRo*kdtFagHo#0(4((=j1} zv0q4IpEhsepJxg>WPufwFFeQuV^c+d>+{PTK&_vObq0RJ}N zZk|)=2NOa+rBu(-L)WOGFGVb|)1qS?dU$(qBAVd&Vp4Bl+Gnf;!!g4-p{}qvsOfoK zdTg_v`k@}JP3d^O+ZNID*7dHCdgfd_NhPIzF3$Vi8-jx^tz|A0JLF-aE>h+|^Lov(e^vDI}8l(7mpfFX1wE6~N zABZKTWh4{C_M~(m3SX2!>F%fa+)e2h@@!H1zC0MM=7WD`E}K+~N@an<07+#-Hm_sT zVQk!J&W8R)8O>}859K5-W~<{fDsA`g#MH_gfm=BuhIGam|h1`SL??9=AUS1zzslYM)!zJKtG)TX`HSk zf2N48YG3M~7wj23dM|Q;_SZ`dZ-zgwP5<$Wo^xlJGK9k5bt#E%F3oH9KR;FdK^d7n zbNi{1R-;8y<_v+KYTtjRIAYG}Pe5M$c^d=KaTvIoQGtT>J;CTJc-;)3_|(+__ZD3n z-|Ba_2DI@_eq+5K6AX$*`(RcwK}aASxdBuFb1&UEBg2mi%BGpZ@+%+?{^ub#{2Qs~ zy-wQbmq?(m`X8k4#m7NkAa6aUq5h^_sRbypNs;4EZ7OxYPIAzNZ~+cTIXrr2P;KI- zhf3(BAN@v`uH0_$xt$9NA*s7%U>04_th+m_$4e3fY9Bpy68v$S=AK|@QgWu}bq0*h zbc7gh$Sk*qLTeZ6HWd4HF1h_8q&?>R`Z4y-qU*BfU;+dmpOVr#&R*dLt5I@Xq8o!c z#vuVdu(+j#=1s-?lugu2Zz72uv`^|rq8t>!WCzpM^X@|jk0a~uDQSg(6gQQH7fmRA zq?=#t1lh_{sT@^Ft51yL4?L6wn7XMgC}7AsmcZ zlC4#;?S7Ozh(ypo4U!Tum8R1s7S&`Nbnrf9L~0#t_{WkbC6iYx0uy zg!AQXb2ivPDz8v&XH3efhIFJV>F8j^6cZsGter@QsI0hd5yjIjA~M#}liciSI5A_7 z5+OK5A}j!VJdOJs(Z5}O%E1>@`E+j_NrTVuqa!KzhSf*hrgq~XzUq-o>G!9?1H4vS z_?0ww51>3V;9e^-vl#j*=3KXeq6KR!Z!LNzq^r|nt$UqtcBbT7B)Otv^I@}S^(9iM zB}K)O;VNd(-J9neZuPwWR25;kP7V;_;ictxg}3{497+NDlCvCAuHE>GjA-nG?O(|4 z8n+orNG-hq z)<`P);c~jMMH=#{EIicn@qyIp1EMdK=_%73@wv{~H$aUMYD@_Zk2#A#eaF6}7Cy;- zdIUZROrStNHDP2TIyLwonV20P-Fp}!LqU@a>{PNsKwRr3AxG9{^i}p)2=@)`ojb|D zNzpV9oyl=?WVIMh*9mWpanM_zpga({FsC8_-Cb!`VSMo$ zhxlK#EepSKB}U{my78Kxw-G~rz}W3?iowy}!f8MtwTsrzINkkzaJapGBF4|vn7S+D z=lz|01bLHZLa0mCvD_>Y73W`&=5BSpiL{g(=z;#Q^C-D zaj^8@C+!J+l`%>xllfoN|!lY<~q~S{A}=9 z-sv2cCT02EuZrTo2VE@YeC@Af6RKavT2uuv!L87Zwd_$pww31n;CJuzYZG_sd7Ic< zILJ;BxG>=Yc{^#ml3P5EslVvquWI&@0cJgQm#v`R`#Z@UBIMG(Lp@~%0y$74%stHQ{_mXWaYDN7 zl+XJr>#HnfeKD1FA3_g+)pBSrO5gnpq(%&&xcUaIL5;Nu7;F77C#K$EWfA9HJli_a z@?@0K@|EOuSpYIr(cLyZdL!1S=x?xG4X{EukKHZp_6fRD%;^L%5N|lzSHnoWdbA!n z!!lKz5TJEwS&(FFL=o-H+NSM?#JG0ns^XbFXXwp-%)X<8Q5x|EGr@!uEN{o0A1+U< z7fMJ~Rq9iMyH~N@l?p&MG5Va9gP|yplQG3o--YoRt{A=VaEcq+tVy$jf4e(QlAeK? z$LsC!nc$=PkDu$We}QSq-2L^I7MUNDXV4B4s2yfA$>{9BqWN@ac(hU#jXX43FsGJMJTfi?`%%u{ zVA>!E3tiw-Fnx{-;AEMX4~h?j%7ai1W+r+N&X?!zdHI?0Y|YDkDs!tjm{je3S{^BY z=w$~Yb07`Ys?i7I)}<9da@Q16>)UP*&y=ZW%cLXMZle-Vm?PJcNlBwvj>^Yf!<1O@C z^IIL!tQ%ipbEC>tatql%i6N5t4N*F26B`iBNMZJ3p!7hif1S38> zXE{21E*hgbfFXvXORjh3%o*tPV&B0?-lt$a96a>i=nHm>Eqt^dJq@<-2HL{w?nXZ} zFPw*xqB@k~M^K7zb=nZd227?z?nj26N!H z+UQ;7;&4sxCayq;u{@`OkaG_7hhz#H&YRd%i#s(sDvq~CHHt+o|7E|7Hm&DM7 z*5%ul_!^D@x|*4Rt0;$qD8wgmUgRrSA27BA;Aso39^h~KA{87h)h1#Eb4&0be_ktx z+X1H;X(|2)R_+@!{4sdfHq=1%HcGu@RFv^C2@`jbiRFlUL+q{dFp;5c=D>GiSczND zs-o%Pg@TU^JnTRd_Tn8!VObo5vyg%#Yh$Z9LF81jTGNezobIQdhR8HM1B3d8#78xT zi~JZ(ez@Bueg*6I<0JrXC#_S%1CyDNaQN2%=r8puORwrlZMFGBst3_K(Bt7QDGGF*T`0fEdeeVbZ2Hdms z0;8m^l$vaKnH*zJsOi71U&_jq=CS#_bKq{YYOYrU#!z8WW55VBWh3~dpPt%Snz|tn z8K@hB;genMPu-wLPRp>QR_r;p6_~K#%L*!cBmJrU z20?9uF#$w~l)+QdZ;r>$E{Q=JYLu@>*5Ngpd{;_vfVXa}%2PkkQ#X-VSIk*{XNm)Ts|XJ@W5pV>U=U}OkkM|r zqmf-@)|x)g8q~EiB~n~^a6d;!KZaS}k|R~b65B_$F!`hR=8y?~<2+1q=O8fdlb$ms zXcswIIRCGjGhK$s0MaO=Crw^{~uNPhB znmZQSDws;J^!{ZA4m$ zQUAfu$Yv}SNb@OV2_Q?B%F>A}0c6SGBk~ZXX&_H*t^3Nf>U!y8Y8H!LrrfR~uf~mx z>_unAx`}%MnK`$AM6Sk-X?WvaKsGZ?*nMM$q_H8GCJkcTm_-!s1!VZ#0eVDkbmK;9 zUnH|tBw3orxLtCm-l5+9na#tua^ox5?layfDLQcOJjACOs{>JwYjQNLPLB?ghVqN+ zqb-oy>@{Fk413+(Z*IRxKd<3QbB{}#n;IFHhWiZnxD4LILF2)q!?>)uX*$+E@sy3H zY>RF{o38X@)jdRUS3q;1bys_0j_t5IFujrh#xwT%S8HL}&LNQX`Hxleexaa>z zg=eGitX_pT``qWGRX4D)pj13lF@v&WQlpROG(xtZSUjBcV_2<_NfBcw#0!w_+t<=G z;e1(B{*FTph7Gu;_CYvDsqv1F3j|nD#5i?Iz-YVXaQ(+^!9#Qm$NszxIu_sIKkKKr zu$&lmed@N3L9(i3rk6%XEbxXt9$D(%Sd(7rUL8EdYowHDPd9oa2oUNTIVT;XMaM@d z2dXHoW1HV7#jL~U!WtX;1&-sIeZAUqo{ZV<^o4lR>h~j2S;Tv^P*t!UE1$44rUs(nd=Ml0B=nPO&dL6W zM}~@nbR#PChkhCv>|ZxFqkE9}5(Wu^M_`lKg9PlI#H#6$`&ehVED`p_czEL(L1ipq zS@h!=(OkL1xWPdRl2wi+N7Q177Cw?8S;F5f7>Kp-{&*JtZj{FXc+{Tr4o2NxmV}** zISBofs@<4}n38}I_5gJt6_e{|p(-ShhY_u7pceiHt)NifB%mMS8Mw%&`I4{+3I6Da z6XEhd1@h+D=}18U6QZr7nQu{(Hta+cqfzQGQncqy2mR$?Dpe>Z^r5imVBD??oA?DO zFpQx1qkJ-ty9;@uOLz|qF0*t;ECv}6cuprc!n(8@KVu}IAs}pNO*_#g=%vuAF+sc=xKQy-Eu2C|2v6_FkgJ8i1r@^6JLU0X zJR&Y|@R)Q%p0|+59&G4mvtbi#Npk_b&T%%R{7-B+a+#_>qL*D3JIrw(CS}ETDyZbF zV9$@U;#fY|tjOkruwsD4inKq06}JHwSdpQHH^o`;x20r7-@2yRb^RC}+FbHOLS6g) zPjwxGJz;EX)YW6x_3RIl*7X~Vt$U~|gWcnGoz4flt~wv2uBS+zRx`T_b^RM~sk*XV zRkOD&NmkdAdWT)tozN5JBi|>~HT8e0>*-75b*-@Ldg=Q~>zcafx-Lf#j@R`TKG=1w z<%87qV#(8L9(xt)`W%A4Rp0(3)2?d=0DNlMhn)g_5V$9Cals|NJhy z@}J2T298O;!lA5%x8m8V{C_23EfUb9Ev4qNEAC*$pZ+$X;^}*>xXc?ZrB=S!s`!GY z<&-t0-^@R4Pg?hZpmz^-|4=Gm)qNu$?7Bb42dVqtB~Pnaa0TlAGQ*8SyN7AvX=0wR z?&nKYRR|nrjPCSvB;jOafIHHiex^Jg#r!f}{3KyOFLAukQ-9Ts`wHdm13m92xRbNB z@Bq;jn3$)J6wn`~KaD5q{UzZul>nqqa=7aG1Y{}DVwp<2YpGjPy*#2q& zrRN~(Jf87saH!sTE$8l9xNpVl?26w5-D95qO+v*7{7)7C{?GA>??|ZlV0bf=s*nV0#jIa+?@lx^EZ#0)&qbhzG z-Xyj-WCT)U*Zl#g9`nqv6Y8GvKh^!mMe({5t=;HZ+mhD(5v~R&Z}csUTd^DcWj@$- zf0PeW_q!xdtGVS8)P07mEbrb&3y+arzz%lFA}W_-@C<7qV@&!V<#{;YONV+?rVe6& z>4^rIvYijo!lz<(hV>!J4?xW`Y84NULUJ!2Vy18N@I3S^bI4Z-JlyAh;^EV`#d+vT z;Nh!Vlk)H)jQ4xs;iDik&cmgAuz7eRAB2a0l02>Et(D;6fy5Qsovwu+p$o0bx)dn+j~4}?9j2VPz)y4UK93<{0&@=QJmFTIkd z)f{&*c=;NJXVi7Vtd!=v@>cJi^?@ zMs#u<(W@9b!6RoJ(LI7lt8Yi2i8FkgjVx*-+qWB8yNqhA-VHJib6%Hephv7UDX4rg zTb1(mEviY&%#*BALMR1pl3ZX@+Ah2ici3Hm&3#i%yVL_vDi=GV&L@Du>;hiM=on-J z?Nht#Y@I+V*|1yH+)pvY30Fg|HxeF89VN89i!sx?3XhQZsTnrxzie3Nxh!Id6rt5` zfX0k-A=}P;lbx9bJ!)lr09jRGJ}gbJ)>SA8DjbR`gm<9!XOx(0BN=BSasEJN9d4na z-bdD7khRdxdVrmk<&Cj2%4~BwD(AkBmD55yP$jQy6a=hp!)`}*#k(ae=Ecv2cSV8d zdBgnJ=}?;bzIPQ=8JSUT!#u2D{KgyR2q{fXqr(qi$c>j!Y3IHsUV@t?oU2N}hLi0q zAdbD(&hNLdN0-oJ<4=9VyjVc2LdM9rP9f^T3Wj;vXQZF(JXA_o4Y#C0NLGre3*f4X z-x1;+KyU}{5M;I6ZV4vHsG`c>YUd|AJFMJ0kO3||uyaPC7UHzVuIZ8$39z@=u+H0A z))fLBEDl6RypLRfQM_ETDgmZk!T6(7JpRA=prElVC@Zm`PoQ&^&*d)&;xmW+$7bt>_>0_|@ZzUW?HB*Ri z@`}2wQkQ?I%cJV@H+8vJU6!cJt?F`Jc%GT0LxeN2vIypHm$Dv|H*a5r!AAx)r1x zFfux@Sp%!ZD*6~t>@P++;^w9z_TU}0bcB5ldSnKc)<3@<3r?MS(Tx0>F<$q&;7N6_ zc%Z=?HT$7go<{L`>tk%Uy@`n_J}lx!oex5GN)US!&AgQFpTT?G7~nImPTL8Z>M@lE zfeU{U0;TU76G>p9kkmh5=&p`5?CEx?XS~5P*1nOa94lnWh#dD@L_X92BJz6^IWkV< zTE5SU6PdBok!BP5(eBOb>VVjd%`TkHvN==2I(Zm<(oyFL zC=T-fJhMK7tiq7z0vLsMIy_oE+?MD9Sk!i;Xt&%fWuXhCH_F}rwWk(izP!htx*P-p89CL&6P&JAEYQPHJ&RMO6%0|7NO#IeiTZLIHf!OUr~BbEtR09RQ$b_%6~zW zSqCq!#otmYH?B zti{=S8w;k`?u+F>1RK14((t&}>qauprz!ty;2XbCJkgDXE+bIk1(2YTsTjM`f>*mY%{dy8W6@!0QnK8~N890{rX*Fk{h*(p{ z8-(zMir@Pl+OcNr%9Il9B+m6RDyh+LEN;#~AqcV=#;pve8H3@oeH}qTUU1g^dhiV# z7~@2{nhUT2W&mzRv4J0DyctB_j@Lp(B@#(-$lmXZ5MPMmd`8zA8)MGQAZ~EX(i9~A z+fMA7qY_U=;+J;fCn_tFrOmh5xxN@W#~rffG?wcnuVHF!dQi=wDPw6KtZ+Tu|grz4eF-G>RSBZ4Q<8=ymOt-z&V zi+2?(VBYsfG*{%wZ0I}g2wv#I4K8v9Q)FSjdMcF5le8p}SLY>qDp(O6ERjmxJu3rSRT&Y7V)H3Pv)X^20_~CT zmvp~4dbp0Q1LjqSqgg`5H*}(N%^Qy0cW`5R<_^_Es~aJsch#Y{T>mzeR@7j1%~97$ zGaBT+`EjGDvpeXEE7-|6gQEjRhy#XQuf~nc5qX+9aLHB|HKc?be6lL39;BX6vq$T0 zgRg@J;Cv0E$jn8ZaI;x=zrf@g?5vUZrlwxp(MGYd2-typn}9!91jzfy)7zLW@-%bk zMyOFPip%Oj5g?OZx53xJ{iXaBXU5BCs+7+g^EUP34hU_${N=c}%YS8qVt~AlJiUzB zB2P01%CD1)RlYo^@?|#SHke|SpW9tNQ>A>~u-QFSWjfkEe^$E#`7mFTy45 z@$>sbzq;m(QBfmJb4KdM66T=(#~X

    1RidG*vTl#2;H1j~r>LZVW`8$tTLtuW8zi zkHc37f3b97I(Z`h%JG0oJKFFWqShlX<~WeZphLr6y9-1f%;`FeFXxpb-ug<~jjbi{ znV+Q>%`4Z!@1hzIK?mWVe;U7Xx9W()53B$5-tRUdbaTC4(5gLKk~3A;p4wpDpPg_& z&ALbPw8ZVt z3c^6zr{hDv=4<6yeyzYM^nmrCjM>s--eDaG&M83!qDw?b(b4eI+`14sj4+d0Ji}O% zj^jebZ-2>tx-YgJsPHB1>EGp9-;WA~cPU==f z%mgBj=X8RL=7PB_d)QlFHTigf-T=rim-x`D z^w$c2;D`QFXRYQ$Mes6YB*EE=U>3coPti}G0LbSRWM++5(S0S0I7!%j8E=nK&2<6q ziDp?JG?x?29EIipn`5USKMA<=yks0(KL#Kk1u~QY&}^p>EF$0H*a3W?sAP>v-;ejb zYCWwO7=BHcS`XL4i*cuF{`FO=;6tQfEqpEBfb5l&ve2(t+8vbR`n#EGyu{}4L3wnd z=hosLowyQ)4PqRm|5ifDcT&?tBxH@Vp$h&W9os_r1LM$F7jym@3&F@_GH=nmN-f+e z#>`8L67V-D{43+wSH_&@C^#vjXx>UK%t(FEwLAfi3NfdM1E~?W8Iu1Y-xpIJY zYT;V}7Kl92N|D1PrxsV7clTBBd~A2Gg^H0E(It&YHZQ%DbVYd&yRZ(I;2=zoVJ!o& zBR^E!_@89M;sG+`lp1R>EWUzSGWX#=1w&i<(ZXh+2^i0awGaCO6%C~BL|yBO=K>r2 zA$KE#cjp>c5a)h8lPj$zHn1Oio19?5OWzYt{b0`QdfL%2WlG3#eVR7SR|fQ zu>ZV4pfmbHIavS`+)DlLX6&%Ulv<_7ZZbiP6}F-q*U5=X;pIq^jY#1Ya@PnYvl4ei z6A9Dk)T8o>E$QL;zZLYTpdb+yIpiuA4R}aYsxAr_nA5lF((`#+>G#|HC$?3`WDtk&(sbme8@6{w&r>)G*FOm$?vKxT6B& z@PS&`1x>HCbJeA6;mu~Pjpr-v-(sK#Ku{ z(d{G!WT}3*xO%GoLK|p_04)Php%!*1AWQRm0MtYCdu%0M2_U8UJ=h?@!NGXR()>Dr zdT4}Xp$I~6s*4qJz+HgZK3G#LA!Xnk}@YJDFZR#R)uYcC|?(pp>DW8WN=y>B^n z{hJ>q(~*g~{*5Z7>zm-9^QLeI1Mk=6-yVLTca$b0JL(0r^++sZ9Mf+Ju2^G)WPSJqg?&&d@9*2GsOg7J0jpN4kIEuTG z1{CHPCZB+0@ygO4EL(lcr2nL^c^BlEoKm>XfZaaor_%csJA?8I*;FK{LC%$bD9)_JS*}?rp zO=F!J`WhR*--xw!-~vSFV=l{WzN{WincZXgd~Z%ApBJnl9}>3sioe|24~?VlbfX%k z#5p`!E?BCEUJ#Q{H?GYHI*mJ&(PuCXQQPJjCJ-VY97U5d0@3l=;GeOzaC}xkD`||~ zQGsl7+QiMMV#hon!KTJoFwl*)x-ZssW`|&n^~Cyi9BY+e-5JDCyOoCY}_HZ1pocDvJ9>kl;{(o5)@)Q=|%3Of9OM-nJDSQcieUL&mqE;-8V!uZD}zsVuiY$}QN5R?oriTXf-^nDg*+ z5m|9oP7Y%0II^0DpTo5nSn7BvLPnf+N<}7le>{gx5S4K^Z?nFtN&C5x|I{(L@JriY zp$S~Fo34RCDwy=4-yOplMmR1anmO|#5b=;sjf$1=1G{L%_JA&Y!nd-uZBL+7-3_=uB);Zf0Yamf`&8 za^dWLMjIG(J6t2C^F<62e*-)@Py-HJF1=__%vpfEIJFNw(^+e$jZ|r{o1NF&X@gW+ zMT@Vk>xaADQiW}xcL)A=w)l21kQ@~s!SJn!&n%6eUMU02)Hd{GZ|v+cNL#KwogN*iyW1Fj@d_4VaoS)S&XC-Z z8V}=WmUAutChv^#*yz{HFpfLqS&l%LFpiKshklYUj=3r`9DEziF(-2oWxgvWT+bn& ze*Cx1^^;+*!q4Jhf0Rn+oXxMKWu=!O%IkY=C>$v*J{R7~61OiGO`VVb_`c2rSHyQ> zi?0ye1_nA@B0dinwJ{taA?ji)UONsf45tA;B455S!tD8xa4udNVa_TevXk5h50m>7 zB|S&(OXR*x?lA{L{w32V2|I&Z}Yf?2>0~Cd83A*+y zYJxMJwEA1fM0_g1-&ftHB?^$z!#~55DFQSfE4Hudm#ij!OMZ&IJ6lA6&gVP7F|o_< zZlM63$!8qw=nut2fX2!DP)r2qBzegbNjICj3Q`>P+k^Y~@WHgSe=_)Yi;=mgEkRok z_coke*fEU1^;=}rNe55NMbE)buA5(ok4QpOY@uO|CdrQ*lDavwjIxqG4P#O7SL5-D zA#ml0_B7@#lemB7@1rVG?!HL4k$j5C!X_@;)YA@L&|+n3q|QT>7PYD*yKxQ;kNGh+ z)i>%|$woaQyXIH1MI+%ggpA_Eh=}DfmIXOHb>an7oKk2@R^myCQaU0AWK4@@o{Rdm z-;Xy1TS}ti%0#5D{R3_G3E2=woJKK$-DiiG#Kvl2%)c?NJ2YQY#0RZe+KHnCaTJDu zU8un6EHO4bzX;}_cU1;*!3u^6oW-v}IsVP$n6nVJE_31(J!;)nb0j>HFks+6j!XcX zRpJ+>gnKe@=|eb?j5f3D8LRtu3`Z@HaRK`yU_7|3fJVTrPD^~>K;BP0P!=c8VH`I- zkJ86@MV&+@mWDqKUWYZ0EP~?PJst@UwIY8bu^E~A?3>9-pREPWwmhB+FSy9#7QXbO znIMpBD3DD|2;}X+Y)l05ACe*Tvk2sa^3vGhH(oU>PeA+x-TO{4K1=h~^3YU-^qKcR zf&d8~iY9LgOY>UPo43t+z7cc%HeqITIWK*CSAsF4ubRJO0RP#&%S9T{yQstL-i41D zN^R2Q9eS6pde;w_^}folOjy0^Q+c*}*W2=dBahI#?tun^zJhC(STUbD1_?a2h&CUN zEh@yZMIOI#9U>1uv7_n}V(MWpcAw9pzN05IHp@)R&>74`Q;~6te{MykeFdK)wneLl zO$S-e{JsJ`^|PmGTR%xF-EYjuz_%j&#yl*Jf2h?XpwlsXc6@%PWleEuL(j_5ztghD z=YPi$qI|YC=iLl`0S4>VSLVwpW^98~14R#UE`>SE5>Bn;=*pcYKMnGka3m=SOFFy1MUz`QyRGl2iulfaZ@{v7sc;)&p_OibWCm;95YP}>7@-6 zFf#fj-XLyz78pJ6pPWD3U&kbrWV|eQ_$o!2bv})jg_Fm@Mey_;;pryQkNMwAGF45& z%i%CJtqP3#*{(p%bGorml|uUM2UH;t`MXf&zLl@}$<71K15`zHw-8qIrPKvqfvGjG zI9n=ndM?&W;SkY{nKDJ6h{n%m+gE_zTy1DTMWo&a{lf!eazXvJjM=(2zmfWBu$&o( zRh0M~{{Bgj@yP6a9JP~6r8l@>OU(HX6oABB9C1@Yu>En)O(nyyGHJr$P@Y&DURUxi z4&zfWS?n;2P#<)de7|v-)oJ1#rrROF{(@M@3IzK#c2A`$p`oi>x{)4Uk4kbXC8Gx> zQcoiRF5NfTIis2LXQB=Ny^?LX)WalD$4S;)rvF8y%Nk(nQgLLbF3n}G)TK0`@P&hD zW{^3rM$S-iIZ*ej2>=z};$1D@ou{w}Tr_hDlXR7&T;rL4WztCrWkoZeV$xwMiAEI) z6djei)S;fb*w<5+(hVQH1|{83fN!576%wQToTDlnGae=$ zFfgid19;!b4C;Ql3sXPDQ$ z&8v;u=1nyvUaP6*3$WNtWVZsD(X35dk8fBV(~56yW&Ut2)csnTsz|{Ec4n(~?t1nV zL_BAXBFwpf>7Y=Z!KVP8LdD0mu#sRzWnRd{Q;?`f53UN(b60-~G&q!9V-SjC*hu7& zdhEwMp_UANL5%bE>aBIfx4njpl-FC!abu?ZgVp0nN*Hzibu>pE1oWZ@WuphVFzn#) ztX`+(AwFK62X?lOfWXd91O#Sf0oCk6a9sidOV(K(C@e?1#TJk_c$fU4+AMs;gr}{; z{6Zl0xKQylv~Khr`VV&Fq99!0>FS^5NKbeotE!(q#rOPN;0G=7m2 zA%>W;qV4>t88iy-%_##hextm?pPH@KDCr(RU&fqUCicEMSYQo&fQbh`SFaxi_R9b9ybvS{8^UubUO*atpEykZj@Jknjr(5J*%SfMr_v z3hz$xwmvO)`E zRSGS46MH?URJjF>0R&=}qm=^24bmGcF~cB4u6h1&x1V&C-j={06 z=4Z_?bYZIT6_2EqIn%?S31cNz!+ve1U?H$kQ@%YBtT`j-+Z&*^o;`$_8G|go5|b7f zSOV0@RCN8xwdMq!ZF*B3Yh2`8L`UVLNpXs!5<7NqxJl++7S6kf(?YjQ&`qo$I?hPT zv)j2d2A^WQ81kPKrL3_b`V;X(nO6d(=?6aDxLO9@KkA{El_4-3r%uAC5*|=?VDP|w zVGhO=Muyu>=Al1;Nua!Qbuw-!$`v>MU{U@djO4oFN)CAe#1OEZ;LoO8K00-!`ImN} z#*H4mP4tJCiV9(*i<06`tj0)I<3=E-pPqdb=B}s*C?59_5SayQs7A15bb36{-Exca z`x`vTbR`ry#^?VuAvee8`kX~fi7lw4h#B8ezd#_mmCJ7AOrl&yF?%$#6xOfVxdMvh z5D`%-mm;RU$FzY+vz3br%7v zQPaurD4LuSksm;8@^aFR9*>RkkhJ%FBp-6DyC0}SHm&{<{OZPKu-Mt;Uj;|?ZE~x% z*P4f{wxM}Y`6lyt^r5=qhcQ|>=z3+tK_i);f3Ax3*iEFiav#U^XYc(!-O|O+Wewoc&14-|g6)HYFX@L`@KsjnD^Ds6Pd;$BSOp^HR zOM4?R8xZCv4^lmGyA1}EDq8@W?ekA*?=;fh;UR+NT&NLc@0HH>i97Q4}1TAprk#W2_krZWtxv^-_kYH z$L47OK?ybw2iSZhwfGhdrY7kTqUBqO=T-RMNMS7GkB2`xf6Dk%ZrMd^ifI8XrzV40 zGzKMOCvg=;0c|!pZWfKyQ}I1h_<#C;{$EtQMo1g%;o+72wJQz(kWM0q|)BIMxEp zRe+5NfXXHs$duFsz-twt#{&EY>PR;pXiETW#l3C4B>;LAV2K6ztODGT0JtqVV1@$J zEx>sSurmSB6u>N|B$E2Jn-hvI3(lwD91v)`F}jk&JpecgPo)Jp2#_4=3Q+=GdJS_x+nvwe?V1B3Qzj{-_(-#p`e`*0~bDz zW!#W~DUni>m{v_oG2b3bk}Nd|J@vNvNgisb)MQD+i8$C_G^TWzj&r*00U2-MfZ+FL z#{h!8rv~RrqhEE54i(Qv*?sFGmJ&4Yn@|8>m7%kjFNd}SyUGpU%11bq)f}`U7a>|u zeNcj$_N;GZ&aj=^N)OuU53Ng`6s!L)B?!?=b^e${sFJ&!y?~sli#RjtJp%%}3ndX5 zYx&w`9zHk)ro-hJLmb97ngVRRAU;SaSlZ0LAqWcS+UVk%2yH~yD~`yqFeC1DqaID) zd8#x8wYng^^+TnDuUNIWioXdX!JPE~b@078RB1FosmEX{QnPZ7nspFZit|N~D!aD1 zCldA{tixqurlhw#hYZR(tV4=r6L!xzG4_9gEHd_E(GXusjo{+}yD{fXMYWXyQUFJ; zx%Gai+i$>Bs$j88MlksmCbQGoTG%!Y8(Er~cXkVdi^gFq-o%{qIX_m$VJ?LN#^L+K zj!FCVY^+@N7BOhiC0wx5s7uU7H;JN^>3*OQ3BhaeSqa#Mo6RG!6|b)Nm6t$ux1E=7 z#(>l^@ubI^_y09+=fT`mSkHxUvwfvFFq^c~)FY)d^~gDL`>zKIx5-*D^d5QEYQr9r zs%H`my%pdQ?X+e0q1EpV6<;k7y$wA>G4vox`jyDpes$c?8-O$-n8xG^G4vA0-{l;C zE71Dp&ZCe~{0(nDgF7~P*aJcX(Z9sUUz*#-7nA|8ta||d7yya$0&4(15SxzB|4xEG z(cE?~iqBwz48VsnZ4J}7u?TDKm<_C;WJXJuakS-x2BR$(#JTWehB>1#W>r{9b8*z2(SpOaXOJyw56W5f78!|oFT z_qbKq!wPOWYHp7Z%W$7$EUOVXdzimU!L3Yy+ry|^EO7QPe~N-@On_rYNm5?Az^$_A zj44~BEdg#1WA1Awk}oe>a4iaMM*37R@aO076alG`A`N*lli|tKb$Tz%?X?%Lg3C z*Gj8IE+23Jp}%IY=GJ8=fr>qwTg@wzjns+e)@m=u__xe|-l@#3?6|qLjHDY6EXO5o zZq)+c-psAa0>UMlTfxJqLdC}Vo#xgE%wvTe3Fg+1PbI61ZEn2ySMX+%?XoM8A`SGZ~pThSJ`z4nLg)w;6{VQHP-&}V7>(54K3)7SlAg8i@@ zAOvMcmq2nDPGW~a`_rw6_Uh?{PGW;SfVhpiV)Gd=P}yMOfwOI}PIKf|Fbdn^xCq{S z_a1DpyP^8zcpJ+G8;BEYIS+1sa&O#dM3*wRFHtraM>yJG9Nl1padf3saLZ8edBGZv z!f_ic!5t&(12D+inB7P}8D&}qStCO$17Fn+4u5*CEJ{FmnSs8c}bAE{>&j62Wg11ErzbI7Al6_{> zR%k2_e{}35!Bm8EW2{mhY=>rG(R82Zbu@HI^t69t%x}cP<`7vljh^O-IhO->!Fn2Z zN9!#Pphj=i;Um(saKZ?znplfx1e04l9zI3%5*SfN(>l)x7^FJPMm-n*^VfL_VVznb z%A}}I@Nc8}%RlMxd-iEy>ox?d%$;wLZD&E^EMM&sE@j2!Ks(MX^0BC_)rUk>t%w|# zZ6BV79nA23HzSGX8g_`I+`GGTAa- z3mI4IUE4rR58uz@YVHK9w2n24__FZdl3<8XIc2$mKb8D}LYkgai|?*-b{p|!us76d z;hi#O((0UAr}uI8%&3bwpHu)@I6|>b3*!UcV4`Jn;_zZ)axYKMjE0zV4nRms(ToP{ z+ENgUC}UKPSlPjC&p~;{qL}kk0F@XfNG${4A}xHL0$i>DmjSS^!8v1D%$d&8cn(I< zjAdH*2nDrLK`jT=ioV4yk2yCVBT(glT8^W#scHom!UIJdg0`Bs-KI>0A#oF-i?kXKQ2Qwpp{{r;dyH0p z7fIi_Hn~G)LLR{qErc_NqfW{~SeG`yz7?VFIM<$o&uB$;TE!jnb1w21&1hBTK~=}e zq2g^1C##2T9&j;Y7X01jhFehy@T!Wusw+MX1C@hAf%rvSrm+d6MW;f2s#WzoYNyyj z58FmIyv~$S#b?TXHeV-WSt7nJ1dQT>EWS<>?V%^745iQ34>H7Xcg!5rIJUfQWX-KRDxqc0n>k;Uv-K~0CokNQmFg{KC)uG3#UPMRq0srmFZ zrAM3pzBi`q3;WM}CQ9o=X&ZWMsEBASh}JQpq6?u;XYK>8FIYpJnQUEv6Rn@})r00UY4q`+}>+9B~Y(=GGJU zZH;Inb@Cy$7`YfgSpnywSb@11tGLkd{yWHSFLIk0(5;>3*2;FXIpT4w4u7T9uV#D8 z^4T%qH=QEmZb9|n4Jzj*7oN_AI%EB9--p;>kjKQKSqt3=c(BI_;Eq43Wwd>Ge^ID- z$AeZACqm3QE5VWTA^8WlSFh6AUojYrvOFogHPm7sqeszuJ2Lmip!{{Jy<7Lt-d;i9 zr@g;c?Y)?;X4T#&umBk*Z|F`OfyCe6-k;@=)*kH*AC_wGIZ505Saivr?fn5ct=e0w ze?T4(+F-SDk`_MZF-Xa?lSXi_%W2r_qJ_~(kTU*|6*p-0zW~88O%3A@(ZX2E1>z~W z_#AgWZgD>uz4|5>R{DGMRgdjZ2!iRqwgMPC)*x|+sq85(?JqCjshs@G^`1rMOej5D zWG3()BJ+em^bV*<6q(1_1W82ZmIy`Wilp#yk$Fmt$F1fpWbRFK&*P)j{DSd&u?1~l zn-y6Ehx`}u#s3MR;$Qy48XnbKXABjkoLPq92{Kdx3$Ojl23v^6B&2Rds*<9`Idx#Y zHK}OeZK9i3y!7-3G830M%3{vj4^uM>=vufv)GK^%h|(m1MT>KmAv4C0_#&E$ExK3I z5tlFUT?A!DEj&U%Rm>=K4%XlL1{gti4F{Z@<|-?8Sn+27tX4mW(8>&#=_u8f$GqqW zlE4oX+G?v_h*D~+RMZxiW;feoj}-!b1!tg^Y-{yT0j7rrnx^9drJoMv{j^ZAM{@TTHZ1WX8)Jy0nSWqYVX`smCN*dc zNeU4cuMAOVZZCyc;!NU z?T~KWD{u++j#monGyeIdA7?Ix;}^1(3w!4T6laZp-H9WRXpes$%ihs@=q%3pJhXRQ z*vEY^uwl~Y;UI{?LZ^j@h`&eCA_+=EqAWdT{-He^H8=@hl7^kLg;C^)HG*B@vS>^H zhW&o-y~)~T9(u43tD_y4tA1a^`-)JpF1dTR%j5*RWFlN!GRSlnbZm&@OJ-8gcmo}P z5@EP|UU35Ak0MzFbc4i;=$eEb#T1Mz%8wMAk$uC+IVELJDI%%OqWrge&*s5@HV_b{O@q9 zMagJ%IhUrs`px*jPoz;NOH>5bX)(r|YYtY;nU4Xw4zGh3NGlqD%HCs3t}|!Bw6y1g zo6JAn&iU%y7^*NUAr9%JGW>6S1y8&~Coji)vxb@u;ZB<#n^1*=m+3mg=S1q{#Dox6Ch@k=_^EE} z0rCar7Qg!!et|jd06o^i1J=N2haWrTLS08`;WGiCzG>I5m^1NL&1+E~2t&9KZvd)= zj>OGr(X^vTUE#R8u}5{I{nk$R5I5}3+LvM4f`Xbl9a|R)&HsYkJlgSNxDAcWQK*E` zbOgjOR?I_eweSX{muSy5c|%>P+D$LwspzEjOL1C?GkP{+U^3qyOZlIN-3yuDVLV!} zD-~c~?G9L9YF~UAs-J^x4ESO-6J{)@2FZDzb;P>(BJ?_^i}($? zrmlXdf681)`8jF2`yHNORJshCW!>)vhsib{oba)2e>TYszyYYmw*nI$$AUv+F}Qi6 zr7O)x(8%0qypLG(rLE?${@4%btKW%GS;#ad<|w}e;{tXW41D1ic-Y&ghyxJJfx%ZB zfCi~zG$L|FISQL^BTQnE{c!EesP#7RJO{_z=AsPjSD)lHGQ37z8Zt-1g+LaGj>T`` z!NnMOg2!P8b>|?|7azYyezW#PGC?Ipqs(qPp}{P4Bs3c-gK6*wM zp!?$+*kP_C(wfLEa*kS(;+TI3KI4%Ra0j_4kvSD62V7av%aJL&<7Ud#I(GeLG-7@&q%f)VSHerC%jm&H1g2{sj9|A18h0j|z zc5H(1`J-uLDU>37v2{XI5W|1^ZA9TtUxZ6AVC|1~=U4aF-BlTLHGE*5=gd~)ZH?J+ zteTBbL#Rf$9r2p@K)7x=^M#Vkb_@=n@}Fa{D0Y303rg&KR1?Ky^L-w6(RV?FTIbYY z?F7g72aFR}``tUMXM?xbug<{vhj&1i)wuMi^8n~xUwsTmC$|NRO==Et8PJ+vUmJ^I zWA7SkcRiAMKJN_YJ-C3H9Dudgf(~h#_BLo9vJ>;KE&DNF7xuC8 z2E2~-Wq(XhZPN4T&Bojd&iE7~a|);oub*E{4FTmR-o9E~c{E%lzX>lJg;ucvyY~^z zY!XQ~f1Qcqv4<0bv3VHkqU1j`uTE`hf`Dg!!gu}g4g$_&F0Q+Po%;RTv3+|Nxg#oi zD>Hp`4QtK`Ci*`U+L&Oq&YD($q08IPKm0{7qC0+Z2qhmq6)R-b@ER|-`}yRzT6w3wG|Iie&&YFL{38_)$+Pv5ie)Nut2yhx&~D8g z!$EKwOa-1Qmbsmy&HuyRo5x34U621+A;>br5{*_NYP8X)jpCAYTxK8x&%lhv1;qu$ zRxQ@07!pJU2~LnqAEwe;Yuk@ny4c!PTdj4ent&t$3IzRFqgIVueK0OjD~lHA_de%7 z&nyWj_1o|7_50)J%P`M#pSzrM?z#7#d+s^s?hWJ*$;f9&$`T+amrvmiAuEus5yS~% z7V-RPvz!jR`x>5;tHzyCira8OZLwuf_FEa8+{w(KHbf!XHEM5-5wtU?J+JQ>)Yj>{ zi`ru*bC=Tsw4y2IG6lkbCulmZcRE=1lPd2t3k^HdK8$2>{d`SG7zwzja*!PIBo7$H zA{{^JO%n5NoFMKY=nY8{E-C#$7R%(w;SkVy30o{;@5}z3Ebs9}vK+992Y5e^EK(Rg zV(;)OpvrpnXDSt!_bEiS&<}s)gZucPKK3xNGqIVkw@MQ8Tgye1b8D-1!s9gD2j$Tp zV5u&n3p+~UnqEQv_(?|qdO;JG)DlgmS-N;4pj!4zmdzE+BZmmObP}TOl@&FSs&~`= z%AyppZX^-4^3*`bk8t82laW}UqQ5knS5TU|sjn62sueP?m z|Cxia-nr&{W(85>Teua@AxhL_9q6T(OC#t-*;G1<=F70*By^76*>Z+^XG{Hn;qah8 zmuG&d?-`!?fxf#ulQ5aPoO(2dI?Ah^XZFQ_LE@s^A*~Lj)yGvvGJS9YhxmOaT8w^w z6JHDcE{^C?IELZ!bDW{YtF{@M;1lS_ClJ5DdF8mGSBfjTE26=VGpyu2B{J5CQzU(g zKvtuTC1%2wbNxgqDr43@{5z0APfB~K(n`!I$FU)?s4@|&=BI}FFGy)V5Z2Mi74=Mb z$QQBMq|FVsE?PhcBvpxNHVvE+Y<;ZRyshLX@z_1`(y754OdmehMhdy^lg|q)sxJe& zW9Q@05wo1~i-m<^dbW>`vYfdjF!v-O-VWq3IghCpn6%L46p}1~_IW4DJj;R?wwT(k zJ<3Y509b@#Y5#;2cs0e~ou!DCTx zN4T4_y<)&&MH_aU@DxEy-lhr>lEivbXxG*ezK^}SpPDPe|ruY_@YYtPZD<=?G1? zIa`Wv_czolKgYTUu=ATEbJ_V>eb2CSr@p)F{D{fiY~+#kKM(m%ilmXBX8PLg`3uhVkUvJ#KL`1D zP(NoQ|DwKUklF7jbpyZznaWlPP}h08Qe%X$GsA2ogLB7flw5BXzV9_V%s=Lvp%^eBbuK4-ttBFZt_K7u2m z^isLA&ooo)K4&M?)1-fqWOv24b9d?#-2fj~aC|Ct*$P&|S~(dV#qPw{avYzwjFIp0 zlg>@D6#JGQq`7aC^&Ii$I1-%&RJ!GDzS@|*wmH?kFOOuZsYd5Zto zmz$!TeL1IHMS|U>tGTm^7g3qCOG$EzW+eUqFNxxNm$K=zdCV$Z?~L|JmoDSG^OReD z3Hi2Mn45XF^ZJ$2RJXGn!Ax2?UveQ&XIDAbd0cY=khwGaD{hsLHg5})VbQ<0nm&Zfe3;Y~(?>Dmx=- zq7N6Z($~KYd)gzP;m2|4gQ#7+S(8dv$&YMloV@+y9IA_OBZecY(djt$2Q<;6t<4s+ zQBWj%c@PEMl*bk>?>J-++w#+N{-=t(dtkFYhk?*7Ns1f8*DqQ{k_LzeoA& zb^knFr~GF2pG(?T(2TSGOz=uhpY5zN4?$;N2BF#oaPTrdYEQ25t_NGm$|ZZo$(8&e z%TBIwH;q{hsId)$LGJ-{PW~69>s9;p_UIS$NLWcX#2L zVlsC*BLRI^YKJrjDSgwd0aIX+mEnsUt%qciB0dhB0JAp3NF#`#p*d;))EsCcZHzmc%vt*cbmjkPy%~8rbAp z_EYI(JK~R2?XU=U$6cRI-xQ^y>KF+Oygh&3d6BBkkqmO}9}{n`O)|(w<$qWn{!ej> zY@}+fl|ntaqfl)AsAsbit-=SaJsMzATaRL(D;iiA@lD?BERvbbKC^^DGvii9|Fo}a#Qwvr~Gc>7eEnzR+QnY&DOTJK&d!8^5uZo zd@X`oZfrhpBwVhaG9%$!eb0=9h`zfc;Y5?U%PD+imm^`1<`uUUR2{ z`KiX5*v0(#d-?86?}hM#rj_F)b@$WC&-Ui?p3RRsexr5-_IeH*DMm)bUK@cTfz9*A zqTTnicI2>LVAH%sQB?wUD}$|$RJBKw86#HXTmVU(&1JDpSBmNa*iX9JU- ziu7SiAA6asq|A7irR(FtOV_#9l8?P*X&TGYI$4&E7l57r#guO_*&mnG&zgO1>m!2_ ze$edWu16d(ltk^#*2DL+EHcrt^mvy@MkjVz_VZjN8O)E_JrVnivZ(LgwWLZ?LDq9& z)^kzTb8*)5kgR7+r6!fQ@A4C|r&UJ|*6FB_xrm3NRZ+QkGC6KtEYKyFPIk$qlhdH} zFqB-4heV)$t)=%)=Jovpr+j~;b~tA!NEpmfifhU0Xx6iIgVH}8TE1Mxsm)~X zL{iiI&fot*pC%?)EKivUTe;G#=faEGYvse@*W_HHrHqE(F0_gzk&&m;&M)@idCBAw zO@r9TIeMe>XUhktcxNw0KKRzhu=Q?ET+Y;QnOSkNzGr5|iTdu&iieuaUCui%>~dDj z48Y9%m|+5xzq1z0A@_F6KJI*;;P_~ut?8g>U}Mt(R-$;Sq$Ri_(;Yx6;i+U73G8Tm zBU1H##NI5o0Z3qlN47(mWYNh+4onp9r4`C)WSSMT#6pi=BVW7Gw2Ry?Pu%Mr{{@4Q zDag0%f8@a)*~N!QRZk?awy7y1%fFmg7wHOrDYIkb5hL29d@Sun5e~J<;4S~lnL)50 zCt4~5KRrK2b}~(bU(E8~9Vu*&1vYbho7fOe#S23%JUfa}7~gX>?P2wW%sCvepoxc;+d>0|nq@m=-55fZlsYmR9uvR~@s_;)?~dro+RHOKDPw9SfNSLWwNaeo7o35FP_ zaPVUVK8gA!bxJJ5Va_)lJ$fHc;=Hk3F=9qmhBbn|+dG6ML_o?)J(Fz<3D3psZX~8M zE~V#e$KOzllq5kU*Rx^r#|r5#MehT$1j)6-#jC$4p&|h&3q?UR7mH_mdbvv zt~I}|rl4_D1We5xDh1r3;*5-Vh20C?J$`;ph=yD?OZNvbVy^Vxv{e`}GrBL;_sr;? zq3`bKwoK+O$Hyl7^W-({R!fA29hQ6DHi98nig!*G-4Y|qhbN*OiTfth34`zJJoK#m zuD1dcn)==Tf*a&^@w@nv;T?IBl4ZnvJKJ<)sFF>?7^^BoM}D9&rio)5=$v<0_C{m7 zZZ!7VA)I~|=cXY3tlS~v`09mvQ%YnzudZ+PxH)kaHHF&PS57HcubU|EdMx5BE*3x~ z2xf8$XH!#%*EV1F(A61}B!SOtjloQ4AV--$6TI0o!*|!PyLd^iRbTInuA}w%>T(j~ zT6JIg@EB|Lsr0$z*D0ph-OjZs?Crh913L2_gHwpu((Q6M{zHAwbY`o*yPbK1$=u~k z`n%~&dS@ytm9$8_N~L?+y2{X-0o_n~_cBemTP*`Jqm5Biw!5 z4wa-1CH5DRRJ%x0Z*vBzD$HeMv63^{CE zP5)MjRfVj-vrsk@eln#u6RTp7Gg3gO(AJKQWH-B$3cR7s;Pf-A_6vV4s#Nq7(-X4Oul4YPeDSXH`nK8vVu6N4&Mdb{WY>{5qBB#tfeQ}ux`)L06=YO)d- zzQ}p$Q+TTVHP-ll_FIK7TZxk%kj#bKti%D2XlbEr;<#VxvkcgQ1Q!0yj2~S6BH~WO z?1#yIjJyE=edR=qQ1P&nEUK%Q+g2i2c6qRGmnb6J=i~Glab(CIZt)#AY_LAbGAz z=v8pl4wN_>eeqsM$v!k~^En|RNAz}kqzYN1LwBrC!^Nr60|exWPsNhQPBfw$>Kzf> zEabStb|fz(w(dya?@{0655%+e85k!D1?NKMXcbdWPtKHx+0PsH!+$lLXtSA#5qD!nGop__*ZUGDC{3J_jRD>#^m!uciNL*nenz@ z8-rMlsM3{2Sv+nmd%3~_GkwG>b{s`u&7uT<5OOVIKkYmq+GHHc#9(E%YLvuMNSd$e z2PdadbU*nQsj5vZLKME$Sgb~?+9UC{p^AhrG+|yF@ZsJ8{A9<5$@^^tZI7OLRCfI%;5rVCqkUbgh(6JV(gV==;gAV zuIIY`NyNF58FHC|aGm`&j0a(gA8iov97RI$!#r6w*U__n6$F*(D%Yn9ig!=i~x>6KZ`TIUL; z5}L7g79khe8_FAJK{ML)S`jOGZN3%wu<3A0^kC^hi7%TH-R7>=-$;qhaF6U##O|Z- z3QU`t4rJg{qTjUfk?1PK{g^hI7Lu#KFU(iR)b}i3B`AjaiZ7~eI^Xrk2RwlqXW>-o z{iX&-Yd`_i(Qq8!dL3gE4_hXu17?@+rq>xF_vL#r#cRwp7A7M91sBa)eN0>5Ybx)Z zRbj@vm}!@vsqQhf^t_eC{0ag(-##34nAa;2-a`-R!GfG$fGoZ7EHY@SbTM>@uT5c% zX2}FoebI%wSC%dohWE>)=6evcB(VkyM(tG?a|eZyWexZ-$CR0gC~B`mQv>Vb;+>qt zWe;uQ+=Av~v@9OXSZ$Kl^mbHK2dmf_=zHV-z!9k();OHY=OS#8Lb79W^S;iddHzVE zK=QLWEIxg1eqMUkrf^0ce{YBe9e=zjGd@5r=w=oXo&_jJFi z6d(DgZ|Q#Z=ltw`zj_;AztDcQPT~3vd%rz~tJc7^p8+Uqzgi+GX1}`gYI)qvepTfa z=Z+%As0X{kM)Aqt7BEyGyz|)0rQc?@C-zU+#O9 zrRE@URriZYy^b!nF00r(xYtx!@%QqZFtbdbL*!KVHz~irNV_9t*<)(!UCrr>cD+ATJI3Og zC(#hJvxfPW-Yh)J(i;>eKmt2jc=?i#MiE?7QgdXEqp^R;!9^Ud?yPFw2 ztG=2{ed|{L7wRj|)OY>=S$&yJwhD9Mi~=&K&DUvgNWjTAQh7gU~JhO23CiT1T@Iz(wrNV`)+BF4`SxFoQ*zW+)KU| z1Zd{|D=50t3cTxU;YcO3pYch!_)U5tkX);Ux%pUMe5qoB+<>B*xqAcW$S-3HM=Z5! z-{`_Ey5$4+8-JkCtana86Pp7_!9s3*y0PEq9||HO1Xokh#HZv2uWAu4KuNXyL!B$- zC^r5cz;y~9;<9CMDOl9rcdQjSr{79W?Vq=wxBt@1mQi=GEIF7b_#58!jUL9mLp*=V zbvjBGSyoUq%oGC6X4MI1S?uVpOV<~TcO)ZOSkjVpu|C2MUl{rWL~>o;bB;~Z3%#t= zdV6711dn| z%B2969;t;H)3!tdM0<>Bg$zuQ8ze<6o~T`81)hT{vaEF=?;AVb2@O;6aSy>z}0)!fkE%! z#E%mV#!fY;ySdM|bj2v{^OY;dSR1Y@`p`FXmFrEojul4})^QnGyYDOc^XH3&1Uj{O zds)eGsO<418k1H6`cPcXLr3+LbH+bq%0mHNo7Xs5#w@byLXl+`lw{o(Nz5BPeqjs{ zPUcH;0GFKXcZ$X0kc+*0#Uqz8Qkw0`g{yTv${ZcX#2oX#Wk0~&(Ek69JwV4_g^woe z${t`h{nL-*$A7nf{)DIe7w(^rKKZQvk;$2z#=?p+J*T5@QHr*)m~!EcEElpc{|GH% ze`bAU8W%l#Av?*ZopVHE3mc%$npdUjyo^52p3Vw0xu26+&dD1_qWEZFCLy`9>G`Mk8lQVrRp8>=Nb&BFRWU=hgP;s693kc&cH)DE}N}#aZ(M&q=|2 zpMN;ZFY8?-z6BlA7Mgj#VWSu|px*EFwdk^q0%=r8TanD@g}{bMeZ4{uUro6-zt3Ob z^AAI1`Udd1D}C!DOF* z#MsHc(M4lvdavnr(b#FVMPuh3PI}Q;3^YdY43rcayIXj->gfpQ8xYHnU|>-vjF@2_ zEtWTLf@ud{7@en;Y0hDiQ+Z4f5qQQZ9<;7}xEZ%T5;$vpf5bPrCss)D)hKU( zq-eE}7oEt{R^Y{j9sJX@AJ;eMH;uCjQ)DDd-6#E;CuaXWf>oo(=aIk%i;gq%NknW_ zxqbIHnCH$d;#b`{Qn`!M{Fd=oo>kau zZ)#R~wB3WtXUeEPUDxMaef`I-zWzDq`7y*)!0wqf0!5Z=T2x2>s2pI6Mlz$Dml-Fk zDie8)d5Q_+N@A;U27vnCraIvd)c+ghBrR9>Z^hRZa1KJx)U)rHsgpVOr8~DmLzw~R>}*wc>WSBh3=#Y z@paO~)XBZinWLcQ6Zgx*-U%J+VZz?kmVKQc$!dbrc8>rRD-pEtbKul%q3n}k4)W-c z{*@{gm1n}?aGr5xMY(%Mg@1+{(WQ4@NK&JwnDz`H{jb>%(qMUCY!U%5=FHU ziEr|e@1`zYMj@UIU{_$(J*}*uoGgYeFbk2&-;A;A%BX_d2d~ma2lFM>^CrgyE$rjt z3s?M$nd{gUO^1ZGPT8K{hzSRG5tgpshE?cqaAXRlreNP74N?7tt3&Y{w`(U>OZQ`t zz%uT;S*#Y-2r1KlSO3yL4$7q#p2Rc6w@x=gu9lq&9iDl2n=v-9qBg9oJ>d?&xDr|%*V&23Gty3Q(`@3d** zoR%6ZbMh28cVrr9-|Ar`4P;0}H`K^@VbDMqLoshS8+#O)i;R8K*3bmb?Rs>W?(ntz zLD9#r0)<-@=Y1f{d0*5HmdVsR9KAksSBU8TLsF%0Z3NZMspx?DWiVeSeNk;;uh+Z~ z_X>;P#7!TmeOYo&zEsGWAQm8I3Lh4F7)HnoqyD#_(c1##RaH3+)GIa{GEa69V9~i& zHlZLzXJ6}gCjxVvgR7I3ICoH58d)A>sF!)B6HDG!qZ7OM4^aS|sMn@8)2@i=GEuE_ zV&XNI+bf6vLYl|^KUo^@k;jd2!a*HVD)XJivMb2V{qLW9xsSY#X)%+VXi*PNAuREg zYx0~oWw)1G?n$?tSGNSJ8{NE!Kxa^0?=l@TxUpNj#?mn5IriYX=XlwYr908OWnjv8 z9{YF@BwzJ%XDa1p$`a=oDvsJGN0Y@TQi<#=kYDQ`DTRK{lpR#Vq^tIWqQ^^hHlJ9d zvwW(tB4S_UC+IkHe8Bm~&tz6ZtAhz*J=Qp^h|UilBaJIP8I~cP9yU5_1|$13O1k~< zA(zDUaoxy^sWkPjAw(Stj97^C>_k(=<2VK|H6G z=K5}M4jf#^r$03?LZk`6sKgO6&5=qWo^VafKGpA(ZU0De}-Bw-5rGuT8!5w?Cn+-<(^n+XiJ}BViRyxtkokc+V!k|+{#fA-zrV_p8E?QCk z$Z%nUQ??nb%E2nV-tUeRo?4kfuG~d#`J)P#(=oV1e@=F{`)3Y-e!DcA3w@KD+apQH zz7SD!qjMD%8|de~pN78LK)*x|l<5=Z&Cg)qFJ*yU?SlQxBXCLl#yZ&Kfjomj(O?W- z`AHgs-_UsWFg$)R2KRfpbLd~^{8qb4hT$Lk(-;H|22XrsFdz^aQctChzjs(Q8Gmvd znjBZ;)UX>miiX@ECW4C)8Nv z&-8Mqv*m^C6DyqQR2;Pf(D!ug`t)*uSg#@P?0<03$+^RlLmj0A>vw`1tFz`R{a%Tg- zu%gIGQnP{n)^`U$|FaK;ITUhhw?LmX80hVPF+i)mMJ~`j4~%5#ziBQT=u4p002E)j z!$mO{=n|&~1P#zHVeOJb@^{#Yxj?)6tKEWr6V;o^GEMK3T+pu_Ou@4}V3MV$4*>nw zLa1Cs5A||qK#w3?`JDsOpnnj0ozkwKxPvuZE}e?o4T zNq$Fj+4TOGw3Z9>HD2yqpv#;&)NJV8fT36p(2L&RV|tGn4ElyY%RC4l5<9G^(4g_j zU~2aKIE}_fVzHNoFW)(GFdDz{a;LN9IlmlSY23{wvr5>>&z)IxTKTC0{Mpk=Y;eVg zd+jGn&lv#iVxd+p(f|D;7q<+w!YSp>hhS!izVodCL@(hUC6~$Z2io0y@-u24z$&v? zZG=^BA567I4~}H%-D07b1}EQHE6wFXJ;lqN&PG${WSo=IP){8Q^%?J_q24{v@BDBO z(D&SLfPQr_@tz>J%isT^xon`v4F>wVUhZ6=tDRPAHl+R@XWlvdedoJrpg(UuIc6~E z@B9g%;y0A#HO^&yVnrJflyyq?j!k$T<~xNEPQ zU8ul$ObeY>!iM!#N4IxJN5tTa9a>K1x4D&@HFQzLp5eDr>zv1dSq5MU1F+E@fY%SE z+&OM(R?6=~p=>(;P6(9S)?r@m44s`r23J~;1=6kuV7XFs-~dEp>8|S`+@_rM!dAHe z-17rNNxZ>QN}_q=={6%QDI|F}=V zarPbz^oO^lftEWEvuSJ%26~N`I~VAn^H=)A0NwG=0YGo=+XJAlq~ZZUf7u24;=x^g zvIj=8bRIOy2KrVZQEtEZz1+D#Th36e8KeH<(F1|r-v!#^fAmR&az=pK%IzGD2U~2; z5R7q@Kf(Fq_XUDe5d(EWCnl|P<-H{70Q}bA0eHax@UGGFQ&HT=0G!7F+}c|I{v=)hByA+9X$R@-MR;iKc(NE!ngTW!S6qV?@leF@U4117rrmiTdv|V z-)Yi(lZ}i$G(A7urC`UlYy4LgfbXaf@g5~nP_=fmLKo!v1gO4P@7{7Xz|V1rJ{M2Y zIik`Ttx7FaIwdIVQr#>1(2%Vo-oal*A)K6E?p(#FLG zIC>tU22Ns6OCZT}%Q{1Z1ryTYOLRIL2kh8EU4O(_R5bFHlFI$xOi5xDM4AMc=bAQS@!|Jm;EMl% z*#xy^ZGyT4Up7HM8C?59w|3^rwXaD-nN84UXAmr>dbv%uy4J5(vKd34L&Z^B25|kW zI_rc`*OVgLwmIj&sk2U-#YV_Cftb)8bu|^$qR-d;&yDPVHnLS?{{v%1%<#oj*zA9P zs}MPFZW#oL>@wY@e=|*H!@cidxPRc~&Vk$Se20n++&_H9n=RXnIR0}n&q(4;A{)5f z0)5h8pttL8c)fGXV4!^-7|GJNX)hb-ON30h)8#9-xF}|T4k0~2@Yz@p5OwzNo}GAPw}v0ByEz zZDzN8%pL*#x9Ct$p(snnak)ar~ud0v9@lwE*x{3z3^GebH?~eG6aF^~<(rx5_RvVxy zBmVb;yXM<&wT$>}FXz(aWi@%p>BtMb+!>mP`o%emiVgQ&_>#wcJ!a%w{+e=MkBl7I zpX_G57f`dyem(AZe~j~^aP6g)zNPqJNN+js4DO|M%MDq8|KRpq_$Lg8{|+y=_!}R$ z_oKvxN68}-m-F4hbzkk~mOP>elBM4tt~s<3lNsz(*E^31Zn<3Z)l3WNs*fLB^;j=k zvhv9b5bLvc{`eP24TZ)RMZIOIrWkblw2ePjR6H@3h2pd?fyE{%+-^ky=9K zkz+uB12*<6yO5XO__o0x-luZpk#lsvG72IxbV_9CEL|f93R=b zB_s7CJz@gG^?!aD?uiE~dZleHU+$4#d(q-3Y27?W?Q0e=7JV%u$q^xq`(uQm#rd5m zF(N0P;GG{N$L(uve#>|qiFFA#tjy~v5rgoyXzlE>c|)w^dHLcJ6U!jBqLz!B_r&J_)ZJC_0 z$}^UuHDbe>2n$fp^7k@-c;+_&*sNcDNW*V(4(n@S-;x)+$+>_yLg3l9@P|q6;>mBgnC+b>p z9EFCGuxmm%1+0XM8&OP8#E`oZLTy8v{}Z|y3!~HMl3sqpFZ)IVI8o&H;o~NsKSlz^ zZQ=KPqfbLrpUNzDJ|-bE&Jh9%rh??sF9}8Bi~XslARNd|yN1PFma`2D%8lm2ZUZ=! zu{+l)d;wT#wA`%szO1i_9#hD>G1?!gJ#-l87& zi)cd(8fJ%BB17@(p#6Cfu7VTVH`NQ>8>eBa9^am7oRl0ODsb+wu9lC>-eIAuEq>qW&JA8f){Ml@W2qsZAk9{~H2EJYvtNN$! zuyC*Q)G%3T!g+}sD&?XA3Ck6(=NEdQ+^TxBaSp>XR>ryme_dEMvCa5Vhqp%xH!)_P z!y-T2FXOh$O2m3ao>{aV&$AL|2a}eZ^3b=v(mb{Flh;8Hpu4roaIbJ-uWZ|#BT85? zSj8`)v9LDMGuK=l&M5L-i+Qa`o+)HE5ivOgc)exTD3vDGsDSQ>N4vj5+ zy|FMwkdwniNui|HhzEbIXci{R+Cv*#K4B`Q|A0xBKn~vNS^RB()c$_OO3F`ci($|Q z=ib9h7^5$kTITLLV#3*5@RzoT=am>*)1o_eU&~&Muc*CTFYc7!Yw-mf_+g9}gbsy$ z$3a?m>_1=Njs0$(;KLB+f7s*3+j)(o@WYnC92UiV<`Hi|NGP_2fx3#hzMFnTKCyyZ zU?q>c@`X>CiLR`eQwOBYfEP_(oQf8{IuT#cC%F^EvdtxY(W+OYiE=Bx6HVmb$VPWz z&^w|+IpMMhTo4O{>wPVPVU%+>E49Je_Dt0ND0c9_#MY!ZX6N-4#q6oU2;Pb4HY0`g zW!AyZ5ObnwZpaQY0G|#8R^M1`cZclN@$JP-ls_cEwf``7;e`aaPEOkSYm7NkO*ING zE@{bFDpH73a=W$dBM(fgiYct5?_hG{lr;Z;`}T@|T+IDuT$N=2T>34YdKF;?S{<51 zrbd@w0-1-e4c8a#ML-!}{BoW3FCl^hVYCXLBv1t|E8#rfGO@UY5Ey(GRS^4hegi)D zjl?gFR+MD)QADAXC>|B$m6=4k6vLtX!oFjfhss0E*U5ZNU|#GJu9FG<6QM3yLWLdL zt9`uFtELn#fT1!R^yiI;R<$+#tgl!bGu`rI?N-9SO03l+FHxb;w#{mQo`J{oX9Oz- zIv(0~NhFYi2?ap#MB*AOJ(H4k_9QxFbvV#-cH&nw9)8Z5~KhyO_-@RjkfXc$Z#6w`}Qw3dgi@*7d-C$L(iv(+rQ8|B)Ox9(bG&BD|tDb@lE=2jc{<) zTam;O6_NNSu=B&*6X1moE9}5?s?4`+Ew+0GJ@jLWo*9)gH)HF?JlE!HS;(hY z^2}5$c_B>kRJ8E9*ul?VwjyE&>G@SryuW6J?0R~=y)Lj0L{^3D&C-#}t`N&W8HzIp zBI9d0jcoFZhvaRPH{g6rkLEpQ+Hbo3KD86BXIAxRORhH@lAkx3zgqsj!QVCf zNt%xf+TP{1`t|(v@b?;jl6FjGeqKI*+jt(&ZzX?{*3aL8q&>;+|MK^$o962Gyoe|D z;LGIr+{CrM!>~D%MScO%Pr}D+#Vj{rp=Y$pQz#2%*3F-chk-w@E*Z*~1yh$Gl~ZMv zvl25L@nszzzrK>o&GFBx>+<=JM-9mux{z7+u%>P#%?20bm1R9x!vHF2h@bH}4Jw^Gw*b7!Bd)FOK< zpykFnwB6L}+{-|eC_dCl3sb*H3xAE97F6l4I8pBj%BG`F=o{u?SG{%ATA<<;J_NRg*7Jl&sWq#jHJ1LAs>H@@gv66)R!KP!dLS}tCcMj;N z{`J9Qw3UA8Z#NZK8kYR%;YwQYCqo1*# zv39+m;i5xKBV}9T{QE}03cNgz$mgGkiKNDHol#=7zlmcjlA_@^rtp4%Y*QLlJE0?$_F_+Upd*N$l4MI>WLg`?X&7Yq&@eF!?eJ zJ|juIKV=_S$U0O99JPBja$#tjIw=u^?LA`-P6wR7I+DDe`2s1WbKV4&H!&BbWTbGt z!Ylr_$>9lhMJqP&q#BmCeFGi4dG~T@8Oh{w)3&$k)x}Z?=@ddYnf%5W#a^9w)X|TR_REi&^8#{~r?A(PBx8-3?(PlKb6+M6)3?LK zq<;p_E9{#E1d{emwI%dfB=6HL`(}YDk{Cj)q~zoB9EC9|`74j$b|Aq#ZPCyJcB~|I zTiIBo&Ms~ZEtTRp7Z0Uqr`E@Ok;gGk8iYUmGy}p@Jg<;pE!c>}DI&LQZ1kkcBSbLm zD{aoFK#BfPlXMJvcX&9zkimFnrn!rPA+{}Psd9?QMT^XI}J9032KtO|ZA6?`82>Y$lRd-z47 z{=qbiV0E}-z&tENMZzYua1KXUGpzlZd0_R!m{DnJ+52PYXbX9X(r(`>quR1ZIftJC z7}3Q2hEMJ?Kl?LTA0=cz*-F}GoIj872#c(De)Ouj4G3fVwGv#5i3XZr_Kg9&cc{wn zyk5`~h?XvpLGA(Z9c`DArL071rawX*Du-dFujjFh=2Nt2@>I4$%8`ODJxaq}zrv@(;oS7f6}oT5lFTr=^77NQE8S!uSA z3V#96e)LXdkrJ7ep5QrZC%dR>Vk%Pmm=Mx;^ACU`ZBG-iml}#6yIl^9MXP`&0mT5S zGf|Ok=b1BANk@@~GOvbPl6t9!*J_~d=21RXDj{NatTJqet3z^Hu+iDXu_!ykGR)f} z0d3udrlECypzRu1ij8Q#R-u^$2v$OHK=>N>76hlI<|7Q=;eR zO~kuQ0w!4gy2RYF%Q}Pm=jFFryPaq+fu<9)PF^7!k4~z=S{&N?vF?(a3eRnTU zVK(n#W&yzK)H+4)7pk*|h1$0ihUhR2M(a|iAev|>3AMdBw667f8u=PvtAdg6@8!rH zy>+eAYPMWhAboYWa6xEmQAL4o>9q4p{lxma9z8hZZBL;}c$^trvBJ6|JF#lPMJ5=8k zYF>0~UQ;E30uEgf|Fm%4cx&@}kwB_pZ+xM$vgVfvr!II=`Z`XxMC_ztMXl}Fc1SIQ zAl&kL4ugCZD`~?Z9~8mPhD|MSto~dUHE_$~TRwnO58+%^uGRW6RsP59)vp$hT{0bg z#p5u#Mm90^*&J3sz6RwvH(s@d;InIjiPJND=99n~K((pP{&TNjjds$6Omr?66Cm4V zu>25#Gegb20~zf4lF-(F?AKP&+&T#sYZ$;{)pf0tYG5}=)KJ+mY5#oYzH&3-e9pe> zxn(7_XP%_W06aW+SH zD9^vjn8KA@mE~<>JxF})&UxI%{wXTZ`^0a20V>lr-{@E?3EimEf9S?M8pl~fBygup zljxYv^4)YQ^D{NnaoIb%x2qU`_Me3(gDsrJtmZ+iKO)IxQexr@H51HyJc|X%6B3W$ z6XR2b#A=>bglv_PNIR4fWkjY-_7~AGc-iwMdmpjHRhrUC)^lEU#m0yyU7c2vx?ROa zCeEhep-yc=X1Ywx@vNGE#q$bT8NW}SkWF;X_@b40^Bb-BCO9qpLsg9;(X*9L2yyDm z*`SIx{sgq~p}@v@U+MS#@%pl+x1rw(_Lpel!{xB+Yz{%3x3KL~{d;(Y6~Ag_e&Yds zzYV3HR2~Y&|IIvxo5`N$in`a%dA;g&W}(wLj!sME8WpCQd#c2i~ZqVdfQ$YLxS8=YIk zIy{NfsBP5<|1$kBprwx{TD6y#H~O)QgT+i5$2tedu}-x)%*hvX{KP%{i&}Rv|5~<=L@O}03|+Ckm33ZJb2td_NjDG$bQcTAUdied&`1P=RxfM9AZ2o?7H9_)pYBrVC zQkkhL?-3otwYN(p^S=s!Qcc9J>)sAY{X~=OwgQ&_+EEfTIiR^3Ug;ZyPN5r}6nS98 zN!#Tvur$vsyfwo!D|ucad(4Yt_IIS_n2k322ZE6Voc`_<+UiuzEB`{T%r?UfGR+OH-VWnF1+@56q-)oTx~gGpuq@@_(ui*IwbV%kcKG$yXXjo4VT?DaTB zHyOB2)~0f(^`!xD2^zOQY_fmr@j+<3eOWHn3^R$91RAd@&WeVFvPc%c@t3s6ENFcA z*9;n;gG8&gF$jrMmKdn68%*4@2S6oAyb2WsuHOj#ayjJ71F~zrg9cZdx-^5V$4R)? zXSis*5O4Qf6v1_a-~>0XQLf+QJTaJ_cVso;aZwwD6Uh68WVw)EH6Xi^zf3ZJUF*&w z|79#IYVfCA)cEKglYf}DF^K$atYCsrdCC{}5PEkoZP#Ctp_zx!Hb4;=Ck#gDjse*f zp>oOmJ*_(nq1*nPjnK~@*<*xiw2eUsy(g-78aiw+5<><+CH!=ki^O?C$=qIf_UpMg zgTw=RCMnggQ>3%-*ziC$9xptc!2|p2&lwt5X-k7psSs4+i*aPiu?jSfOjrfxjQGQ{ zpv5e3F-j&pm|Rb$!!nsUa^C3t`)h*Qsp7zNAaGwcNi72LV0u<&)J$fbc{gchGxK^j zY%Fp6mE3>b%TNKsKLk# z9{`=O{?mXV$Xq0Z%f`hl!`-c$<(IawPs&9Z_)5-?fi6 zIoAws<uvr*#9{%G8=z;CG6W@1~x=lc0Zm+%)kJy-pX3hV<8tj~J=kjY@W|}l;Nx(*_?rArUT9y$R&)}Knj4|!qBK~3Vr1I!)jcJi*#fCIm<(1j_lp&R{N!_{itfbk=D_Cx8IxrW)06P?02o&dqtj}&uR5W*MHUQ zq3Z0ZMoB%Q%c*4(W8-qupIXDAaX@!$4bNtfjAYe&mDDSnnR+?zanAxS%8IX*BcF2& z@EpegD*=IH0JD3?+P7cc?c#HbJt2J`tR3z}O1I@0HQ#<<`6g?((t%aO;M%|ZCn(y%v% zExV|zGu)Z4&@I4pYi>!X>eIxc^>v9R+>sV=&B~&eExS5WB}TN*Mu=P#O@!Y>cOOl} zw#9~bM-wy95WWyi%<`c-s+Xu(Xe48AMr#W!-=uZi#3Ut2*%l4#Wm1Mo zC4Cj7Vbo4+eaSUiY9xhw$2fqMlzrXGMeE=6bIYD_Zz`VWW7GX;5&zcZ-(SLym4CmV zpJ@K0azi4rJuTVgk`l=09;S9N%StfsVho(Lt0ld5SWY@^+BKSgbWXaDbW8K^n@JC) z!g-~5}N^36!yo75FrWMXQxW1+$8-qeNL*h zPM$E0VJW#-w&ls`XkG4Y(RTe00TCdHF-ly{_$`91k#RdCfYo%-@7Pd&|6a+V$=C2s z@@0~|NRuh3n`?^ddAKUW(zXC|Wa2@AfUS-iG3gAHDlGsa0!L!k2cUN^foss=&5l-0 zo#Q)a>jx3v;c{g50U?s7&i=r6VLwJOepe;TJ$BxxJ%CCW7CLGVEZgfxz3XKg+YuN^@jZnk#3O3({si z-pRkwA?!K6W2iKZ@1L8$7`{guC>7tMc^-`KCpaA_zOVj17vCR4o~)&2H4u3CzRB|* z;%nJY8ViCYZ&QeAIOyt1X8NtfoH8phx7>Hw+@1&)!*;(}Y*MBBkhP9u|K#$Dl@v;q zKDb!WH+IL-s&l*^|6mV#yo$QK9{&o@x~kbz%31g7kz#;ATm8EeCRsZ1?fJBAUFT0x%Go&aaHvS5$Ma zR*moqCSiKHRxZ0>{`+J`W1LGnL$`j_7^g~KV~bQLp*N!yb5o^Dc@yLNxzVcAJZina z2h`e0IgeWTfGeZ0lHr&hBzi7ZI^M(n_rJ}>{!&oO!u~s^Iz4^)6VJ*tX+30mhcRM!Vj@-|8yrwCK#TGH+so{QUUY-zgAS|3&T9KNT|^( z90WupG?EZSSeCbF)fA5=-kdSToik3AIb*!#3v;oOH)m|2s7I4`c*dc#T-Me@w+=0D zs^m&N*x{ACNvE+DRPU6XD=$Vc?;FYuVdpS)Sc#U3d-y8cwLp$QnI@(x_su~ZCULes zbp$^%5sLY#x2FmNa`OQH21D>Czcmq;J>eScAK}d0uHj6)^tUnzv!Qjy{D!WXV&*Tb ze@fRW{nPW;hZxLyjJXe3(KXABxX3yp^0%+@fr>T(c*&E}3A&Dmvwg0|`yTV>WZKK{ z=NoL$z6gKLGEnP$b}`Sp;m>z}DMVlrANlKC{#>J&rit(_V0c6rLv^3epF!nL!<(uq zMr5`Ob3RbfEab9_^FHB67!+>=xHDp-!bXaIpL@J%PbW6Kky53-?5%_d!D!W~9z%MA z^s^Xprh!_CZ~@Q02g4KMz4>zSFq)1DNM^+DlSyj&&jEx9q25_~wmcF3RQU0vP5>1J zJu59r3z-pjHN~wJo)%8(`c*C`H7ljkMEo^qctqU5Gel(K5piU~EW<`hmnBaMy@ZXj znP`vsr;wU6{B!qpUyOeaFtD2SpXc51&!@i-)Zw2Ezs%*IrI0&oe)|Jpc&IN5ELdBGRM_ecYvS^s(74Zg>F_`Y~|F21*Ef2OgYRWBFrWZ6LzFZ z52s#N*pYFPDqTBYaQ}XVy#0s5&dCN|MSl{{;_b7NYGf*^5nAkau$%a!P~?T5=ThWW z?dCKG{1xCliuCe~JcGUKKw+jnEzH30{~2Z{SR_SpJ&wEaf@Bh^0 zko7;yrAJ)ZAWe_o0h~vVZl3pm9y#(&uk=o?eAE6fl8k)Q#@Fl>LLz?rX8X-maJe6iz|>0NeiU)|3fz5UonbEUWc^ZnRW)Z%sZH9YTT zJ^Qq4;Cr>qUc%3@|H;Ky_$mwEqfB** z?-@M*7xBIIs%(6ZtTB?K zc6_Eue!@#W-Xxb5YVKYyc>>8{`)0ibJ6XEJOD-XK;yd=vQ0*V&)BJO??4bn?8vKa7 zL<6g{J7a}w?Uzrpmv1Z~tFz&U22`#z8!cBzCrdxnoSk8NM2B-PUsIi-z~fSF!vkTI zGRe}9y=K45SNp|K?fCgqE=&7+xb5vElfAkQ7j!$?fjh~G9nROtUT2>sRQ4(RpIX9q zXS%epIusb+@T6PzK)!}sQbFHsYfxD`hm*nma)NxB*yik$Nji(9&_jTi=j3NnFLX1$ zdwaU<6>icCnWR=XsVkFort-X-~mNIhBchUs>W0Pc!*Sn%>Wzv}bd?>D)z{fIq~1$}1r4+nRO*=c-+)hy5V%q4isV zw^H<-xOy>vO}0#ex$%^Z9^qfaD{Nx1RClX1XW6zOm%4R3|B@!nrPJ~NRB9-DnCkr= z%@ekg1{)fJD1<`xr**aWUqX%ZkFz&PZuSG?x~%xeDf|EvXRV)!v#u5R{`xs&nFle= zQ>ruYs1#~gAAeNZ#FionI63hl!uF?=?3PdDfLF$|{kzpthqYuU*@M>(vD)7$L_)TI zE8s|o$O(EP0QP%7(nc2d8kl5=Jfkx9tMVyTI%kgHxgDH*Ek7c~>NKLTj6N$FZ-&Am zl8ba~#cMbUo!Z55a|p4JZ8;t(*MiBn{1e~dUx}hr&cMexc4}qfwCZ5%w~n=vr`059 zz#+50K?Qb4u;vPJ^<85nmT8rVdsj%!2|aS>XC-gJ)_d>*wJNI4ns<9Y?obqtE0A6HqbbeT4ck$;;wyLB|*;LkGv1h?8SEwTKQIYS{B@%IAvOKRNJviOaPx`Fq^=&>B2MIv zZGT5zz7Fcnpm2@s?<6Zdy7f}t9+H0Gzr`^BZ2sn8m{0c;gY_pe>MZ+g%efn$NIeim z`8tvBr{zbJ%LHpI&^1%?b{&XUZD&omUB|rn;CwW6q>Km^fA7N@ad`ad*Px()o9x~Ai7UId0bo~9<`yLaaN zy=1fRRg5HTEH6YMFOlcv8JlKM#SoFA+Q?Y~OiwB@O{HnbJ8jwSEp{f7GJMTV#q(l|KD-CD<)8 z?K9Oc{BHJC-w-h$4YLkh(W)=PYxP#jXLWJrJihrl%uyspS8*3OsyqJ@oj~p4ZIX~! zB2y=yA2x3#=Iu4}<~MIwnK$u8BIjSsoA`b6_9OFFY2MB^Z`I~)mU$C*YjWOh-o$yC zx6$U!GH-G7R&U;7=53~V`?7hPZQiy(H=v$l-d;0rb9s}FQ@L}W?+ZnD$QNPGWa&{P zu-+{oQ>ye8EfH^)^5WQhh%PNdW%j`QmGyP?$7p!aTQHP!OogQk4F_ipGlb*A@jC^mJr_iku5b&mHw z+mxT{y&JksUEsY7<>1?98F?;BLMM`1vyz%MNhd^&z99UCB27cXxkvCQ3EL=vz`q&w zR2Prtl6F|m=SSGF{5T5)%P$N0Zrtn}&9zs!4WPKGL-Hb~5at)5t?fmPW4E^KxTGjA zub^pIDE`VP`W4064w(|dEl)M!)gAI%o4M(;fBrs{C}{kiE7JSi1!~1S=K;Pr-b#5>|jQwT*R73nF zqd-R-ibBZpb{(>LPrfoKaednef8g z^057W*zP4R5T<=Z5H6-pksXnJasI^GtOv8S?#^)UzK`4yAj#pcpTF%jq5MXUq{GUy z96*P+_u(=VR+b&1nbc7|+%@q9g5y==l~|2oahvlizwCm2tyfC#CT7*t%&Kn;VWU+e zbg2|trsHtTU~WizQ}*M+yl>QwRmbd3b!p$^EF8_osbkAQQj@!C?Wq^~+6#q*T{3u% zvv1Mt3;p9@wmGrs6E$JfjfN8u2j!l6VY7)qR5Coy2w<6_6*{ivu7?DmZ6 zP)BJ0f=FWD7y($qtrs647Z-2q*>7$0E6t(S$oO#Ln(|2E^vW2)S12-DZ%!WNRNj(G zO-~j*sIg>{x{?s4tVX<7iQzKqp^WPK>RGBj(EWY`&nsYwY4-AMB3U`3YjsKUY(Wpo z$~#5!;!~cnH3`BfNbYK4Ld^WMOB3-iyr2mhIr0HBs_;SdJ-CHG;gXO&%5IPM7M@tx zG*($0L4{~VT$(B2_gBle5hspld|f=nN61Kjp4;)t@|p-RhJ6j>E@qrJA*bF^(G06e z%2oKbK1v_v*~>d+7HNKENb`<@COp5E1$ngstC5a-Ec*=hz{|9&0#9RsNQ4P9c@aXm zEG9lhq98WYf(&|zcU38VY>b`FPA`rRTb@v!`<+VNQKWB~H0*EsF%-9-K|YsP5$?q1 zwwu8Qn=Wn9A`j@)V1P7Kf$$kv?ezno?IQSJ2~qN*zDey+qG_*i%R5cQSZDSXhW^Ob z(*RC85u7st)|(}G49fF8KmzzRf&=>j*>sV*l2myqcaHg3W*c9N=o)~4Pjd1I!)L39-JmYW*U9*9r;a%3$6BM z7U_dWrj;Z6HH~4bQE9iDrj~D#70fvcv2yEV?q56t(ZnwjBKM0LVPZG;i+=k&Ancs% z2hjA?#4gGBWa6AjTtwK{TbZp?uWTW*-VcdTL9Tz{lDDPrM7YCwYFo6_@K zS7-m?Jy9k^lPCTjQshMocSeY92C%Y0h!l1Xg4dXjf8R8r2OU4;nFIg#WLA&a=lRhU z#UwEDk7OQ7-bim=SaVu(#8+a;>&hd^%c~S2Rlo(&l=a3hiU}OfNMgHKoYPKihB7+}ucD zMjo+C)}RB5B*K8zVoKNUgC#aWuc!yhBBSrH;(?Uqo5Dt)RFnZzEGsd^3Z?QxsZdcQ zpF2=Yjr&lc6pfgo3Qg?{R0`S*@`N_i3XqXxq{fQZO0Df$D_)8rE0PFND=LN{wG!IY z)H<$$YSEQaZBc~>XC!G=TeUeR-Ue1a45~Ru&HSZ0Vu25RH~mJn7Q$>2hb@YMZw-G; z93JcadPas@Ht@V6wMMulV7TR`=P{P%q?AR^kMMy&TsXx~$c7~Vs9Pex9`}o@>CPet_n$-ZAhY}1lBbkRTo&_csfx( ziI{O-88M7>LGK$hGlI^5;mvG{*R z)ZAR<5cSnLbtvE}`SOBBAblByL(^fC?B(wpMe$mKTW>Im;uJ8dqWC>k6h9`4;%IVF z+2b#gdpRtvy4yzK5*7mYjWqW2~>W)ka}_w3Vo;ONiF{26$9d!Pk%+Zm6%t_JFnFplaIx}W_pc# zRPmM5;WE2(Vk%g}&9xdGGA=x7B_0(9IqHuZ&!=%L zG}4^`;au4z2x$Pd@=5dEZp-6b+@lUprMYT-Q(b}R_ za#h0uy{e(h3hY2WNS?UUWKVHCN-TzCQnZK8u*0Y#@F^iq?WJ0Ka@=+cAFcL(l5HAS z0d%BF*PJG;CXML9vXm&wiXu6OkMN7`@|N?gJ2zb1u&Tr2`-+^co9?(W!{Kj1*E%Ct zMLbg!53^N!zQsx|QMpONXyVQ!6cqQl^zyY=R_mjrb(cs{0zZKs5zSUb?6nej+$GbO zbLBmymqc|LOs}GkBZ2nDDG~-Pu(9D}EAW0pt%PZB-(DC=E+EQF=e&LCgjEDXJvAr1 z%MVT!auEX6Q21A07O>I~*YMS&*9AQDX=~$=BIa(yX9s^ZSZi%#CLRi#X2fs9W5h#w zMJrE-jEqY-!L$qEp2UkpyDd?A*j^BkyIp38QzpOM3XRU)m+f zZ;xdV_#d7jdDtJRoielG2yWacYhLWnV@~QI8kr^Hu;@cXo2g@qja$qgIqcM#eZ?}b zrZQwXFss(E)Y|u@D7QcaYp(I9Yl~#6+DcqNd#WWfl@A>YJK~}C43ajbxHk22c;j{Og;Ms&!SO{ne;lsKtwK@q9=B7q7Sk-iZG zMLo-eyG5gZWB(|T!avS$8rOUShm)g{lTe$<2!bXARmRFO z7=J6@*K#*tlC81yp--e+Xd;-E&-dWu@x(?8`5tWZJ<`uL8jA_bXHRBL%=>imc;;0E z;oK5WNHv3#xg$mHvp|6^Iy0eqC37%Pl!sE^6P>oszcx{Lrn3x;&*waHK*elmE2GJ1 zg+)wCn>cw3D=g>Idt~lzqwEpB2hZFeghUFtq1xhb(&Bb3D>2IV;Pml!y9a#nr|qrN z!2p;Epr3_k*4!kILQlWmN`hE$2K(XXLiV}{7k1dsi}W;FZPUHOKNSk)mCb~P-|-tE5U2LRI%EcTG}o}Yorv>`Z3@4duGm_vzr9@zJ8zI zAHUzL3n$OaoM)bS=9y=nxzAziBtKPBg=RH)%RXz~C3PG3h~VP=g!#Q7MFGIj{gSgr z81onyyQ3lF}8{Tavn@j}Z!vtTzdj2Hz;1^-3v6 z>7){fXN{Qk()?FYEz0=ag_XIuII=Vr8&LMjX~g3t!NMF|e8m&m(4>SCP?oV5^>KD`jjupKIILv2fPeD;5{rq7di>%Ea%p}a+^eKAC{i~ z=+Y;wpc>UyrM=N@8O597-dPrW;J6wR7o>d)yeAwxT<|uLSxt3pZ!0HhU_P7>@?K z#6K?}JCz}Q&-1z)@nZ@QiaN`Q#71LlKAdMti3H0w8eu z{o>$Xi@KVhEV|$)9DSzV{8#*~$MuQF%t9>*PhFRwOgk=S_l)z`Fc#}=&C*YoS_1ny z+&gjJT~1&lVe}%~)3A(xe=HY`j;C&W4B~jE_+Cup;r@%wG^6JA2?%;IwEQ#{i@Jct zyoqFbJHuWI-R&QSenu3!#G5jl>_^iqvR}bX0dIf{u<;J?@tGFc)x%!I@uvd|mm#6# zFxQ~2)Otf*sr62myizOl+rTg$?y}J~S^nzPiscX5xV!D*;^6b8-Ep@zTQ_#3hVH?q zwy;8y%3HBca%0DZpdN8!M{4MV985=uZE_S3QVtV_^ZLP5OAc2+4u6NGg&aPJl*3p` zHA)VDS916Oa=?A1=+1nEkDaBZ-h3t~aEvo@`P^Vjk8*!d$^9{uwN38dCihY4J*d)q zOv)V}J&-q#41|8mvXwV?z))d?flEeMTHz0#M@YqCOFhxNCkV#*1BXA)HeYDQ!9n>K zLoZBWe&qcfM>j>?-(ez3O}$4V>Z^iqkREq1KZ^w`d@mfSmJB^zx~LXkE;uB|zNj&h z=ceI2UmjHf%4}rAJm*WE=ce-e0~WsI_cV*&B`4+gSQg_de%IUioeF+o4N8O8?Jo*? zothMDb%0Bz8W_;9rxqMT;3LuU1CMhcue<}>%kC`>{@d00B=Z2%w^mO0c2GMpFfHqzv7dK#aE{vD zxn~!HQ*XKsnvM_RZF?9#(}F#C70p3J!nb5n&N64FHQ3t5pqD~_ZXJewWhAylTRWWS z7cDecC2jRH1kk%yM_~6|4}Z`HHAUFNKD@!#H?*@-Ti6_!PhMz^}AkybSNP z2D|n8TQdm3dg+fB%MYfTUtF$Y>hpQsPzPu$@}dQ%=$ zlgd*hl_$akWy>@p8_StjDf9pfD6(xmn!#5Vx+MF4%P_oWZFmvdR@!|vzVjEr`sQX? zk$FMsAIXEfKyWGc_oz10rgSn3udP2Lz->Q1M`fc2# z)bDc|PXX#tjpwh=B`*6q+Y7f55fCi=n0&`_z6%1b(Es;B*i+4e>dhpzb zXE!X%E)LoZqA3In_~g1d|Hu{ef}-tyj9f9yH9|Q}RATHh!H5>5`EjD>La`VsD?>a6 z_q9F3>xqiCKavAL)CJ0HV*gNR3I!?cCI^&Chqn=3c0sl=274jhtNJ-wSj>%}GS!Eu zgcu(qRhm3lVd!`_>Mbl08n)Q!_?Uwmra9x===iB9?=&)sg=kf1HV(joxE% z@Ne9eR9<|+BV1WYy?H0rZ4|LQnLajt;X|xC{TD+ilH$^c^s~e1S0R1X8I8j(MSY=Q zH*aVPr8h8N+vt|Jln?FiSiC?4VAO|jwUkUDeGz*|_hV^Y9!H3UQO6|&cgX0qvl(jf zu}xxGd}yAp|1t#-RZ^ErTsE0wU~h0*5LI44c-5d7QBq!JE~=z}F3Sn%vXrO~nKG5A zlKGd^&B&6uVFL*=N-?uMfw-1wPiA0?3IkgjbLap?~tV8u!BPJd7;#x zsH|m1F-xV(M{iI?`SQldto?%w*;G`c{Y6S3s%U;Iab?k5!J=tECx}Jkg|bH!&F{%V zw-cQY-Z9;sd?S`)fR3VZV9CPU<-!@5WtTyGuH!|&6As>6MzOcZM5D4(62&01(~QoZ zigFjlYAdO!QP{{ESm1{xL6#qcTfk1``b)c=n>(((@lz4Ner2l${*kp!2_sQG$JPoyNqM`0slxImp^egiEp&28{A z*bz0_zYEp`C4L97>vtXG%omhZFl+^a{Mn-z=K#;WX*b9*O>#s2HTbGED#SDEe=*NwsB~^b<__ z&&xbV75KeTF-Si)O#kDWNdC7(VUvEENxx|%`pcqXkp3^x8&o3wb`<@rC~QogYpb{M zQaP=HevO(T!QG~o>3f$p$Y%N$nx=1F#QLAz{>d21{;eI={#j)0_76P5{@tVOA3iM_ zVf;4A5{VM>5{ppS_-(E;g>ZXyq>$62u#q)w{Qh7znPIV1M~S7}E|%vKBE>S_6w5h! z{zXbqwg){DRm$UcmyafouUPc1v)Y3nfn8~S@vW(${l$xZqsFx@nC_Jy!-;IYs-hTP zVP|-Xm4Ix*@=S)$W;tq1$?xD8W=-;&F)F{$NhE2^Z2bNkiI^XmDPHs=*%!Uh|8I7F zuG@oc|CNqFiu|es`^ak!Yts^z_?1x^SZ>e2oHw~6$qtE2T)#zOpWAL?$ z4A>815s8$rxu~pXDC10(d^d+q9H;$S_KtDWFMC*9pXSiZzITf2Z`dkgp8w`*pMm8a zeia!j*ncPu9>Bsw%$WyWLGZmMp*UEcj1S=5=kPnRQz*t2bUV3xk&Qp8Hw_?j^vQj+ z(_Nj<#Q-PyNTzcAY5_9u2*kT^6e7UbIN2zd4$!ke|91ZvjEje`09@R?)UoY-e_P4c zy=-%ew(aw;#V)gt`d!=h1HG*thp^LTumGKM#o^!53+zC@jRP&&n+GFg`h&XPC;tcJ@ z{+Eug;9KBdk_*L`>YeMsOEaV5kK=eTL1*;~hmH7Hozkf#XDenJbe0j3_h%?wfA8WU+U|(v{vq5~RyD#vz zd}AY_SBVN%t;7l+xBOq`#`VaDam6Bw6{GS?9-u;TIp`yta=03EzGYo!V=EQ+5MbFB z>ViRfY1iDM1&xp!u3E>@)1ljViit-%IQ@yQ4v}kI`id<~2T8T7_#Hfd$MeC0LWh47 z9QWc>$YX$C$MY>b`|)7Ks2H0)ioeS1kTH_}KKxJaFaCbH^Sg`o7k@Pq$58ecKeiIS zq1$-ospx{{yYM}e{Vp8Qfcsk*Oq9I=&t7zC%09pc#ojrRhGgCux{Zghit~Dl^RR;p z`%_ne9Uim{K8L}|0!*FZSaltb4K%3*V_+1M^wKNyp4MbaT2yHD9@f7 zIEsR^%zPQj<*n+$$;I1_x34b^PQu#0+GmdPb_8aZVIdp4|J5?=0^sArK*39JU*xmc z^Mua;hkmd_@0V&H=i-%;!Es%#BB&j9vP|MwFZ*Sf&n_$)PA;V3;OMera?~;IY7wHQ@*B<&Lfi${-X#9n@*-H|2J* z5-8cn_)g{KdGgo+nnm;~oh957hfM_q?#>fvO)zoA#@y`=epZU_S(Yry#fFCkSY1!O zX_u__P1qiTiJ^sVb?o;Hw5M`QB=mZ$)kp^480vQiLq{@H8;PAq88gpDdI*Z)ic?`) z$)Z8*b$F%>Z4{bcd}$lKhfKMoh1L+uerzhu&&VPKdkFjS8CmWn?8CXz9MJ08gh*tk z>noYqAen}35C&>Qc-s&1zJNJ38!9AA*VE)!NlRIXGj7)i{E5ija(mx*4BMCB)7F;-qf`Ry<2&P z#Q=n!z5#v5_kJoe`lGC|D~C4#pzF#?ixEb@`e4MDu#ce6-CZ$=?-vdQ7U5;!pCNW+N`q(x5^!{Z zXFzbC;DSPTAP<`hv6ZZ}s}Lrx(2W)v-JL?zxu=aQ_F)5^HUs!91v>g%#5nR4tVIE0 z5tM=qH=+#{x~*8&em)2l>Wo_nU(sv(L9k4I(Uzh-^T_v1h>n_68tck5=S%r<L~U+hc0%<_LE~*SFDR4Czdhf{z&dOMbek!`H*v-mr{3;QK}QRVd4;N_YH&S6}X3< za|hFL1HvnQT!3+a-|tk7)EL=FvE75Ib4-WIZF?(#Ar7f@<4$ul4q60P9{w_RWSH8f z^FM5S2!3$V32oqq(!%O|v*p1NXDoK1Y}3O{P)RURncO2u@@tMm8CBU-`YR2YHYzik zSF}#ZQw|5lp%7*opsiA)z=}x}tv!T!g{wY&&?qujHxIsBCR7Br=_%M2sb@Nwn zi_jMQeO@_R=;Ya_oX{*2FK3_he*lvnIv&qVAaV}QY!g+Ti)CGun{RVfh8Mw5;Ct{% zif9&z7UUf-z)GnR`U@)Ij)TFJMc;>Q#rP#rAsqu{9Fe_bf86PKK@)ogM{7iDm4MtNr4EiTBh#k(Z?(&%V2Zmk^t+nB!TK0vmQ~0P*NufnZEHs02 z^}!RNNfI6m{R2ZHSRxEffbr`o*j0TnRENw%6l0LUAMqL=ln%s&K7?!jE5C4r{sEG~ zeMho^xJV%;0P%6WejZ=VGz4&;gNJiCQWw#HP`_{sgDjK#(F>D>v02&}tG1ag>wLC* zQOe8T!CG=)f_v}YwEB;5rNq%d%+Y=By$8|`pNp&`irFT-c_fZ5U&;pMgi@+2L$@NbV>27jo96=N%M z4zW7(dXJ=s{K_+S{((>@29z6A(`5>xy4ey#`^u4Eu@|NL*uNH?(r$>0a@1E^*>rMd%>Z z7g?c9GfRVe_mpwc`+~TSFZpaJu*F>)H~UibWV$mB!w*Z9w)KMYVhFWrPZG4N{sLEK zpc31=Qy+8T5=6A?vHfsj2Xk^qX?HUAwc{M?qL=-d!Ap{{q1=}Uvm3j&bWtz9Kh*hD z&XIRu)3?2%1l@Vf3~?QWlZb+f2aJ*BmZ`=k;Lh0bX6Rce9!xUfD8@~11;)6)nz0rl zQ^le5tK-5O5F&QG>38D#mT{<_4}(tt*2morU2>)_W=y@45_uDYqWKw<5wY{n7rS4@ z2k?J+e$>Lb1{Fob(EruKex5O$s# zXX<=+Tr79pqE+s49DmB;PkbE16%m@tZWn5wrV)I)6h{luVL=bPJdG0^Y*BG`3O6wy zM33*8f^2{DOXdn7%9x>pF&eKHT$*)7T1mFA>5+oB_M?nL zg=jMvFqHAW9726JBr)zdC>7v2v}QvYvp|Wex7giThf5E!D`Y^vl}Nm%hW9wgMegnp zU%-pr6Ta@pVG|m{jCUpJZGC)+Rs3>nr(!a&3*9o%C3mKx8-$*;{4#dv0oih@80eE# zW7?w}J5mY&&W`eh{=-MK6dzNslKR;M|roVNbh}oRF^3g8I(9~fT^`C-jo z3N<3Vp*P%}`*4utPdEw+)kOcCgE?p=FJF@FS-8JNTIVQQXAYxv+UL?5nz=0#{F%1) zO$zWWxt|Y5WVs)4DC1%2?R$KzQ&1?z6_Ca*hCw~hLcF&?X=*dd0ZKOJSFqvYg|i{K zP2Yrq{*>aw`r-oAtqPVL3LIIXs>Hnu1l0OP4DO11;0}d}0c+fDyxAXjGeVf^{Zr<_@HQq?QyC}GKf7}*$sj6E)taX(Y6$PIy z3cljn`k|-A-{{(Uw6Vog;@Udc;`cNaC-oNv-zo~mP27cf^{xMM1^f-gNpA*UElPTG z>(OPNmcaZ2!4pMEugHcd^={mZR)l-%y2?%j`?}U2>ss|GE@8kPvd)z8NOH*4+2@=& zc=1qbQCHmFZwYZ>Z@feJb1tloj}iXt3%vwW;&y-h=}_kh{+F?mVhrM(hd8d($N!w(*0Mit-A*N|{c)?|f@PGirnapl#Gar_JAC-RxeeNF!5eNsa+L(+YY9DuXkD+{_X_w;NDt|G%AOh}D^i9zp{W4huXCMy2m>ii3O za|c|?-p3>m7nUJ9HV#~y;S(CZe|o+AA$BfM`vy!%e@5q_U=*oeY6 zZ1`LE?qM?5MMZgkQe+bEC)+1S!6^9KNqMD{R`-qu66pgH+#Pg3q1>F?IxG@pxSQek zXhOitrT^(GRKCHl)UjjZCGJkX3)$R@KbEo`AiZ)jtJA#6jBnB#C*w<64f-AB3Veh7 zLY)}5pY+{$k8};X7t`InV~}oRg>*}t88)0u8_uW8EgWIPIkwiKbI^wKHp6vV{WhG3 zZ8&>uI1kuxcG_@mx8ZEJ;WXKByf&OV8;)Dy>}DQI5gws$te{qXE<-hS3~@5PAHBRc z=Aq|#r}wj=ov776j)w$owK^QBJLO5@`83}379Qnqr;j^74Qx7r=_o4*y*n41idYlB zh`SI`6V-2Yd>=|!;*W{I@7}S1`AYwyQdE>B>&s!MFauzZu#3m|FDx3x_3zTiSr_uLWle1 zU_$7bk9bkIDp0V@D_k7&4{Tk4zDaND?R$fJVZOsz>9`J)ypwP(N;}HXi=BEHNt9KG zK6^ichHe$Q$_j9FJ{R30JZN^y&?XiyP{UDw^8R89fD<7{;zMslGNA~B6Z0Z5DCDX& zNUS{c4O?PbTTotXSvoIF{~A7?;H$#O>0jN@#YZ{85sVvxo2^V_t8z95WoYNZtxA^k zTdm4i>DR5wr|3umxi7Sjn>6>w-3OAYVTro~PX3;WyPhtHGUxDxNI1-!)F;JX5i$G= zL#N?=C=-85gK;i|VXF2V0L*)0>D3uJJp<=AFY84*1|#72uhbjj5{Rh#Njkr8--36b z_rhhM*M~o#_a?$X?UEYj}@_9U3-i=+%6! z)qbajcWP*+m#fRihzftDNXhMK4IkF`&4bRbVl7=UiD0;7G__&5&)6lEol^T|6xIn{j{-=uq^ch6YQw|Bu@TOmA zf0}w}K+8TwtAa!KA63^$W*j{N5ei1 z2Q(biP`shyrD>R>VS$F#8g^*7Q^Ot&_h{I!VMs%9P|G8Y5CA_N5<14E1*9?olBv4=P@!8_* z0F4+Ahw} za)|e3HQsRYq_?W2*$RPwX;YxB!IFfGM!XWGN|2%LCBiFP1GOXJFSEIw2g;*BG&%m#;)nC(O^SeDYUSnoJhRF9@ydJxsVdb8h zO~wY)+f8kFHQQE)l|RrDX!Fz=zM7Ve9)_sY6fb7FVLE;?U~JgpCn{>_Y(qwl2vGE_ zAWg8awWYpshQ2kH4ebyYRtDlE!sYJF1a5N~yvtE#AY$*N^luF}g~m#>7UsHD(USzJ_AR$R8C z0_WM}&yH8JY^nNo9$vv}A2Dk|Y5%oA^A84|8?7rV-<%3N!zR)Q5Bf2nI#X=N2~5ly~d0tPQt7>dKl^0Vv%>zKX3 zs8Ob$?-;vH7fyXs%|==12($D>n~l*`OITVeL<4A+wA6VxQ0LFQwz0m^Q>QBaWv+_lCCisBG?uoa;$6+YN2Cq$*7#s_ z{IF$4U2Dr(eq`QX+n}wuvgGrYx7DwSL(VPb-#4ga#Z6gG>|QIHNLtk*}c=9Qk3ro zq`7$$nqJi@P!+ktb5)?x=V=D(Z8>Kf4K-~>ORL}5;PJE=wJ?}4{H-m9zo7KCRbP4xdYxkl9s!jesm*GuK7X)$D&ado4=!Dx~~GSecg1%l7h7HIMtjV(q1mY^Oj zRULS3sM*}u>N6Va!6-zgO9w991|t|nq?=DMv(0!M3KK5taJit(Gtl?)7+2T$TELMK zMOy;~1C+JhZq>Fz4Qu@%Ow~6^ihwZE`+90P*oD#65+bv8TrDZ zYmQ9n11+`eswN?AgZE4WJdS1x={&f!bP+2Q4f)vEpg|!J(pV8VFB2I&R?#Vc{d4 zFs0dc`YLrTxd{9SzDx@Nj%u3P&u92+1>>5Urp9ZiWUB*Bn`&C-&c#cs+}o&a z9v>9GMJd1GX{p%&DM}KkU&U)G(T~^=*jVNBcw2p{i?X`8ro!VzOeKDyNE@mm7*-kT za@o8|m3m1FD`+hW#uBa6MUA(-RWoULb;nPuE($}JP(*o`w3y1M8HkFev=0VzuCa1U zo8QwcEBVa8F!H9nG#_Zno9bwlQOoL#jlLRh1Nz*?RRK@HW6Fj7tBoFim8x+Nj%7^p zEcu?KIl&-XciKh~KC&7mzam+Whe^-E8&;rFzkE6vy0 zQYfS-k;C}&j2uYChdh`Vk?-@2oIp!seXFl|R?1G`mshgWoz# z(=?n5)o!V6Y>K2qUR1uoXN})qQ`^9%&O#cNAMo2&)9gj%H<8Tzl$InInq8ey-L}xped`_M4VdaPa91-`tsqFO}=K% z&8wd|Q{%|)M>EFEMh%*X%^uxCQJIO4_^MHqc&#n4nCPR}kTjo5bqgb@qT7aHSwz2# zv|zPSP3-N3Wr|t?%^P4)(PuK-D#PmS>6&HPb8{TvHK$f2hV?$&V#)@~W;Z{tNkVSy z?UL2rjnpEImfq(zO{2w&;PWI+lodZL2diU@iep4%KWvt4?d|CCY!g}Gh*o-*0UX{7 zu*+?9`Xf$^jb6T^7SIvvlVIi*UIncMrsfoeNg+(7Z3D0CkHU1Iy zpiuSqm`i5RiZjR7q;e33$)_eWnN!l2Fe${c_=-x;HoIz^G({!EQRzg@%8o|Mia)yi zqezAGOb)F0QTfp1B4&R_VMGM(EY(@kW_Ym-#4cZEnU)fzibI-nK0*{+jlMiL#Sm4s47o)C^DC zdSmfQU+q$i{^r)UZN4bcwHad`Fs9-ean5FA@p9SnUxX}UMtTF+l(t~4YqJNl z^F-k<7ca{iTN;}K%?7%d3TduUj^!yo=2_S&36H?ZpvUKH^Km@_PA>>71ONx)|jCI5o_7NB9uo)YT8F(qy}qea+anM3lu)gP00~AnV8dP&nLS0 zSUoamfZ4}4Hlo|r0+V6(fY`xnL$4YWU6MxI+2Nd+xqyk8H`g5Y!Q=BlJZjJi^DE~= zAVS|3BTPhPV$K|lHaRY6GZwF|X$p8K_gr=q>l)is`S>6%_|<^coEjNP7&cyV=NiQv z0JHgM9tu^4|aG7%tQdcn`ZES0ROg-(M+JIFoEE?)4EjH&qFd!gh zb0Sa;YOuDZ1lod`nT5t`Q`f7=-t$qG2w&)VwrP-fH-sJ zFjGYyzZ}tzmhsZYO&)ZmRi&%h(7M?(8*^%$%&Kh)VC;!uz9ImDdriXR3WZxC7ojLV zXd4tw)siNzr9eU2JW@|7983~$0+PZCXF)}#RAlON+C0n;BQ{|^hMjb^#x>jU%-uNG zm_BD-?(BKz&7QYlPTqO5=g*rn-DD`#6=Va3nQG+Ac`2}Jj)rL+E^1kKZW_%Q_Q0^VXG#jY&LJn7O00Zd7R8-XAjfnoO*6U zQMdR}FZG-ir$K8>`)msPWK%GwW;x`{YIwHB5_S}0VGOF<=dh-lztQH+W*5o=2oIFa zc_3?$DJ_CB&1}&0Jkf~n*-mh2ka*}-!kC*O7)}Rf%f;Pt2$dRkCg*{3%(^A32GvE@ zuCwLLp<>*QZwnUTYOM1|+d#TA+i3AnXwtrs9LLWbrw6dOVfrmfk)~;8Za(IuXq!m3 z2%~n{h{H`jWW$5)9-ELIb7otbLdkV_%0O~dEUhMLKU0@7lt`S~k-yT`8aXi`UvX7L zv*gqB%#tn__9R(ji5bxa8Dm^!H4JR{lsRHb;id~qI+90SUMp?p9XjJ__WHNTtVPd5 zVzsYD@tZTV@obg*Qkp!?w`owcSC~zs=^x%GsW+qBZuMUC5ti$!<(DpBaoKW1h9mPd zqS3o;78=;E6Ht40NM$BgLk)zjTZd;l9>{WCJsykh~QP6jKsSo za$AH%6M9Sr3QE$eXvZhgF+YTy%l1`R~(`1CtRC1V%xb!hlccjDlSiSnl&+&CUnr znOPN)g*b z=XFd4vC}vecT_r&^dn}>qy$HNuNj~wM!t_43F|T#E|*~=)i7oxy-`M{VeIfeD7i0+ zB^cTJoNevzfV@U{AF;nfqe$6D(J}XVgi)g2oBKUZ{+?!N6h7p7759DUDi=v#?cPmYQvol&tqm(!~GK2yJm|&^6s*5a>MY4?K0D-(RA&*%`9re(?8knGmC3`yb*Sz z*%0k~oMd-eIEJ-fQfd52_qgO?$y!@ri6?80(sBD%7#o@>GB*(FCf4i-QRNKK(!)(Q z?XnAGI%Gcv?^$(-XYNxgDqdYuSX{+@fb6CTIbQ(iUa_*WYIVuVk|m|G4=DYGE8N8u zYNw&{y9(XKYHMeyYi&vC<>rn;9SAzp<8I4x0`_IZ@NB6F0pgsND3}x@3_$+4x0`s< zWBbGKzH|#OPUDeI|3v;(KZXy$@78|C>wf={o{yf{ebJ{s`S6vp4doXLfe*V1?-EeN zb4`($;V;SA)WS`TRd|O-bl4x0@7%LuB!7K?OlJ@<2arFLZ|2X$E7y4KBjAza-WiHs zKOp6hHq*+VNpCX3o2Fa&kcTVkFz}$e#hfT@K6#!`rhhdc1U|2mWe6 z{!Ds=1=B1%UW4F-H=wUD6wX-vH##q(?a0hR1v?7y*y*dnQ}-D0ebfO^BaKPx-9B40kWI(lz;G zI^qJ09`CZqgujE~_?h&Gam%?D9`$DrE90tzcr)a^_*|v@y3na@E7QaW?QOJbcI zZ#VD?P65w*ig>xeJC*V{G4IsydQTB=JMg0IU$kCHJvc>rX~5fl3UcYsJ+*x70$%kg z&|7ng^o&!)3&Gxu)P6+Up&sBFQT(x{EsnBRM<-Z778HI2wD*M3_OA!=421J%+5@H-ZU3rmcqYA(@QhLLdS^%F1Ac>v;%Cxh{_3Le zw!>8uV}m)s7P3rY&x_z261*Xgn|?X`0mRd@vpJSFy5a<=_Vw z-v9LR{}kRozU&(tH!pvcb)<0Ny292v&pNd%sK#00DbgyeWwx%{(AcuhTI#6qGx;^GEi*wl!y5=k#J zDvW_fR9GRxaw5Ydz1)bfa!GGNWSGP&hzP5cc<#tBiC2!W_6v|-A7z4(9qQmQhv}ao zPcqQv)lqH|XT~+xTvNA!N05z)YWy|V=dYbn;*x3PM1-wXJln%672bk~FojouFgKK9 zZoA@T`{Ee!9@0_%8SQWi|AjH)ZHB2b%wt&jMYOx&Fge-ZfG}HG=M#_T*<;X9>bU1K zOrS2T0hp_jl(y+U)6eXGDMs{AuQs&Fxd2g(^kM9w$m+y!hw2l|-B{mP*ET~P9oVV* zCv$ymzS^Lzw zKSLxrCJ3h&7kKY=h+VNU;zrPA)Ymwp55DOu5{SK!?IF~>c83`f-<&=SRGmvgZ-gs4> zh~EL@_aGh>A9;-mbEGGT^g2A1W58FuNY6|2Dn8=Lhu4Ao+k?ka#LZ79i>)ZnYf+wK zE8+%Xy)o5}8xzEhl?ljiBJw*{q*frmZzqel{Ym2fL&@R?m5E|peu@}ZkvN#p8(*E3 z9Vaq3qz_E+j;~J3#a(pBzZ>~y-XY#FWZCj(LgiCeJ~Au~^HRu(d|);t-VB5lz-|8e zL3;r_1!e%?5Yw~y3%~=@{y89jA8J42^7ltTy8a5t^gh(#Df=Aq`pK-s7|N*J%tuyI ztVmiiraa!3H^;6NvFo*DQGLvbdAme${?kb!aY_7uvpTjs26&E1PBE!MNyITRUQ9$e zB%vJQ^J2W&i=aef@Y(}N{})RjSNLlH>Ho?lxV0Gm9|6+8|5711e)lNE~!6ECJY zYMf&RVdqdjtTO}C(VpbRi@c|+oTvG#oe6af&Yd-A2REP{z&xmIYhJt3N&iE&$ScE; zb_5Q!1ULdT1ZapT8d+z=p&gqeQXJ=qcwJWukYPtU>@svC9X2+-BF(GX>I6}JHQGcx zA)S`ppS2`K%=3R$%&EIsoSnZ-OmSRq)m2?j3y|k`Q6Q8_zn1;mfRu3vkTO07$TAbZ zLnwM5#JyhiLPWtesyvnh5;x^qRUYNntMcgDs{B6z15tfJ&S+BJB4@@&$`o)xA9If{#d+-dwPrvGroa47Z{E_6UXki$IFs(O^C6#8_2e23ts)?8_Y*6GQ(2-1h#OPAA%LkCO$y6CM9@P_m@YuO}(F zehj&uaOCbnUBzRZZqh&Mp#LL0`3yroqWuwZq+E|et{+3LQS_u-hli`YClAYe0rSqj z;ju*IJtHRVPS`~}4~Kb6ilKkO_hAP$Z_$2xoUGI0(8vD7VYbW6f3KPULk^@NVtRjo zwaB}Ln0AKBtKF~ooG?uHap#uEJ~LjNc`Pkv>TBacXKKvOdxSXWN6-mP$L@EGi4|iu zDBYHNR0(~_PZEPaL7n-j5ckM9)IsRzuyBln?ReBf9e?0{46Pm*9zQZ1_0WdrNI=^N z8z6Q1tax$Ou|zTDP?CVI3FGHNd>c=$p=1%^kHYQPEyTxoOkBHP=68ubzfR)N1?@T6s{eA@k4g-IeW|F+m{$BtOb{!&`1U)Kk_xdh`L+C~F*lW?|%3jmQTQ3S~L zq8u=7kHYQ$6$0=({c&agF90O&HGsrT{f%lbt^lOJ2axZ72FNtB<|kq)I71k9=_vn+ zA`@e<^QS#H&V{{55H4JgLGz1?2DcUX}hKK>l2NHT?x+#aaF(;>(OJQn~K z;4y!{*J1Qmzo2L}1M=tB{w>=7GvJc;eytbpYX1k?Zy(Q+KS!$5n#V|IAJjHSz)4neHb#ea#*0qS((@)>HUEP7h)Ojdq4-M>y9b2$51#ZdDE{nzIe&t_O-9A| z!PJ2iZ*p}~x#RRiaXReH>G>1I=?_gF%ov#1JKmdC?ilM3W0#jq0+O; ze<}R+fRYB@^Opih8g_kR`Xlir5FVZG+fnJ}>U2AFIvM{~>7EYAborjY3r9$o^z|HE zb+(+(#C%GAf+#@T$MEFjD1C_V+wcaCWA5-H3(xMCWnP&q5`E@;U6Sbd81)oS2kFYX zktqF-AnX>M7T??b_Bf6>w4-QyR9wja6CpO@dClb0=66gQD<=8l#H2dRAyvkTN%=8i z(g4bCLY$beVVpOmI=MV4H`4BSlQB>HnUb4K!;f)3{C~n@&iUB=YMv;Gb3}uvy}j9- z-^4stB6OE>G+@D2;ijkYHsP!SkViuI*6x-xapX4^rLQLzAF*-Lqxv^ zEjzxd*Oo1iyu%ih$2i3Oc;3_L+x=NH$BK#hQ>^i%oT*U#q7Kl{nwcmjqc1Vpmo_+V zAh|!WSIuRxuS`3eg>?ze-OBkr$c?&I(5rNfup?ddfo=ul?`c4~-qYb>o8XK0sFhDMWo-V=M6B=Hk_oZ(LsXVj&l zFEmb^;S0~*mpgFX*>u>L=@U*LoHj6}KeKm|H={ZowlYDS-hg?d{4|kv>)1ig2W78y zinSaw;$yeovmN4Vc(%**P}>n65hm*!))}A&1+yGtJszne)HUdbfS!oxcaT{GF6){< zUR2L^h;BR&$oX=n6&;3l(uOmhd8!*HPOD6Ty_9u%&^e&y<_%ru)vzlpdn6gw0n^Vg zW3GCCEg*k~0BO^PbolrY=9^9=DSh&4+SDiGTt%k^kiQ=Q()EN6SG=Zx*JSXDdE|=t zFt7cZUWbidzm9iquA;XJkS;GE`9B%G`#^7}#`{3W&zPs;=K?Z*)|rXoOn(yENwhm? zZ}qyy0P9N!;+Jc@%siF;C4e&Bc}Zek{sQRxTzuOT0DlE_+HIl=6prx2Mg4^dyzxU!vM$b&zgsF$)6-9J`_2MyJ;`@McNg>C%xXN^antX;iVU=^y^Ag`ZoYF ze7pgBkAB8f*Z{LH$NUUn9fa@uqv-9{@scl5^hyBfsski`*7->yC;xPj=}5QD_r-?C zn|na_R^WGN{42^7oy~xZKc4yryK_2hRaT|4bMd#Ty1Nr`bC)Z;cL4eOMEg$`uL1Fl z6)N6Kfc*VK`^TS^EY4~egK^?yltGHy+p}h{FhYxY(UBH z8HwTy@H^${0*H#ymG`dE>rYh1LW@m?H_*<`Q{*A z+8Pz_X+Zwo(f;u>lEsXMILt3h#Qeep*w*nP`5~)Z1>(xz45IXF{BK;T z;y(cRdHPX}xC0vR_9_+cLBP+8w*c|_HQr6tD&F@2KQCSy;`M4ge~pUwb->Sy7a9{I zc5A%44JzIZfS(ty7x8*DUS+L{*9ypZj!cZ9IF3TwmWjS@W?uT>gn=~gc=Q`*qTiSv z)o)}y-UWJ$JD}U5e*u!}S$(Qcun@4iUeW&mkiX203TGK0HKhfR<0s?GnvZxn)-IR* zNsc+OF0aN1=q~}M4bLF@{@I9Q_h+$R7TGqdJqO^Czw%9r{=_E5$3=kT{m0sWQ2QMd zupgo>1MA4?V#3iRZ+05`dWR-rZ^Z<$v(+Jf=5>e$>;Y%`cE6NUU6#nknoh=1HGY7- zQ|skgPG;Xq_NT8>=~uNWe!PHm1p&#IV_du#cPsW148-?4dolL_T}eQnb2r{k^+N{a zfnoUIBK>#Z5$Ef$x(U+Xhp=>pfiB}bU%om}kvhl~6kACa5*h__HK=U8z z2VxYCv`esQ2Fhs~Y+Bl_R)2uHdIEaIbgOMVf*nzPy-NS44#np=TNPi$fW#j^4eM&? zgHA>t)Z7a~di|ispI77U1LSY*jVgWsP{x;i?veYk_kl*Q#yh`L#lH%Wu3G?Q{FC)N z5r07AedQ(<|C@k}7aosQnB(;X(TjJ3cye)OBb&7kV-0(lijyJ|Z`E}Ub1i8%W8V&* zIvEG;z0E(~NQAt{p>L4FJ`!8ae$L`kK5MjmdcUgpxpceY?+rk@jslX;&+8ZEg3g`5 zC$864cJyBhe@B<1bLI|3|F7MOzWAC7ult6wGe5pfh3^Gq+^i++mrshxb|lAGnqu|G zc40iUHVt!H8vltNMgQ-BjDOJ&l)oNO(!=vOGBHQDl%_XZK{due9R$cbl7ZK(=fcM|SGwXn&t9ThJQm`k>kr9WzaF2@&#~u++KF30THUM%B(5G}o z?w`96VQk+}?p|FPzyE$`PL|jRS z&b+yGO-)$KjzPKLqAGqyM&18jQis`MCzQ^&rQEt?6D5IB0;BwcRHEYG!`34`OscNmSk~iIjUzQgyDJoct zT@}PoA1kbq7qJe*_(oinWU-1DfmYRrx!V~LUdub=!I>ZQkh|8as|us&xslw~xT-RL zJHE9cmAk63r3x3w287tA!g*1bxJkKrLuweION9kmBu3fF)rIBr@^e-0FBHw5=4M>C zDHhUI+u#%8Oi_jJ{Cv?VpaC^?;#`Eac|4mKhV*Ni8aLu&JyZ{mxLVZLv)ZH2ZM|u(;U#a_0Tj)J zmVhV_Rc)SzDqKs5Qmewj7SYuc~+lJ~7%>;yxYKXU&pwFLK^2;(coZ9jRpW4h*PQ0I zIruQx9AL~rb!=>%(=>0+yg91k4tvk*UhN3{5~7`g4;BVjPTEnTFd!9NQt3Yl2Jeru zzm86WdUdk*O4bn=X*Rpgh77k-!?6F6q3VM?m0;S{#%nd{5|yzf;me;_mZHpYfl!_J zxwFFGR9NF>?*um<@yx4ugkhz)^)ui#KiE;+;^)UhpWvI-jXqv2x~c^tMoaSL*pfE+ zX_<}ee|Xxc;;2ntDvinx^?|ERCMpr&)v>eIa@lWH^bR@PbRQD=O3(Zd%L`>QAPI67h;jL+|6F<8f`eO?aj16wG8|3@Ba@H z;9Lkt?Rj^`2=?14}NQ2Q}-4qAI>0L z>{`v{0`{!FwV+_c_fegtQRzrLe$!c+Rn<)@ko~KX(+&3+OeB-3|3XZ>W6vKa(K4`4 z4zN|bc}C&;+HF6VY(KNiv&`R}fRPc-GZXH{(+)S`yLh(4JqY-ZiP+~&H{gruIFA9h z{roV`6<1;#mVy07gg!j^a1$QG)~q793A4FH3vT;aVV)r#I34RH2qzrEa}(U+3?ZJv zb35GqfDQxa9N0ZS^AT*GD+vdw8nh9t8aQg(!E@ z0ep2ObPsO(IZpc-PoDjpdO7&S`A@><@f?Gj@Vd3=8)iYafIqkbyuochr^)l1zgvfS zUxX7ri>C?he!xrDt8(W#%c)h^yNz%I@OC^bBc6xcSpyN^d?DeI4G4#u&^+TvxULy% za|kEgf~N;=!oQ$5yBlu8Ss0TsJQr};X6Pi`<$zzhTIrbm>>|%JZfHkY0Eh4>o@Cxp zw*~9yozOS9c?NLJ&6q2PyBhG;F32Be;d%hg^LX}icRY`GYd7)(9Kyfhxd(1S=hskQ z;7$WH&)M0})A5{L=hq=~;1J%1CjsZr2p`0g2{+-N@#MfwICCd-9`0N~^Sqq>OdQX~ z9m3NA9Kx^O3OfTg;Z8jF!A)qMf3u%+>1CbrT+kDMu9_k2cA#i zCj2FyWSsHp1uVK-oi!tDy9f3V;e=nqQwq2J+!)W3?fVJp1;PpcjOTW^?dQRGe(c=) zp<@Uq`~jXmxCswxH{m;Y1`uvP3&yi!7yJx15aEOy@FW<}H^A4ln=t9;Dx7cv9s@Y` zvtc|l=G%>4Cc+7?!LtT#`79ueIg3 zuoZB30G{!rvJX5*b^l(_MEGvNW6$G`zUiosFQDzi!#0R#Pj0}odj{eHp7)}v*9Cx| zX}5Suh;QN{eh=WK{mRai13vh&a_v5<}N(f}+=SQTSpatj;88q`E8d09 zA3`}I+za^rUt!1L9t14;n`$e|0l$LBk8tsKw7(&hcQ4?V4x_z5cs1bNc-USH08amh za@)_k@C?jVM<55_6W)yH0l4jFU3dm2_I;Ee!U;3+^uujG>%ucIx8WH;IN^PGjuGbr zw7+=BgBS2&?d}D14yy1pz`uV8|5?Z*;MMlnbCZ z4q3rX_!b`OmHjLT&xqWKXAN)&d+>PSCJcOxItJeC=RE9ZKzJ79fAI7GpD^YVq2@I5?(a1R0={S>^O4L<(|XL9kdtaAZ3YPT2gF756C{3{;z ziv|HN`3y2a+;YHXJgkE}EAn$ZIS99(3*mW@F+->)2q&D3rxI@a`4658xnDShAK`?* z#kw(?cA<)u-c}0NnO-6+B;2nT>Q2PPh@zr*PZPNbu~$ z7p7y+&@9+kzzgvd!Ce5j=S;+hyBCmW9K3Mb&pPlN#J%S@#CH%*$TJKNGcKTc?t$cK9TRejZCwv`G!nvp~fIlq3dOY0R8-DdAkPqD4-~HV(oKJ_l2ax-=$tPj@ z3I{47(gJjrV{ZW5++SVzCB%iBd!`pwU@rjN-2Z%Nr9*_^=APz1RAS!%?gr)lo=wp?DHNZ`%P=U{H(fHF zE_R8X#fK?HGU>K}6`Hl_+O283URE1$BhwD1AVw5?z=zv%cUIO)w{=?PHijY=LCA|C z6_oYGz<>x7hvtPtKi}M~+{R=t1!>^R=bK;d{<-_-E_dyHbAPi{zS_gG zt*4*i);^XEuUzQHU1a?5!5pXZowx4hbEhu0qs9kWci5C;`S3CL`mon;!3?IdnZ0{x zlWD{PD&dRp6O(?x2lJD`3}i4LshniT7|Z<7$~rso3B0%qb>PJz)P)~`Z7+DsqugUB zxy5nn!OWtvjNhXwb@6BP7hZgJAKUW=eyaet?f2@+C1ww>zSNa5JoOT7;*}Rv=5G!! z9wX0JQypH-_KD$xEHjxs@oAL8iy7aGW#5Y{$n|NOV7^bezy0L*MyQKt(F9(ZKJ%3o zp||kL-&NmayTU6gS2|2P5A#{W?~brN^W9Z}@0Pv(3Fh6DgDW`f3rsJ*jn3nP`8Va@ zUjCSEow|4!)$qZLo3d~3qkCFc);Wo9#w+XgD~jWTxi;n9enBbf;-9D&AI!BW?{)>{ zsf%|UWxv1&b8X7I-8MxIn!0!wn!-n50qIy%gz;(ioqO5N;RU2)O)%%COk56~W?u0l zRKbg%qKo($sN9?Wjt=JEl#6=EZ2`tcNQL22rpu!QR8wD1R{X?}!FvtGR!{)Q%){xW>+B>M=y1ZR=v6)*UH z5k^1r>M{5vQk$Eh>DTk{Csd-HMcDp1+duv>cpP2C&%*9GzAF;E7d(p8&OB^5#W90= zBkV=0=i!v^#Xpe#*HnY;UwFO)7E%4&0B$)=PLFmv;n!$AUir1#&v?@)JGRsJ%B!96 z{dsubS#SFf!#LW+ykgq-gYf=ye6Ld1Z!(*Zma`ciKwZ=);4NRW4)_S%gNk?y9zkim z?&qd^zZKc`#1Frs-|+f9y?#HR;#^L=;``7y>?_FPoG=E{D9rhlvi!>NSNR@RR#I6= zc~RNM7wL0R*+tbwWfSGa?Y`H&Lw`qFm(aHyXOVmge&Bn~+?F3V>4DX}-hdj><7gD! zjW(cXP|D|apU=ZP(emT}*aPgNtQIO>yLMilId+3Ox45d`4^sC|-N!C?{Ugyfo@1PE zZy?B|v2Jb5V9d;(VGVYeG`4m=(GoWz%4qdw2DyrzGzwYMXnUk#bs}S$+1w6>*rb6W zGiea7Xf=EKv$>4fk{e(&9&@-a-@=9T){L23JGRbX1lOT~K0++pg6~FCkw_vxy4TD< z#ib{|wKX?s?SVD*`IysWv|EO?E%?u=)U!5(0(u!cd5r7FJG17}i@}$@As-pIrb8n! zd*d-Djf_d$X?S;b)QIdIc(g|srjtfrX3)$Uk?2x8iRiz-OCoyBG9{u*E641K=+%DX z$@L$`jh~XtjTlSKEdNr%)K$iO}@1F4Fj5{a$SP+w^>gFyv? zlR(DfRH{~~TA{7|iG4|bS_QR|5J#VN1>Blu!-{Abw)i>UFQ^5JdYn?U08=doSbe7Gy#CgL_*Iav6cJ?uO zR?$zy9&0}EaQi*+zr*icde1?;Z*Sgo_m}c_&34VQ!M1sOZoM-8ZJ+HF z+tBpkHk$(tyiKL9J_gSY`RvA%dQP|52C2`9Ut1Txl9>2sw^c5U16i<0RfWWVn9oY# zqwU@6(ro+q(3~cVvaLw7txe6)nPw};=W=hFt(y6c@UI!)?@j&YOtYcG{g{|cm{Oxqm%j>d1wKR4LtES1gX9*ht5_zqgx zs{7?@KC4<~lx;*?Z6lGdre8kySwEgP|C(Bxtr(weYpwDc2FO=aTk|7i>EfqmE8nt# z@?CrVO~^=E+B&U#P5tss`r0Kb@+@n z;d;}jO>FgpDnrchAd)9I7>PCM3;e?HiUeM9y2f9V5y|6a|KG&g|yTN_n^K#N? zwlD&Dm6GGyiKozbMHG1QMN5xeG&XAu z`H;fG_$fyG^bD|2nax&KJS1=;I;>E{Hj08t`6ASkC9}IaYyQ#Q!-^v3?eRD6*BYXD z@f0u4)EZW@(uhbCCDW>dXNkxYE07?JZNk`Ne)Ux>)_|&tP?fvs*4Z9oP&m|t??uhs z_dubV5pt_=1_i4ZnR7}@yzbqDu^`;^OSa}+Ge^OzY(JnbU z&in<)>@%*+*CW$1%8ExQw$~vHQ`p|;Y6jb7I4qIPm`;^Ej%>7q5bU zNkLO_+Yj*_PwBy#sz*@7aB}nxDK#iyCgp?bo-|9!1itGtrh&+M2K7N%ahle!UY0zd zjCso!@uRg82S@?Qn7Kf};^>b+T5`1Hqb{V&E_oE?p-b|$hF{^y7x5+-+sy{-g)B&+ zY>6Zyg~lorkTUXx5pT2)lDTNRx#3e*{7~L%;v3pj?YV01RjvNWdAnq-7o%s%2!Vn` zkD@l&m-Bkp*v*$@|W)uZbFa}q3dT@Gz#HyD-S4cC^m;#z?s(3l@oj8eCfWCnm z`FQ~O-3QNf^LZ(!q^!87Cxx=hfIr$8ys$UA9PlE1s~m;NLBAOT!XirGS0qR@gPjp@ z^&SXA0v(7!cLk=~Tf+rgp8RRJ~nV{yt$G(q=VlXYC6fA7{jzzYReSW2gBBda}EU z!zcVu9tC01>)P4-YR8sECZS)TEpqNhPV*^b(`KDN?vylJG)IJPA=$2wqb;37rM)Tl zGtH5`_heYqvzkBGu4vO{?VhKd-GawGwG%$mF3))q-%y&L^+x>7^K@--r}+u&@8Sw5 z)14fH5pS~yZG#2eT0bEd7I61^q@vu+CcvV!&RRE$MIjCp!jf4K%HE|4`%S#C+fxc# zkt_|4r%gNi{7ERRc8V%!XI8I*I#ofJ#|tV+DJX7_Cd)fPm3IJ6V(ovs1gVB|tDe*45+kPv)1(IJf2n$m@_ z3nl=}M5j5M(g?+eHWgl(aLlZ6LH`W)bs*0nO=@9W4)9NXuGZQry~D!kU0QTkQg@4j zHk!bNw0?_e{myMw@AT!8X^BxQ+vkMo3diDAs2xYm3}FnR%TmfXNzH`)E#m@%V>uc2 zG~0G$>k+P&8oYl2bhfi6LI<;I@&2xS4;?&h0bX8_FGh#?{SSeCJKPzzLQ1#xNhlTN zVw9fHW~bD5_M(}aWGWQxpeuw+jO24eU=TUiR;_{7pG20JmJ>}N+Z)p>Q2B}!OsC5C zB&Oe!?-r)tRhc`?d!sf$#Zk4eR2CQ%u$>*#+Qzz}8@8GEfrC(oB(0nMJ~!U)8>N?? zATQ+MoIHng&xLUudY&CShe8rPv@lD!-U@(Mr)8M|3gu91AyR@L3uC$?>GX(#;q-Xm zZh`rAxLZE^Mq!LGe=XS|B*K%yXOkW5ZQ1cxNt8g%e4nAi&1wcIkBJ;MXA!zYj-vq9!BFhWK9S4&u8qUBWb)Sy`ox@gBXeGRU55;gb{+O-^+wCG&&Iph#@32Gy2dDf5z&2bYjXdYb8VX8+_YX zwJhM;H?wx^d8N>{G2U3I?2&D zlLywNkxV0HQ#cZJ$WReykO$?*FTakr6VCc8=6f_Sf#nUV?~G}V8gPeQaYvL5Q$z%J zuyv9<X5ehArY^^_;%{#gAXVo=?;H~R)V2p zASrj)8Y3=KyL~7>^-3o9>vlxYHfzhE0sDLv1*XrnS@O)iHqz|9ugF@^&IOw>GQnN@ zEYiB#BxSFXxz@>CkqL!@{#==?)Z(I2nc;qB5U!oIZ98jh97Bg#ejj<2Jl zCzF9bq*YAJwc9sH4Ztzj{egUZQRc-^8cHDzX4((+NI_wL86v^F2nJ7rzS`}NEAZ~$ z<5xDGrbef*@P8O`A(qo~h^>s{-#~~scPx?_USS3)KE_%N*i1to40466DuwkwQJ8Z) zfc1%X^a6|Z&q~EsO0SpApMRc$Exkue^GNw{J;Q*l2?^s)2#G%YpQTI_YxY!Sms7h(9YFt|}KC1H391~up(o|2o;hP@Q`2;kJl%yYoc zQc5MHUR-9SptGI&N35Hv{$;}5r+x4=={GHs;bt%{p}sdtL*qe7TVo>R)OVmh znW%rLd{5T@)j{^XRsYBG#YogYy?_1G|0=W!R0^fMK8SG|%qV(2I#aBRV}{{G%0zlt zdSuwk60@ZTr49BS1Q}0*-E}iETrV@gw0iU%bV6kIX(IHGE)ky86U{R3Wsh4sDEL6Me1q8J?f@);DGpx&w0V!++(YXllJPlpP`TuRNJf-V zR5xB=R*YkJ3y?Tz=vY*i^2jo6Aq8os9g+g$N06YgoWDZf!E8ieXd{%=&F6~n4=hoc z15w$Q;Q(e9M@#PE(8w5-%3X{KU)PW-%L@Z}XFYSsN>}_)3 z^sB+ur`jaT_+Zh4`!8ju`SP+Zu|mVe35!%JT}&e!5}}=%lnWT8hoa|0&mipGA$D~q zEVti_1MiDnAHXbHFh;buOM8|mw}1_A&VZSpCL%M^MB^4P!+wYbtGiE)clV%J&LV0< z^!EBL_I5RT`#tpbWcK#|Fv-{##iYlayhu9`}wihWpz;6h-Wl0`so6eZ(}OdIN_ zm9)b={SyLPa(0HaqlYF2qO$l$s;)~|{OR&NX-9ts)G!c~#s5$F0z`ep>gZ;4xCnAF z*B7n@H&8YLzc(kgWfVcuZYpLNDDD)<@r&Uyq-YBaPFj;<=^;5PP8X4>7!_MNDlYpH zxS038MIs3h>1_}Rh{qv}t!5p+DZ@0?T!6iRu9ltdqd$rMR>Sl)5s|TrgXSIY(hUjZ z!HnMKS?9E|ePwz4MS<`JvSK+q(R##35b+npu`r1~c)z7jeVDJi46`h<9=~xA%G+nW zgMs63+!B~4xrn+Cy{0T+pR1>4hR!6=NLs+UYv+PenUc^CzJeZLCycVZYAO=&a6Xia zkFgl@8L3j|fH8oXXRc=sCjh58bk}w{PM?K9iRENNOM({HtGSbEAKF_fjLkkcEC)LM z1z+f{{kk?~zbM#2IYc9>R>HjieFe8*KaJDb!u4^jE5TJjZm818R(q5xBNvVl_FZ7a zsgYq$Vc#R{FT`>-p&+G?dqlxuVgE)neiR%c4TvqIbX);{4K74~vs$Tu(N5`+l`%j| z3*tfMc70hpCxOVR2|M)t{>u0!OBw%|%D5k*$3nE6+y~R+<0?9-o`GvXV{3fIW*z3@ zHaL?BnV~&p;8KAH&z=&Gs4Bg80XO{S39!}Il-zev5uFSMvW-7TZw>Dch zUItywM-O2x^8`P7bc`>2oi*eW%u8#+^-b{HzWzfZgY4JGV56Wn5Q%LEr3<_7!8=@= zKy9L*kH~Pb0Q>PAzk-BaDC`KRT+HBeV$LHWD=l-GtaL%|D$Ost;S`-L>;0xSc(i(H zV2vRESQy80y7r;QxX?W^wpzpAFk~%^fi0U0jJto0RKM%Jn~z$vA2kou;>vGPzod1k zsy~i-_i1>C2YD7aium8WAU;wC9^eDl{!XgOLF<)LikFl^_6;1t1+v+?{UuOO*k6$o z`0>00)yg4gQF}897R#yWQ{O;PN21gP-vSnh`3g1%Mzf27q3R__vO^z}ub|Y(>dC>+ zA&0RJiOj({8_F&p%+TtC2;k_?_gr;V1(cq*Xp7I-?L#Ccv~jTB@_`*1&aX|ujL7c5 z33@>X&7r|ljf^xsjG4gZEIo$k%hn)7L$lN~(rP)9QaebcHk)B=NQ{w^8}^#NeX)e zi_K`!PA|(V1Y6_9hiRH}SPkj5Zk;L`LJVxi&*SQu8K2SugFW>VY-a|tXqRwphYlYl z)}n`699GKnEFlPcCjcizO=Dr&-?I+W)xgIGQ`P>au~q&tGgeEb!L7tY4&gi5pI3I;nP#%d7P{Nv^7rHe1)W&z1a2onadiQHCvJay(D zqMgCXkefNeA)$WC)z57C^K7Mht5QGJ>ZeZq)XSe|o7B$=^|Mm_tX4m3_@ngcKn_ax zS2LlqlK)gZKVi*-4589hQ&Gm^L1tx+jOuPPr;NlysB}5f(-$2NQ3Zd`yy#Y`D0!k zeXyk;HmM=?cikFNjiiW2qoZ{2c@Q0}Cg9#3GTJz*%JqV*fY%p+;iuM@h|pnY?Rk3G zjiJ?2i+I$f9Bzll1qNXmSA0md)O-l>A!2QiAZkdp)WB+#cF5rj?e?b`(!(&S;y3{> zJe1cAtm4CK4vft3lX>+>qU}I-Fc{{iy38s)L4>of#GiAm+qoY3{MV)fJhlt$Og<8yeu4YwbdG2<%Tz(n-Zm4t~ z7&?8?NP5~4%*TfZLoB&#N5krkq25=(D<44_m{l-;*b#Loi!93#sL0sv}XiAsI;;A&sN=X&Vy$l!N`0rivlQ{)U{ z6o8Jd%nb0C;W?t`Es|H?y5>kt-q#zzgOrGMoAp@bPgdV1z0_3l=E) z{S?_qza?;4LBC!#W{(9x7bFF0bOaHbW{u@67jZe+LDTS}H85zI+9ofvs5VXHW4uiz zeCbugm8YPF#i|A^I7Xo~2HA+xhi?+4!09MDJ6No0KZZq|DakdMk0&ci$DpEmm;6&F zO3qUy9~*D%->Jz^^4+rJ-T;NC1K@LLG*mYt_&rq}^&!^4sm4Cf7gp%_Ugk6XDy%O6 z{1XcP_;^zuv?@GVHDxTyJ}!>TD(hS30LV-#L}Bw(VZ(xFDrA1fI-RoZM!)JmJs%L~ zDToAfUC(T)Bj)ol-Vv4k!0)~g;9o+`0K<`S<|<{jICB-p3*cBR>Q}(kB`DxYRX~E? zqmhmDKIM)6%vJ9KK&VfRU%{76)VQB7M42^y%9mceb5|(}n5YUE9Uo4+neDq)hrPfT z5X|Aq+}aP^b`iks0GDE7d~dO;g}SjsP)XJM0owgCULj{8WSj7c58bUu36<`eNTHnH z7n&ax^g?7eU-|-T#~x15%s$L^^_HtB8Mfg+-dP)IIa zMqjejn*OD>nbRJUV7*IyB~fa%EOmbXgL4lj*KWyTp=ZPk9RSVaR}gC%M0xzrd}LrB zdR^fjQrOYL*`UiPrEciNe^k7cs31L#5yMNImJpWV6(h zSUv_lc@nh|0=HOci%uL5!@-L{o$GTdOPCc>&?-aSvONp^L@%_4rI$cI2MR-_|8QF> zc&JjrnAb+p$FgWlmqQ(8r=wuB&wS@IXcY?nUS*EGl3&XhHbwYPhP+{GX(O;g-60&> z2px>X&^IuGHO%<87tu&xsPrfCiif0DjAhO#ZF(9S+x!XYrS?Y++sr>f9P}+PRC?04 zDCs$Jvlhae}je9{k$X{6av(bmuh;!GW$(hv` z#cfT8)U*dh*xwl{Z9=kb@i35Z7VG}ar*ge099T>dbFp14=ka&oSaA_yB+`tc&ACDv z(7)nK2{K~F9p0qM8$wQLs&{Ytg=L=6`CT15C8|x^gl9 zvn(Jb?4L8(bJ_}7VD?$>qPFYM94MpVSTZZKV4nN!XeOt-PlOdR6j^}!lO}S$zYi|% zx%h%Tsz|m*V!ZYVMBM+Py86=JFwhfNg694SJ**ou9I>2#bm1vH zr5K6p5{Wxhq8o|NCKA`HMEX*dJzqTFV)HqW@V{yxU61tY;iw1A-j5DodYwNqqnyhK zVdhq{K2K~`G02{0rR6CcW}sxWEf3bu7@%GMKF}Tyw7K3o)_o>uhN=7oBW0T&j*#kAm%Sc}KiQ9CI5t|vUh zDT`|NaUJXwCsvco2AxKnFa9{*pe`^tSKbvL;%L*1msT3t|NHLH9^a3xn)OI}!{x_0?g5XV~$p{T}Ood`G9 z6TNUFlp%jss-M;TF}#pE6-}a`RgS$zndqum0iOmAL_@RC%C?DcWDWARH#Xq^q$EZz z;wp_z^0ASfgSWMK!w67(H?vw1^bTo>1t@24{wM63-y>7e?y|_la){L~yqHhI(a{Y| z$cGy-fcXfYyb&S=L-zqEIWR&*e$jc^~iule{=_x?7y zAm9%9(U^>YMjICd+#=r#_Q9=`3j%&3-$RGxf`AH@8OF#Cv!#_S39jah(C?n6}+dne2sKOw)dDjWV)rzcIjvy;-L|0MLL`g6v%)dwJ8FA#R&Mh zb>minpdaSd)WSCl!81 zX)=4BR{L5OR|LbzyRlGq2Ls>V_m%vPwkmAGU(j@wT^3=AxgU69nSX9s@wE=WR6WZctj_YAEs1L=6fjVKSHkWt)$J6uPo6y2#YbA$bK*~{|VhCIFmp-9YZ3LecjvS|M%H5_sKXg z@S&{y5z~jO^!hXa+wYS>WBYy8>a7YtSdOHV{YE9px+2*RG3hRqG&@aJU#@u()k73dfmBgCZStQP%nf~cN zWwTbKq(8;EAO~$&}TsSV@-{E1K&~=bO-}}gZ))G ztwP1@MEo?C^Hbf*s!nRcv)) zsPvU{=nJ3fGt%Uy%rM;S9=8_SK=l>LzM~7(zwr+k(0^t^42zGl7cgxL(;h&YZV1dJ z&D^QmvuMjB@XET3*_FCIw=Cj51{UXF{9##{_gMUm!{2H6TOz_0m`|_SjqRDpP*Sl| zYs`d-gsiGYanKyotE{;L`$@t+9NAxqnO@fi;;mTD zaezQ|8bF``xR(wflsN!-AN|>c$39T+!=GW4mqC$XGA zDX4LP`b2BIP(iI#PzM0Dra#nySk6s=B27vv4rq<(3aD8Dq0@i1P69cY5GO<|mNQXR zwFFQxt#LbtVQe`EEfjEjQL``nTwVu2e8v(CmYJAW*oo$}n=h^+A8e3X$k-QfdW`#3 z{HdZHfXH?iKVemndHF4{1lOQ5l}hGrB9HFjR8^tafCf24_mG8sSlJ#2W| zOuu=Xrre?>C>43$_aLiz=zW~eq)++827N?53 zP!@H{G#2H(5W$IDA9muf(K7lkf1k|jtq$0X-5BN;P$oNDG}T4HzPL4iPLPKsM4;Nb^LiobNJuKQv|Bs&vZ* zYJw5~QIh2FFAJ6K{Vqv?1v0&dKW1Y%{0;bt5HDixRo#2vVl{I7Clgzds008esvrR< zW7#lZp@tB)#V7@PMGm6+DRfSoD za5ROTQ@G|<1%4Q41ZYouMHM}D>%g~;wyYW(+@mtY$Ub=SibgIdF3{2vBIXt5(xTlO?_V+|T2YV)dn z_U}j!-3N2pXaD~GNx9Adwq!5JLa{4Zdcgq^xh&0m?>x#FnS_81ybZHy6H}Hkle=lc zAl-PLkI{=2`mDey*$Uz?mJ{2CI!llg#F*&e%*mh*T8U0>Ugnc9OeVuGx!se!|U+uOGNRuLZ^dWhNO* z!@er45r74)7^^%~`poI1$eDc^t2qN$MY3<$i=U;N2ruVSyPteLE+a1P`{|eXWSI&} z5~jjF)|@mIII-O$IpL3aHgsIOH#Q8f&|Cvq5N9L(QT#oQzo+rH0e|f<6PCns#%`rb zrOscXHSUmB#avnyON22aVO5M1b{Hkh{^b@~3AH`)KP>aFs%Om#BwYHbpZLS<^2RP-hZI_Ox0rWluKi&l2bf1yrekx{^TPgaOc7D1n|< zKm`hj?4U?~mnnf9Oh~E82Oyu(rGWN>Lxi!4jA8+icPvSj0(2~Zz#6Y8sAgp6LX>9Y z^%m{-g|A(L==TPHG0=~0lUFD)%QJ5>g!)7UI{YWt( zEHNR?l|c*%MY6BoNxpu?d}J@iLuiRZ)0RDs`h3QtSqf#p6ksTcw z#4b-{IVV*5;i&^5F3lduO0GP45zdeK@hgyT;b6Pr9rqZ?6+B^0ejv;9JRWUPO?%2a zE^qYyz5WyS>!CLL)L6s*Dz+x#|3t)=HOWWtsxh60Czq9ly>H>~ef)ibKZtHVBbZed zzTqvi<~Jk@Iw#^sE(9d733nm)7KwhW+fN&8#7kedUYz!Da;5OKbMTW#Qyx z;yp-=+e=lJjxFz_W4Y)a_R{QFP6bM}>?J4dr7Eov@l|OrO{Z>k(r5yiIZYV3Yc)QZ z6D8NM2-m7LNC;dYA~*;OCQy~M#O16+B6r3)qzB7@t|+G&#OX?44>M@Pc#Bzj88xY9 z&`DqxnnBOMtjwT@ggAm3^mCcHcP-rwfK+BsjVh}z9^F4gj_qo_+og17P3Ws!@R%?% zKRLyU`gr%Tqp*q++6#LkEbRr?_G{#TYs(!Fat2xIb1n247plSjfeU4(qx|l-nIKKu z2buQqOGtYGX_yX!KRcT&9sXk;$1K=%C4DP0P`h^pC#o28H>W&9l;&1SV+TrRHu&BW zL6~$=ZZKAXk?#fnhj}`LSZOU|w(Z7N3f z!ctfYsE@{yK!YXF8VhKR0$PC;iZO~Ey%z^xk??lcLv&VuWpiDInkOo4+Q7^uEaTG+nlEVva4 zt}6v@@4#?ROSonWP95j4YF`SRIWXK33Af3DbE^8JL{dF|7p2tzbeSUIIxIL-nGcF7 zWvAao-7)~2CgHI01-+s*u2pbOV5m?U`51 z*PddEG-7($GmwnY%i{(ONGADIx@^AZ0YRR^Nq?Zk?C80n(qjk2K5o>!1ndvP(KC+& z>ZH!>(4tM4ZilHt=7^qScstfOgQu+NOjtN$VBvJ^@5Hi`eV)*J8G2|(mfrGlIt?8+ zTN^iW{7p1+sE3QWHeu!pqEyZQA>QGw$6pht-0>ofo$K?eP3|+}(j_QhAG0Rxo!46> zffM%5w%3%s!@i}xbMDxIx+t}g^It$C&)Y#TOI5+%Aog_5XwCrJHh* zgwqYN02Vfnj2ClNR%Jhv=VHu%;EkMH6eaNx!@aUObL9~xy|rp&TCQXa&Q4*StD4Kkeup8?2=HrJ zWa@gT>t;-74VN=}k*RLj$tWIcCJ=-=U9^)@7-a&HEaAZJ1jN>2?e4KmZg;!+78Yfk z)@W&SkHz9`;ly7){tDaN#W)YbTHA*%7yVYvHJf?Yq43yp4se=yPV7I@8bTaNwC2d? z`Ef&T;dj{R<6yn{T+quTQ*lxinVJSZqDfY;9gGk!69Y&Yhu5CRuaF6x z?Q?4mjDKwec3_MU`O)f7=?ljVRJ>aL?TcjoxJ8zYF3&+NX2&-4GB9@2xEu;Yn|yl5 zfJI0?g}=gZ*g>#_c_(32cmh4#i+iLgF5?rI(cZ&(Q6CnQTt;RLEGuXEyg0P6&M{ot zdx&Kw?Y&{L_I~!>BPv?40sfFV5;e;cE)|3RfoX@gB5esW#VvD2GSys09jdm>a|S1< z{bYz6nnW*n? z=~dLna;{VWS@7fPwZ@;>Nr-r40^(;zA3Lw2DVE~|2vI4iXwn+zD2Ns0B1$O4tMS-d z4OXm(~>R$DFA;c{1V@mQHTV7#x3;JRH3uu9GV| zfPRP6$j?xRmsE5pgeh0=Q#xP(($c|OzAOz7PCk@{ukA8tEWvk6FdIFA zym&y?Z>nvDj4>eUgNo%MZ1b+Gkw4EI2W0(*R zr$U}KE2wJ#1$nx&muRFsHT%~Ed3uL}g6~3s`bz~x2?Y$B0Y#bBCv*#eItNfn3PYe? zqktR=r~^P*c1Jl$SmxSC@4 z8Ps;DbtmV_vcflAM;CmTe^v&+mh*t$=y`{%iYia!lse) zB|2yH^}l|>>LS^u^p4~%+)eJe-RA%)CChjA{*a3gaRayNsG;X~{>-*WO$+HML(dcvSZxCL=n<4sCa2w(HM6`u=JXBFPZ1=CN{`QGPhrDyFOMj|af!^) zAF1GSkk7L)|1r$!*Iv0X-HL3M|0LIZ?J=XzD^7T3KnTm9iDb{QFnSKzQ$p8>s8=G6 z?oKSqO8D1>1NHArSxLYC-KzRG>shuMVI7oMIJf%;tV-(YPh`wF@u3(W;{7|@>R)%i zsRfLU>+?h`C;d+>HQB$jVe+#+_AmDJg0cGcuX!vXQu`N;M~duUt)WrAV69AinN0s4 z9fTg$o|-l`)SVV^Nxg?f>`0k((wZe&10wIXGg_J^odjJf%eo{VM?w|i5tYoyirw!V zg=IB=z^j4?WFGE7Zfujw=SHo^CgQ7NicFkXI51QE9TG3j6rFqqQ@nD_(U{@_*#mpb zJ_T|BwC50Qk;^%-=xU!lh|yY-H#t;VCM&>M znY{xpa=%g}`@lMsAKFw7%dmSA>yX^=c1r4Zk*ZkXUj$hg46Eb(28|$EZuR($^JJ*G@6tR(O7osiFNq^!g*u_ zD%Kj`M{cZ?)*8@*sVwMb5yHGShnamKW?aNHuy{+v;MNa_7%sus8Mptrcnq>80xaTu zsWosWwwJW=^BCAxj!~z1@tg!3JrDC@U^aRKk~PjoyZ8(?S|M}y-PNyT3)jd;vOi%D zVJXL5I69YF=Bfc95^S_iitm@rN@V98U7ZY=P_LjhaR#s^E|0=bUkqkTj>J!o(uK45 z>2qwdQ2g}sl+-tn`k(UCFJbTgXZ$o*LjIflbOwOFH9zG*qI_$90!L!AgXE{u#iW>8 zMR^o{8VSq85=%D@Iubu+L%_uODUHwIC%4S~ulQ*ddkFk=5srf-KMfxcBEe7lq`G+7 zbRhfD_~~EktKVg^A^r4Kg&eclv+1QJ_0_MRj&sySy+1VJ!?)L07c*)NAu*duOA1w( zaZarfG)NT`P6~?I=7bT%ftH{W6oE@{NHpw_-GN-_5||*p1akL@OKD5&z=pE)MTi=w zS_fJP-%vyLLD>1xYGqG+#OG^5r4LDnzI}8_iaqg_Y%^B(^{(*Fr?@z@VnFajhqa>? z>XEs~+@}C**e6WLM1Kt!zyH$ChV_fz->Uj+4c(>GmE_$Y|25uaclP_X8{dv93gTLY zwnkkn=RTNlNn0Zx#NRwX&*hu9Ys{OpG6FXVV?KKBBkoO~3+uvKR@Zmq{AN?kg`iOe zraV5*tWJzP9y>pydBa(v|7&u5IhHkrF`uclgwr^4N4&uo5Djl6pYgh+JcGAa1? znD-&O6yoeni1R1&>PdrstM@S&bE;Y943m`wywN#-EH$Q|e>62=J_oZI(nU|{UUIp=)%r8w}FnBQUB`hZK! z-za}%Qe^N8-Yi%xJ&n>=@ez+S0K<HiRF5jMLa$|aAua#%-P4Tp?2|AG>*1=U0WW{_ey;;%Nh z|B8wJLi4xB$drBN1L|R|`Exw1@EDt-#{-8m5kQX)#)BOX<~$U|qX;l))CN`f(Hp}Y zw6BgW?L6zIT|78^^`^q0$y4kxV(i z#sexomlc1sR_=AeDrYpX^kFN@iHq@od9c%vg~06YSk4IW6IpL13grgeDd`kn_^@s@ zv&qN5$;Z8jdSFG7N_v1v@Qcf=lJd=-EV&mN05C=|VNOW_&j#D>7D#X!-UT>iZ=3lh zP8zZLag+lD!v@^!2cQVe^MUsUDIHWL6#z8aqiP%fO6C}a9P3mLk|ge6*-Kj0}+aT*@*KYIlvT*Y!L zfEt;(JeKq9uh86*isixap&onfN#V@9zeWNvdiyQ}CvG6qhDztiT6x^+vrweweSbz> zgUz{MX)OOhW#*G|==u+se^3vvnCTC()iXEHR&gL?wKIO$_Wa*+6B!6Fh2f-Um#JaE z-oh}z5WbbC2qyTS_s=+SOr2QTvI1GmX|jA=)i+T+xXoWa$co@&qoR{=hHi1ZA{#1V z_fE>kUm^aA!6xy80WFIIr6qM7S*zhwfAq0_nkaHAVv(x-5Q?aHrjJG`^JS_|7`r=$vJb|r;ny1hONV^rO1Kpz$ zH?Y1H{#rvlNG;Z>D=&Y4OuP+0Ud1-N`T*My$+%cVCeZX5upyh6$xI)xAs@om{TB^+ zY~;UaNc_GYoIsA#k+z3n8ioqiKlGwNqOT#efQQhhA_{w8U`a+OFVFc(^!j$%)^ja|Nc4bogWP z-#Ff(qr>7t2AotPPhl>AoB{tsx!`UVUZW#$jShq8D2MmxV3jG(g>QjY@M)7ee6FoJ zuF^3VVf0~a;~)pz>;dk>3s$TQLy-L#5t@_*W=}MS2okd!`e0V!GlnKHBbq?0&mBKM zdx*%{{!Qe%sQGcW75NamY6i2j`yi$6&vNooVlV?F!-?_$YwOy9xe(d^lA9oZra_s; z2QPlV)%d0SS==?4|0U$lAHc$~oD=USpUpok18@V#pKN|Nr!7Up`mx#hO#K(~hyKpD zlRt8N;8rGdAMU4t@I0g&TX489y03N=4qA4|(UsD*xGH54uF`~Z=#(9Z8xUH2khl-E)%Hh_EvFVm+so8xU3mRh=a>I z%>zSJM;7h&8Fn9vwzr!XAp)BiMX7w2(iwQ{zH)H?{z;7gL;Y35jOHgG$vOmw%e>J*)aVIiJ!uK297QaY*c)3*3>)0<%HXI8!%#Z8Iy;Pv#xw-gr z;x8W?TG3#LH`I$K9MynSjI9~gxe`?pqixv7h;t2EVXv4{a)>MWqL)PITW228Jwnj#8^}9NFT5AVS zYn_V54dI^rQ+3Gj5+M(4wMGAqb*>J*WGK#UmFfp9jdo|X`aq%hGg~g@L*i&Ur{Cy6 z!_j_tPy@6cBhOdVjHY@gqPQ(G=Qyu1>+sG^9cw~Q4YDh--&-w3e$}>>+Nc=Z3-APVTNuQ(9-K;e`*a-H5R_{8{%<|pu!jaa}wWFTN?O2 z36o42(Jfu}K8Jj)l;4sFH_7)T!gcc9LijqBxx?JOa6p86i^G45-q=i>#6Q)Z0RCUx zrRXgUiQe>nRenmMzeT<$(O)m$E%g7WGIyAx21dURy#p1}-iMXnyUuI-)H$YhAKbD4>+#RE-n`9t_maq;ZWyfL!ghr>8m&J<_3k4V;R5Phh4X z$odLm3AyH7_i`Lg#xfL49WazxjzyZ7Czy=V;3rOY%9Vybyz`^eT!PRO5vutNo)h#$~B-y?TQ(Sb~YSKmjz(5nt{J!W*7ky;B1vn=IEts6rf`0%EM=#e zd$NNLYj0M5KvBqp^WNl5vv$Pd?`qBX1L@PsF92U2IkRl@In|1 z3)paI2zQF(XR#qXRR&H{@0Ut?aR^;ts2{vzfIdAF`w&49c_-UmnE`z|N+KcpT|i|X z3b65zxP5LzRrZQQ+AF*WfU^X+B@P~%yy^#o&eRJ`?&RmKZMgBXaf{ZF4X#yJx?V0V zf`V;yC2=pR7V5%)gx#M!U{7!P3UDoYK`TzrlODovf5u^xLzIhl<67||e?bRtbXE7c z;znQG=c>E5AYRs~yY|2r{fukkUPchn2QpS8^x`W3EA90e4SDsbjy2%!lf9k<*e~&l&~64=4VSZnH=$JOSUB(qqh&DF8RD;$ z9x~VRvwEB)hz6bqYg(kum7xP}=#lufV%7xikREErRV*e*+~L!vY?MbFH_2*Q7Cd?Q zy%9Z2$DIS8Mr*iktX?8` z4@<8_kAllF;d-NQsvvhvod-5j+VZL| zwdA;$z?Jr~CCV_=rB8ke!`C=5*0WAp53o@t(!*KWDqlmNqy^t(`Ke)(M48f=;%LsM zBcBZRKpczE2Wi1~tvG%h;9u7-wJGY~$Z2`&Sl^@6PyHbW4V@gy<^jz7UQSLY?@fSd zKCqL6V>YdC{2|Ba<9E>xpwogQP&1nS#;{I*WNat8qztU)+QLIecF}b~n8p~19tj-h zF-De@Tb|1sM!CaNGE!*XmMF&}`c!K~&==wgb3{}vLKx?8T!B~d4#h;tY5q_~S31S| z$CT-Wg9|Vi!1-8jrSA2Tk+W$iYP&ow4H^y^!t@U5S~xHt;RF7DV~9WEahN}Zy-p#B z+Au!pGfH!rCO$4*oRo!+iNttsDf#r@q;Hb$-^z)EQ1~zO2TsczDH0t5SF#+gXFL-phu*=^k3xn|1bB)T7*Lc+`9Fg;le*i z!QTDxw2b+kj_~le>W@xo4*q*1gz_&(20(ROGQ$99OO_OT(WL#T?&Dpbi!qO#9SFCW zA6yU28(+tH9yIrQxMLBHb}oWnGMI|zk|Rjmr2Jf8oGkBw|3-PvWO-BnFO-++?s+>Sq|r##!OXGE9^{o`A7Tq3L+SLEVWG`sGa zm=(kcC|6`fv!)tzaEuwxi`^R#9%HEGpIn*cn&t=$Mr!c0XrT(<$64}^bwIT*UrF5!c8^G(q*Aj z!@hKswAFkUme2|j{=QtFZ{VzD>rwzqswV_^@A%LEz5T&H%Sh>`%#`OrDbIud{r=Mt z3{3Q&NOa(*N9n-l=l{R916#jc2f8n#Lm+UmC+6c3!iFF7Db9+|84J-Sj~s>USeWs- zHhZJBoMbB`l1$i&!$@1Sv(pwVV8k7wtrn|5M!yUm=I!5cCr88FW-V;+-Db@`%Xu!b z1tw9%YDB9u)J3L{Lk>a<~Up8+}$qz=rLKnvD3804^I^-Sx z_?wGAX9_T+l7wHL0*@{I{o#`^N4m=Lkce<=hViQT!A}Ofzw(4d=rUTkiL0cgOjPl- zXD}RW0LFg1Tlv6ftXdc$VyC{1$Y)Z1rml~$=G;fea=t;H6`Z4AQ9?_)!&BIC?euc3w|l%TahYWA|4nEDFO3AcHt1~ynnbG#>?`M1c)0{ z&HG7#Ew{+f8|}aXYxBffev#plo1yl4$4W%_@_uOv;^0BjGi7cI9;NYvYP7M6D8U8y z$TmWG;nsl_Q8+PMaZauzSY(>h{M%d^#J>b$Z5j?|Jt9P@MdHHvz2I>i684ar{wLN? zU5;xku>_G#Iw)_OhwMS=+vE?OD{@pm-K;`$Mg^8G;s6Dt&A}36oT5;OQxtIP54hHt z+RoID82WaL;XcC8?M6Y1cSo~m*DNuS%+K7xEFaWj*$y>zBkUJOEIMh`drQ1P7y}NITZWlUJS0VcNIb!=fH4*kBSrJ zBHqWfe56soU|pmeZ9*p*-kb1Khy`LlmgJU)d~yleuJE!yt`%dW`dxzvdWB# z8LVBzFnZf??EjnE?KSWMBxTEt8>WD$S_94_=7JRjtLY)=5lp6HOz|0-te84AjDrDr zrl3094h!0XK&;|XTnmKL?HXU#8vcn^hi=WcamD#1&iY%%Vw>+eq}AV2jKfdmkdE+R zyx`+RyOIr#u)K<=(a6U=IBgWkv)T5D_W;|ofe8prvtphi7V&NX-mwlnfUr~{eL@nrT6tU2tWL|D{=NI;g^eL1B7FK(*nV)ZCxAM%>8aHKTI_ja+@x+oUR zGqdErroc6*%quGi%1T%zD?qtKQVuHI%EbB*AM|#iGPC~Qsi+mstuZ|dqdMY=A$EjN zHcz&+#}HIMeK8sz-$%sGs%NJ_{`9|w#InDzi#92g5NN*CGt&w-Bkp={-fF-XA+iPG z{ZQT-ym+~j?YX?Qcr3#lxzVkM(-|oW4=!$y_dI9~$6!qQ5O{4u9!^Gw>hf(tSOIR( z!5+mmoaNS7c@2!K#tI#5xI7+vvj+>qLoImKW9f1xn>P=HHT8(-3`ofEunR$2?RK@t z+=qqsk@KI!f!kD8Rb{}m9##7ySI2zzcX&O~CUM|Xfw1&SCKtTp3&ZyKCQVzm9q;I_ zA=-#xd6E-&7fOfUlN8DILxe?eWG>BkKS8Gd7mTSv4=#J1}8 z@?0_gWyHjT)gHs0?=fEVxLOxy8l4`aHPn*{!Lri0b6N9W&=d9@v9}`=zj=%+o_S3s z(8ZFph(D&=F;~?h4tyC$ljjD6v!dgX8_Qf5yc9hqbSP6D=&ulLmMQLZm@mTTkX@aMjLbh zZ96zqF@BD~(}oZH;}B7V4= zk0XKmv7ij;?n9V`?f#OQ+#0MpQrvrZ2B%RTksOL8=80*G`QVJ7pf5L*g9|>;!)NB{ zp|8NYejUo9Bh#?Q;RVhm9rzqvtA~f>MNhHV_VpCD-RXC2 z)*5dHc_gH649g=>0XodyhR8>*wtNfXb*g!csIYlzR8ls>7KWT_)*7&)1$Sj#67xr{ z0#oev*%Ls4s( zfp5{V3Tthdhi}8<15Rr^2k(AWfAIdb2N&{y$jz1_-(-7-S}e~<#jI3h@T~- zf<(v+Q|-8`eT#NF<}c+=UrYz$GDGY&4hz_^!Wn}xN~xPR$pKp87#B|(6{b`{)IZJC za#-qbGFCY>bPKkjJcamLS%>Dh0G2Ft=xva)rw2P>QSKEbld}Tf;hmuvsux93vKQ%f z?$DBa8)jJc<65}AUaf4OmLKeF*b-O(6Gtjd=sl?<%>oQZ%`rxEYy|?ib_xjWN_MYA zg(D-)lN)1v3=RP%D`24FXk#pU%78*I7jh9;I#Ur6OOuJS1>J?=mw-aCX_dI8k(%1t z{J1Spg}Q@X=HnQMoMl9rsA0-~Qo|oFRyDAA)h3jCp=8q$7e+^EjzQ8GITa~*_Q14a z#G@M1Zt;0PEniJEmwU)BGCtr`F}=@3s4mx`aRUal^};&Y*@Yy(J9w@z7DB&WUWH1f zpAnkQwpe}!;>28FK`#Tfx#_JUv;v5t*J;aK_~jrW<91x1|Tnaa&#?k_`8O@1WT9-|MEEy8l)``ET`8(8{uMXtigIu;J^l4*{VlwfxLi~(q21`3%KD<15&qdllX>A zOkMu~uo{@nNz_`^V6I2(VJgo(_8y|xgXv9Qhua9`L{CBo|o*}jx| z3i3G?UVo=7Y5fZREN!9P6w5hYqT3pN)|4Ng-NzrY9~+&zbVT)Ee9?lNd$bB6L1CNo z0t~PuYLvwORNv;&K z+WdO5#j}&uJzAA)@e_DnVJv5hv1OnpA6uWo@(s7ywl8NR`A-=4tA*08ij7JlUj>q@ z#SCn~e;5Dnq*QY*fe_rIO+I-PD_k1{lDQ}< z?yDk*R*xWBJ(YaB*P%qs>|DR`sTUVXAXwni9Se?;dm_7s4IwcaP0+;1GFn>9^t?0% zT@XO&Y3a_ae>)B9@@M0%TNs15a3nIhxG2_uWl!E7c0wgBAc!SgE|U(l;ygZ_G{Yu> zl7~q{ZP}C3;E>(nGtS$~5G3OWh#eplfZ!5?NGK1g5T_6_w&O;)tJr2V?gKbp3xQ)7 z9{DLB&wH(Br_Y#(A zjUm9bhp@+RA9u6^6y$%9^UiA;FC^c7^e)9#+Kg0b;E_<9Ec{ zZ)`6!{zN)>;r77VBHBAXB4^1h$7SH4mTtQTT_ucBZ4+@sZ=6=?zVHvTJk9S9E^E34 z75xMUJv4U68YUtbR0Bei*b@uOnx+`@3Hn!~ydsvWCz2TQhz< z`@Obwt#K3r8h&H>N~HK*Q=D2T9SOQI*HN_JQ+)Gz!3FT)5#6k&0rGv_Sk3`9Ju<9W zk9GJXdJos?bK2IwMnt@8d{}~t+dX$-f6Yc=fS2X#i?x}64!!^t6(7$q_jYv)*VYBc zPK#t*0FSl?Yb0NdwRvLAa7^trZ&I&Tp(?n?c)+McB+>k57SOI=a-J=C8s`5_4IReT zRk16IAZX2)SX*k&XOXk9x$X_}!a1nm- zcjlcCkoxuiJm2>`9eA_$W$oMAx3%|Pdu_IzH#|tZWg^4;5A>6J$&_$}jEtW#l~GIk zk(F4by{lACD`Lmo`Q_eCUOG&jr~Ar8hZ!TM_jH)&OCsOVJ%uM}R(PbRe47`6<2}XU zMLN;jIdE)G@pT6?y@!82r8lp0(?a^n$`Ses6q3HOQ1z8E5i%YVy+GawJ$z^@J;2L` z>dr%HddOVv-0(%*3VXXZ_Nt0|4tDFX?ig!XV|MawGO52A^Omy6-#Lqz{}pLc&O1s7 zOW%|JZ?=E4ANj1&a~q@OcD`OSl0M7_s7BlwJvWuJJ3ivzDTfdC>XQec13Zk%X8A>o z2M5f}@m^s8(}iQtD~}vz3wN75qPvzY-(F{S#U>p3**<^3l*yUMcw4sM8M|CNd2cGA|pz?`-qCQPv}0E*dm@U5li>^szaB?tJJgHw#8U;6AVJ>18QfN1qnM5 z-9np6H|uNOvJVyYjLr2f*w;}oRO0->=W@e?&~G#AW7Gqd7SK{z7C#?7_m5D@Wk6QW zNL#l!LT3#AS46cQxOxS6NEFtqlbC=Vr4z{%4siwJr_u{^}W|6vhhpIw54!<=WWwEgYcG&GS5eU z{sDtn54SNQz9Q-01v1_h$&gv-o8RmXxj09r&1tvryMw1Y^Fd;-hrcq%(Z)n%mY9A4 z56c6DwQNm)gKP?iWjz5zU0&&+1{4c`)O8;l?Mu04f0Wu_&7a9Tbn28C z4cSUDarC?JCk&#S~fI$1(EFFllFX-3*WB=FVyEnkjM>rsqmx z){$L9WM+@pL0zJ~LXEC#E+T_6wUXN&3nDQ$kLmWMsLjCUxKasAoGP>|Rg~+T>8_hgV%H-Viobp2O zPVOvY{Aj3+~h7&caM+`(TnblLb;Oejk47u-5ua1~Ky zu|z&+Q%~eo>R}u{DnP8StUCjW$Yumjfof4x)qkaMdOxXT6YRiiRB_}l-Cjg^cGkvn z?zmb~ThbKE*&rsiO3QoFzaLPJv|6XGe3z-JC%)$+l8qp`+66J^Ht?ejZ%Fo)^NlKe zPNO(iC}#qy%&}dh_D9rU65ktYlJzbH;{a1_YAe|o-a#fuXoTaOhm8L2Dj97{@@mUx zEpBpNefWbi=ee?=`T*@oi)%k{es+jWQWrd`%(1Ua^UYEF2`h+%8p;&0oI1?H)H(Bi zBL{x3Db=$%&q^T>%Zz^WSFzaP#c3#*5o;{xnQ9rL;vL~M9h2IPIu$dcmt3)cTMnqR z*Z4NCwG*Vr(UUK46-A7f_s=UnU3vXPVwLws%*U4ZuWe>|fB1TGd0$tBikEkmj!BjG zY|JNFUe@b+M~Q{GH03=W(Q(EeATPytH|3((%LEoJ5V+6PHDScFfYasw0csbpJ}Z`6t7KPu(=b;86l}GBQLK zr2z^e8T`#Jv6HQot^A&xgRLgnZh0-4Y~NC}iIeR^a!&G!Y(tngi%P3b3WYA8{IbaE zg0e_Qk+ck5ln z@1F#C*TXJq!h=QsOO8!6zjVmf34)PPFiktgu zB!;FqX+c74?h|uvQ_fkj*ve-uvS?zB8H>y@W2uZ8vaaRglQm|{(1lPmIS(^Z;Bf4{ zp|cK;a)V&~Kbm1;v1)xy+)BW) zHnMmupc)x-KxJgHG(aTD`Rnm=6B1ZdrX~#9Wm$ytFj{7E)v)@@c}N|uEX|C*{R&mH zNRMAEm~*nE^(Dry>mZ0#HCJP%YVKC_6s1#(E`}w_*NGs)m zrM-R5#P(_0=a&gbu2o(9C+6d{&p)U$3I_3C^XQ{}5KmW)Fy1?_($T5jS%vwN+owmh zPTe-uR0~hx(~a}7yiTd#NV;bmKpH60DYVYA2rK}b{qOS*Mx{BhA!qZrJJG$Tcq z^-l}Eq#4;8(Mh+T;w$X!=Xacbewy~Pf>nUNPyy>5^ykWvw-N8{@Ow`G87dFEsopSLma^< ziagYxOp%vVvEwZuPp6HX=!};fMW~MUE%+TKdL3&)fWEed@y8Khqqmj5QdIT|_ zJU#C7oAmhi7n13*OA4KsMkneN6g^y+Pl6sv{LO&WPBMSv_%EC^f76SveFhN*$HSUF zmNO5cm<-NlkYYJ6l?jzJ)A0{7I3D7%s8o)5BnD>-oB0+$9zFFuvFN6y7@W_GFAUCh zDGjg_C_E0Vp5}1^n#Y0F5grFtM|d1q9pP~*6_3M4VjdPxq*Pe*BCLuzDu|Q{=QErI z7f$DVIcbvl=_rr6*f|}y)I*|*ZiF%vPgPMXW;*fRdA;ONrl{+g7-fp;7yJ6B2LMT& z4(&IozT8lMSbbf&RiaY-sDUq<8RETL>F`(ewZRv!FG&i19S5l|v7KNMdxf=#wK1fd z32N2XgP2*gm<(w~oFN^m#T|xpfas&EW{G8kvZ(rcL42vFqxcH!1Ioh?y<%63B3+ll zc4{s=vS=)(t@y)&V%nliX_#bIXPKm19wOJ6Hi=W|#h5yksfZd}8!4R!2t-N?08Nq7 z*?`(eX+EG@*L|g~d#Zjaj!k6?WIe=-k9KA+S5g_W33YCb!;@_wt-Ufxn{pQgM&BAiv;otTeP-VCd} zf8U#2-kGXi<5j*!N2kgg#QaYz@1rI5@~*)4Y0CQx!dd0rkNG&|y|C7--*LmrEA7fw zzjx^9RC#a5{7)?Ju3~$6{n$QDd4EYbtGsVvK2CYtxb~&m^XWW!X>ZTcu59JKUq`3P zdk^N5C@-f4Nm3(n$%HkWx%>E_TEUk`-ezl*XZjBJ=Q%bxA}RZ$Krj2H3)!{e5?$BL zZbKc(x^Qx%SM&7mli`hW%F*ZR>BvnsBNl}NZ%vrJ<)P6jX_&K1#fCfR)TE#+6-1NIdk=iueE|r~VvY(;7|Jc1RH*0Vjn8(W5|_tFMqaL6ihC-?^$^tvjbvkQ*6(I zCnZqu*{&xgO%9}c$WGFIgVM;y3nLr;+9X*@?d3#bmwID!y~yl(Zp!fOd@q$5Hu8{U z$H$iZbhV>M=_l>z1;xB}+2aLgeB%5-yka?>UMc5M%H-VeRa|^~^i)b7AVXIt6qYGn zW>;SqPf$A-_i@bTWwi}kk8r?El%(BN_#S`d7Mq(bGn&-m{O&6XJe%6*p+>u?X9X>! z+Cu~pUa?*+qNBg?b>FF??0#gk#N7Re%;;&yE0o6y-}4wvJBh3MpP?KT{$Q}6Les}cTCP^r;BT#C)QO8V~wQ8G)=Ysh?Z4+TvY%{dY zr)~FYTcx($pl#LK_HWu&t8E`aH}ct}ZGY0X05+*O&h>k8zQ_?2`ymoa8fQ<=cX41o zTMJSw=Y18Tt5d>@AnZ&vb&QhHWfVx7HjCA`Bz8lkvsk7K<*(_(^3=3z3hAwV!Phie zoNxKNb@&3yuIaS6$g*onE%sP;O{>K|%kI(PD=oXG*Wzl+t|_*-*0O7wEpD>xwK{yj zvTM36UTfKfa z3G=?}TYpu)EWAIGA)$1`i@fxM&Js~OzF=t0sf0)gXUj`AP23UncJ1XoT4KZ55$X`n zkSVu2L)GyHb-~>m*;`^RpoMW_u(viKSUieFrY!Mc@+W_Y^&&8uBJ!fLo)p2+P30n0 z`>b59cJoeb^i2XB`Wb&pB3){%?0#7DgIuFmlOh5Ls_XW!Yf)lec9wSt*+DPjzMkG- zgWV~Is_tkdo5$2dEi6vu;ipnUqm=CNV5ODO2zNUfsm<5NJ#w@*Qm%Kt6VG(#+g6@8 ziG)MsmFR3KBiWJ(QLbN0Hf=|YZjKi@X8dh@*6`Who8ktUbt+rB%{-N*Ed<=9N@QBp zN=81~Q`!qlq7=FSLx` z+gQQaV3%K8H@Q4gQNU|`Uf%qm3a=l_*2C^YXGl!uWC3@g^!dM4poYmtaDFzDZ~J+ZZQ$AxM+rPn{0H`638kKBDkIH&0T$>P8~h%gHB z`Jc1lHVQ7+rxIt;9vTXq`S9nJcy$y zQ}mNVT?puK&?5>eYIcd=*o()84wh~>7#b@*{dMH`a*Z&O%ACJL?sWQj7(VpDyiOrE zF0BX?J}I=1+`UIek&>h+^MQ+!Am~pLT)7J!K*B? z6v2O|>isxvCno9iaz8R(!u`=i^~+CB7h)#e^nTy;6zwG=P%ITsIkMt%wZEmp-x z)3&xaRWseVK?QkC^*NnMO%({A$ZO=;VY%(&vUn!9s>_`VcS4D97DClLz+Jd^FxC45 zmXv|$mkC9~Ejfzc!8tx^V_u8}gp@5`bJ*Dbh;#T}iCNWbv5qLkD5=$};qtO7~T zcze9gy)eQFe$1@Uq^#3i!st5Fl;*OLEHwTvXFiO^AF6%HNEtszXMDqz)Pwn=m8QLn zduncDLC`65NNRS4y@RhL@lPn`K(`)4T6nyu8{iPxlu~s*ez|(7*oo%M@VP>(!wg7+ zw2{l*@P6S5Eb{Xsn{-llzaTSM^g}saFP`oQ&4x8`35STj9?eV~M_NOZNlpkWB1g&$XBH&l<=Ye znwh$uiIpyRCN!sH!Ly+&xdlDs{K2)k;Y_b;u!9*Tk-n11Yo!~171_mAU!?r|yr_zO zu{5Edi9%=kB2)9qx;?wCH7*KQkq#(8U%_}ag>Ck zS)Bc)8(-v>f4n@q!oQ418k(T8kRfy#{J)c1A(#*=k>+$s)45T^Z)r;uN%RKUs91FQ zH0Opb%6B2XdDD-_w`1{UX*=2 zr#Cuc+r)Rd9zc(GenP3Cstq2Mr+4IS?3q%4UhAE-Lpz~1;y8Wi%S?g zn1tfo^6u4co<_;XjUU-k#W(1q&oQ_-Ce0gtbe%e!_>S?V~^~A$N9|eDh=Ka5Gbe_2Cxe5aJ$m%$H zi1FFSGn&(Vk&P<$o{2|&vyPwv3be~J%OksEIqBq!>nP%l)G>9GX1i6}&v7E{MR76v zFe}ak-3pv4Bk{C4i8WCiMWQ?G^O&a-$yX+;g|V(xYnDU4p7laPb**@^rl9C;i1S4Y zZK*Gn)J-ClbDfNE+~9hOcU43Yi>MTbOlQl6pmm?zq?bsAA4{*azm%b%dy!oGTJU`E zQs092gY!+T=4Cx=krLn2F_}tuip$|klf=Vcfvs0p!Zn5O94Fa|=XeXg7QG(CjMrgN zHu`+@XWTnMj|@%AE>8_p$n>Jy81mHF6jCOZYJG*4eFvNT-O1daJ*uVrnLg1_vQtKg zr9JcZ79NJHt7aX3r!=Xa^#=fv-xQ5R@)tOdvv8)x3t_5)z%cbXci z`;=nK-7HdK{Zd4v3~MA;Y-Un7vY?Ue2L{|DYQXK8_X4*Ss$;aBo;!t*iuq@#v}6){ z*hnH;B?tFa%sxk3b)tcMUi+#Jb&J|^qBtI@ZlCNUR*Jy5d9POQnkG7w+7B>wioT9W zuJiFK-oo0?i@o8AowvX>jqh2=vQTS2QTjV?&GVqh-MQgAWD~Sd&USQ3GT-5D!*f1f zJdL|nsDSNt?id=?b@cOGNQ@LSCmJ6YPor~kkv=)Z&n22;ozbx-`n}j7k~N_^KVl4q z?rv}Fo?M8i#$PV2b}wNXnC4(VeOSaWZ29F##rX<(-;vN+(6S|+GPv_}#fqc}B5Ac_A{m|lt;TIN+RGk=#jxUvWB zj~u2yF!Z4ZN?o%l>-vZezTWiT&Ep(pJ!Rw3Om*seDNiCBPmqm*6xi}+Q0%=K^WyGvP z%PR{DcPeKDIXcOVH0oN0yctr!0&e2;KndA0jNPkuxEu54ipaxFO+$$4Mr8Qyi5B`pq0(qn3DcRZ=RnftVWE#Jq3{51wYf;v7BNNc> zj>)_u(OW*Nj|u3xqtbEJ{iGFSfT5`@@?12FEZ8QWGi}+NF4=TGe#KM<2&geeMhW=) zcGbQX!_jUP_A1`=<}ltC4CZuxewqBhSer zH{QsrTm#aT#0VxDHk7&t?+rDdJ=DsgH>t--e}gq_$%7l_BuIL<8%=hi6)N&kH%Vw)Vmjnv=Ni*U}9+s_l9{wr3Q?j4|Ms_AGMnsC`G-`km5?OE&HbUF(fF zj`Vvs?h03W&wMJkGf3d~8vet-`Zn89iFIu;N-) zrCNQF^sn6x5V5V&BkbDUR9g66MfW)@orGs#n5Ma~wN>6dr=*BuNv4|$j!I+q%C5X@ zRo+HMxTPq$W<+7hBC|Acy~+2#e19w|c7#@A_#V5QKL&o0?<;(J_+H>c-;X?Wz2Xse zK)-?g=X*-OQJMDj1$#>G%w=1Zhqjkti`^@G_$BjO|DCI|d%SEpV*Uzixr$e@>|{Mx zf!@y2HwRrL^JeT`AK0aHdrRlCVnfn<1=*2v!^pGD4Mm(EBL>hQ+h8qvf^bKtaxs!c zOZohVHW4)Bj@&DI2<$?le?dlYH(RTmy^Ckm^Qz42r z|4n2>>itw?_`x+d%1QhnbkWcBfOP|F0Uul_N9;IlVdg#vmt3BQ^#(s9+zF zmY`S#bvHgb0gVAC+Y)}y?$=4S1P_@Q-V&~c;ZZAP)e?%1X$gyIl*elca?CYyOZd|0 z6Sf3BAi#+w-ZNL)CE0jm=w#Z`J8?|4N~sN#(KAdnMI6ZvOebQD!Yg|iPKblni!M|c z;W8R2FVO8@bG}FssdIm38iu6dVpQpbDbJXoq7=l<%iR2Ndl*0Gq z#mce;M`4ooh$<2e?U!XTyZ?feKzS}&ajHP}>QEj>B5YS`y*#+E3DGa=#erOqj82oh zt%f3!zwQoZQ?^{XI5$k_O#f%oV zSaiW!(4A!=*FBc4c5_09&oIdoe_zv^FFgoWMHoL$=uosLFW_E@3L_-ws)o4V9t$tP zGyI3BOkCv=dVMCEgwSIl`^EE%ec`z*9$6}scNf#8i+$20Fe@pU>vpj~f(jm~=<-w; zJEMDH=HOE_dJmxlloO3CG-AcRL@34osP!5|rF$cFjq=&=s=6V|l3%Kh`kX8Ip2_7u zoq!hwgPn?aqU^^rCoMCecd+$PL)27S*Z7vJZ+9ak1) zz9Wa!Q}JY9zdI{A_I9UwMURaNq1L~$pFq;r@bPx!uR=@oy|aikCzjb90``1`W^J3jJTA!f5lLDL!V>cz-nz~ z`Hxu-kRJ|Q8S~#)Bdl3&8Cp1^<=BI|ofscu$|6~M$Q>MGjxVorvWj|mw@RDNO!sBD zp+DyPdX_n&Q5GetRHEM!Kc&}FR8GKNz1njH8>eYi2xg`BM)r4?|ALqnWlF1Ga5VU~ zGE)|hl&xA-Nb24_CC_mwN8(&cI#7%8*vBh>kW4mnlr~B=PSDS^zDzYdMt5*t8GV%H z8@dP%iWxO4q;?>B88f9q=}wI*?=biKrT_NlR0^bGkUdZ$S3YvTGEe~39fS|YY*@qt^WU0=8B2_n()@6*Oc z9H=XKJLF7^3NIhE{n!U3l`p7h37pTFvyzWWjDs>A7w7^nV4y$^+87;awnXZgd)Tvy z^-lD+bi%t1M8@566JsmRn2mFaE1`1os?;C6MS8NjfF==e@25GOpksAAih`QEB%wh zD``rHDkRhU_8s~Y9R#S_%$;LZoz3w3B55-Vq{@&sMz=U?(TRn08|M$k-Abmyf8Z$t z+SD#yBsdb!hx76Dui&|HuR1AZi*{4`o!FzFla7&2YLGQ;Ztvj4=uwivGP-bY^gWD& z@7!g3Cr0)mR%j*7p2%yoIOm-*jYJ1ouaNy&3}9!|#{L$5a zgR6XzocB67IN`J|^3)R^V(A*VY*ybkN7+k#<%MjJl@9RHyA>Rg)!~pcJN-Y2`dGj0 zabLyV!uhf?XOH_jQ0lr8dU$Tw^#uLji1yQbA&by7w>+|Qd&RX#4$M!#=1(8T!s~qX z>33tGGn6SX*0p|qWhq3h**=EW)i~dq8(z-#o%6@uiR>xm@+dh*cBsa7miLTh-*RGJ z*@FF{+{oo)*)R-dG3=)9L_K{l6IHtTgKsjX%PYe|=&qadjN8PNMMY%*Q&hd>YJQyD zx$k-{dN0+(rUF~-8-5iY<-0TIYmiL05#7EzFZnqx#$B(4(hxrzPxmaMDgfsL(B`e0 zQ@kza+!7Ruyv8D+Fz0FlcD;3(?>EN~b5iSTUi<^*9bc+>GBe zZ7gWC-S8NO(5h$L)H1$198 zPtByiwMOuJ()TL}Ef|-29?tW?Cn5hQ(tVNFo?uOAM5i+z(^}FZ&pvlf*Uar?ipj2G zNN#@gQ@7D2%oXAg%UQuJjNZ%@8KOm`g^}<1K+$7n4 zOwE`&1iV-+%c;s7<~-nXnbDEMXQmWECKlnuC+7)0=wH2$w!^Ai68))6rKILTk1a-~ zpRB{bI<0p``eV>hDMy1$Y%>Y9)b3JYukPCtIH2a?K1!{|tA8^GN=NtuU%mHYU+T{F zMQUXH?Gt)r^yP*RbZy^2&c?5Kp=x=3;S_i^Y zQ4pUNo)q!9d)5#9rRDeXCWv$WM~3ICoFbWAQ*;^R(EGU3kIltQ&*|6Bmm*%U3@`}q z!P+VoVkCPZ+Dv+}SAAW(I3)ND$NV%)79p~bqeZIzDs3hyu4y{1Q;&&@h4LrF6`Q$V z8ghKj7Xn@;bGnE>T#6x7N(Soj%m6F+G?+BXK;~J_oP3K zN!rey^yf{}b~=aQTir{9<96fb+mq&yloRfeaf@1T?=DYsTaHMajhk;d67UAf9u&AF~|5g?Pb~7Gwv;?WO@usmrMzmrCt{0rP=688Et#g z-&g)@#8~qY8S6%NyC?mBl@BZ0p7ednNO5<@=Qi0Plla#kFMiRVOVB5d8~-DT`0qP@ z{4-As8qGY@s+Z9nhmNq<#*FB;{Iny&G+PdCGIZ+0zASx9+2%x zZ<6GlrQS>#_o+aa%&fWA#MR}G!e129Hizi3gfpDqEQ##glm2V0(p1;KU+=9bDTzE& z5_!?P{(XO2sM)*zgXT7WnRoqATgcy1I;OuQ@~e_a`qhl=-t~X-hC@xIV_u8AR5IqZ z^&c$uw}lJ$Mvj(@c~MP5&Tk^HU4o2#cg4|2U-u0kcCYvw0(RjlU)Q)WNyfZgeQ7g? zE{i!!y3^B%@seI%{R!n?Qk#*&!#I}&1WZZa_R-&BsH42;G#~u5{F5?%YgY%)>hvMy zpDy~=5JL=4UjLTi9FoDzS!)EvHx1wa#&4eS6;@&8R5B+^t%bs)t9*^{T2+gRpW4J)ZLvP#uhrItIxeBeI-UlA)`CJ7LzoJ zB07iK|5CI0zI`$mSL4U{POa|S7n_+i8rsdAsQyk;e?98&1e^QE%51<<#%6v^nbVXx zL;cNEf3wuzvFh)5_1C5TPEmg!Q=z`A{%%%(Z&!aq>hFc>ue?_to2l_dA$n})?=$6Z zrZO*7f9I*c|403;RDbWHI`-|`=FszBMa0Yg6A}&kqvIfiYB>xwaJaIrD~+&O@w&I` zw{UcOy@w6n0NEcpMK*~l>MPTZ94JrS@2l@CKXZRs{jSR+6QO>wOD=|{F3CLXO?8H{ zoR51`y9S2JQ(5Sc$lmJ?J@md_=P4eNF};ai5`bA-%Q3Gk1i+ z9eSi*Wf|<2?abKBAasb$45`0s)L)^e)bYN3iS=)6K)YCf)&B4b$*2;dTG2$KYeM~I zv@n~Yp}TY~;{f;WWPX4fWEfyOd70qD1-#>n+)xXNa(!J#UBT<+)e*MwIXzv&oxLup z7T+ZX6HM^p!#k(TJ-&A37V#eCxxXOx8fBlw66B-I4(wMbd)G&2O9Trwa$ofOLnOba zIG15HA+U_JqbtZ9aN6!vmRYQLuVjg`nQbCKyM8L-?M+T-{t{9{d~T?n42p9*2VVzQap&rG%o_9nVxlDbfsewYyU zT8ani8Fz)?-DBcCpTLv!^RnbO@bzwYgQ%`dMpNnQn#74{M3(c= z8W0msB87VB%!4W;5g`ix|^; zo%f;Rmo6C3Zs%=mj7>sRa4q!`;Cc@nSVe908M=!t4zp)>S9nnM&*)Lg$1qjF7*blK z+4#`dJ*93kC5y+wjOe$LGod4hCl(+v8sUm+603~fU`tGqT#XD}mmF;3GJLQGZgUr= z==~0%Px=d(Wt2!B6|P|_qaDUag)(JNj0*FW-5eEWYvjJ@y`0JV_N1>RN7`!ULI zcluH>LB{+;r<1U9{MVmPM}c|kghMin$_@=%OqhAIwtr~)518<26Mol(cbRaF30IoX zXTpoj_@NCfs1cfC)cqLZ1m|n{bK=vrKsKMjijlChRrg789;D zVS@=PO*q$tV@>#BK*!T(!pBYcZ4+)bp;c~VL_89rf3ibC%YE!<)9%v#mb(Q#ncDwv zCbZm77FI9P;iKa;tX!|*kO})u|3Hp*|Imah+E%x<-_+*zudVk7Le1@MW`e%O8gDrM z%DM)3T}w-QeVs<`I}u(mShvpI=D*3^;%{3OQqzPmS?BT1oKx87ZtKu6Y{FI(vW-5o zr6qjP%!0W9cWZN--EKLn_}2LAL+!zfmtX1*wYR%l+S^v?0(69e&26g^j?Har>ROr` z+@^b=y`xzbVC_s@PD59-an+KVa_=>zRW;=+mU^oe*K~yH+8XMD4K*sbS!j}284kJE ztX}L7Eetm{`hykz*7o4KD*uh)=AgfoC3i>uIqs&q4tHC7$i334kM(|MJ@@%YMG>sI;y;pmzrl*Cg}7YInf>)PDP&3#jIsL9<@x2`?R(N5x54?@u2 z5pD^&o7>#swg!JAI|L2nwW)4Rb9>O;+(<^7JKU|RWNu914)I@yv)|p|U(;MK3C`?r z?{LrTV6CZtS9`0LmMvX;k$dLNR9t(|@4l%n*hY>tDLR^HY;(XaCpE#j$X(V}-yRH7 zjYJ&u)4a+`Kp4rtF<95?C%X;)wGxH;t!b?*CRE7p)s3>9;~M$lUgD>++t(#~#PKWJ z+u4t^;b{l_m-$0x7`nrX4u7zuE>x!?)%ZF+UGKK=rh|zXPe;|KJss_U|DvGZtY~|@ z5oO+~jeGrkpVtqB+W7qx^tni7;y!6e-> z-IWqFmr(=vjrAz9nP1OH#`gJ2?NPT-txV5dr ztoek$)SH{(qFVK}#&+6rQv0EWHr6#u-^-ubaL#P^O-)$b?ZM_%&24opE$iGJ;re>N zzrioTZ0#G}|7(PW_mC56_n^M|CY7lv!ok3}%$I zF<#ZRG~XJru>bkZ$bMTD!xAMAm{-FDenH^^+I&_2sf%afX7Yceau9a3VuW9hF z46mvQ`o&LG=vA$CRsI0ugR+6AqITOi`&C3 z4Q*$K@M~*uuL{-$nwsl7+&6~(VZl;#SmLXyS>+Ga=#dR#BnIudLMQR3HIQk@S3?tt zPj-YKMu!qL4jw~=1l}>j(=gn4xf9JzkCY91^zt{Tq10*yiTW-McZAwo-2qzvi2_&^ zRw5NoO;hj16lrB${c73+L!VjWr#Y|vj2iyWQACj# z6>=Q^e0M%n2~q|NBiVnhJ3rjk+}Iv$9ofIoo!`#%7PNeI{JLCfIvt+fVd<0K2H2>k z&vbjh&fLbCGfkYvAT6|&3C&%{(7wiR&hXNh1()!esZeEQS;WXzLu%>d=mlFQWTzX7X6Ujd?RggP-{u1-hNHfLp z()*+-<9Okc@_B-(BN5)5wr!+hC=R-liXR_llIU*IFHT}BoH;^B!Q%lF=_LlVV`pbO z-HxBH5g9Qvr9?W%53j3Lb46ALPbQpMuB3n`6Hce6xGXEVlcJ*zm(2KC5u8lACZ$aV zQHM-~qk@wW;+!H(id6qGzN!Rt;WoW^VR#S*0S2nBzCIjeR%}@}$Gt*L!SU8(tLUnmS$%hS%%y>2XYyegPAr6z4_!Qj5`3bkR>afAEf zWx@JItg7bJcdWTI%e#h^OWjIYWN8E)@Vqtdi_86M{4JM~TXSo;)eXm|k>WI-s7&TW*0L&bR-xsYkiSmb{Zu@orWY}u0KZfmjZW~D`-YHa%H)YnegnHVZv83 zlEcKQwPT51I;n;t?y@M59kbOo4p5ax8bL>#)l>N8?RAP_RN0eNr;3p?=astMnfYAS z9uCSXQ3}CO=Wh*!)+ss-U$3hDqQhv4^A;Rq8b5?thc#{214#1o+1R>c!!7xgM@}GT5`yVU; zVm(g0!ZBG0DtE*2l4q)65$gowe*?rcJ5HbEd?vP-tmKmYN93?h5Z_3gijFxs-uMh< zB|D6Mw#muFNDe=2rcgPuD6GgjEWVLO^brD?v_7_8bh$Lkk*Ll%iS34Da>rv#&cBXN z@ljR-vBz;T=3(h1$DhQZDhiJ1Z)QMRVzU3Rna8YyW9f3t%n-*s;okRLwopvUC_I%9 zy`^N|BDbL#XQbad8A)HGzrvwp+J=Cc*gZ=fhTDuww&(@Vw8nwBXM*oGT|^S=%deaj z9Wy8U(w_HzwB^X|ZI}M-`|rP4v9j_q!|3D-EHsQhKBERZpZaz3H~v}qdEgo?t5$HW zLrHj51M?NS7PcFXw;g;*e*icfF9r7lS4(d7Ps$spPUPDv-$-~xCy5s&ezzIFJw7+? zVlMy+T>~TFiT}`5@>l;j@;8oRA6^c@cz77zcI<8)Or;}qk?@r^JdwI<8Uat@E59;1 z-D>P3rEA#mgwC1R-9Y(C43e&-<3II3lHa{pSU+&k%&*?m-wVU@^QC* zQg1~zyc5OO*EDi`X}DWIi7(fNXT_&5Y~>YtHmyp;XW>cy9=IVnzWkHKi#CoN-+j1S zKgr+LKT9&JdZYtdQKsl?s-Nj(Oze^Puxm)w)Y8>m0I z__m%TUf?A0iog@eC;1(ZE~>w+bQlk4v=icO2hVd7cxz7*uL!)8p|f$4cztyz7vE;^ zd?!(k%9D&Q7rad;5#NE@lgr;0@M=#YzS@(FFMk9)>Aw$(f-mzY{a_n*w`sT93*v1? z!)FtCF7XG?Wja}S%4+Lx(ysCe=N5c^R(~65+=w!cxD{5SosNXJ^*DIGVdGHs2zXMy zcdu8uR4Iww1>W%XC+U`4myB19-K|0;%3)gLKtdd4o_3oqQeOErzH-ZT0wvz-`N~lh zP=2wF-}2*rg)xzDD}Gyt`N=%;A%TB_-%p15A+n$JyD!Q1&$DE9Iy^eZ$#HH!Q@i(? zFxeNI8OR>i)9Ele7dZ@MM~*?a9K`1VoBvjuS#Tr{>xWo%Ou6w#;NIKT9YFWv$p+8>o(9?(i zTKrIU#g|sCS^AQsQ+(0&#qACL>-Cwh=Fj6x-WoY9ynbbK+x1sjJ3ctI`0G0SHETNN z7My>LF|Bkuxnq7O*2#+-tJdU@Hk z4|}HAi>sFBRTp~&r+=86I37rL<5CX%d$Joho)M=_aw}8u4GnV>$D_mCiY0KKJuhV{ zzRV=IN)?|g*-hc)Cb=zFcy8QOSykDVE4%{S^3R~WL7|DE`t^qn%buqsV`R*05V|Rx zyqj;nxnZS{OP2#R_EpD=N!hn zq@(S5YlS2SS38V1#7)L+Z;xb;IC?IZs^F7-Q6 zIC@}IAfxu-TwRtKJl~eN{1jvJDTXl$Vwa#vRMJVdrx?aCoXKbvoq*Q+aiLR)Gow=C zt0bI?Z{7rBUCIi#60H(S%6@U>?e#uxc?c*;A>%`q|4nApI#d=z=hFec7*1#~_#Bp(6C?U}~y z%QGoY7UdajIIG44CeKJWa#v0qm=MUWbrrxzP?jNh1#4`!pe|i}T9%@!A6?TBzg+e` z$dURix7oOPaI=0gPD*?ASPnqv6wOa>VL+C&#-FP_uLO!+!pQGEpqTy)lyn?~li%q; zNpEsiij=X^%Kzjssm7Rvqbf6OWpYePGbUB(k~-cPYrGK}V?1O;K$-K$=rC^s<#&%)(`^e-!mKXVS0Deogz(sZ4HUk*vsByPSfu@r z0tLTgu?`;r%I{l1$y0XzI3vGdijlW`vf-YWYnyT{4gaTw1{5nFscY9_yv?v{g!C^pUbuN^a^cvnd`BfcnLiF@D*JD7ORF;TfZ1+O3ZvR0+z{~@ z!{d|ujo|L!3jFAM(g&nJ%$S~LOblnKGtSH~&V1Qzoc8o|`r0)14GrUEz5$6#=BtCb%6LU{Oxvo z7)8gGb~=tB?^#BG72s=po@EvtGn74-1!1|kQJ3OxwBbfjN1-x|Z?Q*=DmUCo=0 zGmVVi#Mm-KIy0D`r+(-*TrZzNnP;I%5(9Fd7G}agX8Fn1N2FSs&(6Co_#xpFWHHea0AS zq~2Gt2WiG*x9fP$Ns1>w9?x0Cb2jnh6AvsDg4Y}PwwdwR?YazOjjSMLQ0-sW+W^W= zmIrD7W*EC&hsjd&f(|p9x@Ihe%bOQ6jM*!6x$SmGW}1cm#=w%E{AvqZXJ?0o#4ivi@UvJZX zr+%IW$?szz)n+W|6q{kRb?Wha04QOy6`wZ7$Ovja?E&H!_gXXVg7q5j5Kw-n+^+4D z&l|<~F~c}*p4)IbrWscL0)&4XJc%Pf9JMC??}74rsmlyMj=5^3o*#T>n&L0v1X@4I zdkS9(=NqGevQLl?>?H2tywZ2g>i;rv3Y-y$4*0`+Gq7J!RToFzt4J zN%EI{dWLa2^Ls91gvGD8&3q4-VTOQ15uNV&cdKZPOMsHX6+kJQL&o}fr0qVfzU?Wcr`~MHS4r1gP=>RQVA8eQt{8Q{aWS-D4Q9 z@NKTveamiljFWZ8SYuq3UUxV)W*Qrp3*Qf~3=gmP{#BXAs^MU75gC0+5C-z6iYEAgA_x7rzMovO#Q0>XZeZ!>Al zAZ(J|hS&cs_JqEjz_Z&Y+xm5WDeLj?==|t)Vxqlh{y4)Jz<*DcKMVE~R@~o~IQUKcp01~hf#M!-Z_GcA85zb5$LU7S&?)`dz0S%Rd6`Du!6`;= z=rqICFqOIN3}b+BbGI^|srkcV_K#$jkjQ2^IV4Op3y_2YBMTlT$^TG8yXJ}=rr6v zG>l96GOrhUq>%RbBsX(?ZPpun8hv-@QQFsI+|-sZvL0uBYlO)Aa6jmige(0flwr7j zL_6V&s&&4kIoyr@X~W5uvNeo1PKU?LWn5rxA4(h0^T&Y_bN-;+3H_x`1WJ8bc5&-} zT)WTyvDggbJ|HO@j{pTPn^WJ(j1BfZgl%Tr0W)riv){y@wp~SITnUsG(gIZRsJxCN zuVcw8>(r`5UN@QXb=uV!jT)70p;g4-ERYq#r}Zle(lrjFJKP?B|q8dz?;4t zo{;sVS_82n6S_rBDiy7{3K>6LjTepKB03|-PmI|6{DK$1~UbA2ix`p`q zYxH<#w`WV;)Ay$_UQAv-p1L1rWZa|sO)vS7-%y{9{~LRi)z|@4`9DF}2M9Z0;(hTM z9qzw?D%{y)jk9@e=FEmuDgPw+wH(9wLFSOG52XA7Gu?g@fA_OG{98a3etMQMojjjr z(r(N>qXytc743Y4b?wu5PXp!mrfHu{dv!OBH6}Wy7~_V<42^2Qo@yxiz+IE6ih35;#o#@HVw>a~Ed^6NG6?|)f`{{ir1!e^S{`%V1s{!)kE z36$`F zmzxQ@#l&0wx(?S4{G@PAgxhT5mHbMFs{wveI1k}EO}u$;=x|p6KPg-;;Q}UJ-mi7I z%YZ7}Ok`BqgP$054UHcd+n?2&84zAU`s|QLk6HcZ*!)|d)b^>TYrgSZV67R?AI*4k zf1}5*#lRGB+knE4y5G|2nS6!j&*AH3KV0!>@DLtwI{3CW=rPT1pDg@L@?56Il|J$) zzk;`Q{C@(ruY6l259+)nTiL^dl)-Xqm|%!~E8jjdufy#UM?8$|JrTxO z&P+8d4Rzi@Ma@V{G1BHy_CFw_##hj&X^?COaTrdvj1PhO+nTT4Lz)gHA87i#0~GqC|4F-NPt9b!8OuC6#u#12{AbRimBc5%0u!$bD8DHm z>hKo;B^}K>j?9nc6Gx|scNi$Y^Z%^FmjhM!6Yx)>iDQe2x8Sf2e-%)|#n%B<)_GH= z(TU$SzRp!bQ)md!Yz&^ z#;J_GwD^9tKUAAqkW#77R(c7q_Zv?$D*tX6KjN!w((NY6uJ$Csf%|)cFOFlg+t+j3 zC_}A7w->eFHdpPp73j5GmazrD5Bd7dw36(y)>xipY-3PQOHJU|?V8@|Oi+(K4H-sf zy2JQSzPx6$j*fAo{_HsBnx;R)Vf>!YI!mbM8 z#{Nl`F)o-VZas7&$q$b$M-XE z1*$&TFphrCxp9?RgT`@loDSnszL*)W&dX?H9A|-!oOJeQc5p_r-N=x9QqIvUftl>3 z26f%4ap-p3WPhLjF@gP_33J)UA&;tj1;ELyWkRD_uVup5jxr`QXH1@(Ga%)V-}O1V zOu5Jh4FA#Vi~`)Nhxxg17nvOSNk0&n%hxdR*f=cNAv<24nWluZepWuV3`pw&iDAdm?YMJCF}OgRdpwb_Z^Tn?BTcRU~@YQw_b5;OT_0Bw;2w@ZFBn^Za&;RHn+QRYr<`l&Fx{_w&Awc=C%*F zEw~-;_bnl}H^Yo_V&FTG}(4*y@*Yvw~Ehs-7DwY+x@GXH_ck;HjT><p_)0Wgv!Lr?#;}pxP34LFU^?}4I%9^-# z7w|Wj7p>~rRzcKGorW$+oT4`UE@mWY;3eNW6fN1h3>rm8%TAY$F*(<^{62zz;y&I8 zSu#EghIxZck;gC=;bOZpXKceBze8xO!v?MG*|1+DGu9}bMdzJAr=g{V9sLw=i^HfC zYYS=3C1nk)3({zG!F84Xpt?(o>LO|awLoq4>x_b9{7@r=qV%A#q_k>jX?fvX9djh* zvYKiX;Ax#eo{mt1$8(Lhy82qLn`b!Yh) zj4!3vs4Ujd9~x^KiuKlK(;$iXmNN#<4O+{!eT8vcVu4!(o;jVR< zXA-nK8jV&P8?>3b)}c5;w{{D+DU6C`R~1*DKd(TS{t~0r-`d)~#&29CruwF!VVr5y zpdqiJ23nz{kKSo4G)R*F(gSM9TSvfO-`v<-U(>`z>lR3ATq>bE{2|sHX6U%?!J7K^ zz&h?+9pi2m#Bdlj#cS89a+f9)7G70cwPNY=vWn7pUY$CI=5}Hv&uvI72jm8MV@(G= zyt$(zP*?9a<{K1TDL1K!>yN2vYnLnCfS^AV4z|e^L5dSNa(ms^p zvACDNZB28qy-idXu{UcdU0NoV721w+p-{OwBu&Klw}iRIZ>%uTrA8dIB4t>v^V-;o zP`d2$*VF`Rpm=aqJ^Kj>8~K_2CBSPaUGnJhm^*m8qwUnIO3=^4GLZ!f@HfY?-7` z&cLAkYjIt+jv8%UPNuybi%=sVCS!SOS-e>;E2qwV(rf-Ftt!+~To<6<`it8cz}JS1 zABkHzIt{{sMW%wm(zZ~LQRXM&b5(ON6s~Jo(FTc`B|ev0)}a)htrFhC-yu!>p7a%M zdhlw9k2z$7N-~V9^kx3galHe?W|pQsxCkYc4je4xR9ub9tpRCkGCs>ldWVcN_2Txg-~YcTAbSR~GO*q! zd+rAJi;XX9H`xREj61nJf@^)@2iW_~OTH&I=Q8hAG666`HMV;^Bl-_xZBzmy#iQ^< zx33x~HB;kJ+|bDB#^-knDLL!^vW-LmHnc9`^Hd=F4z%ux*zbbKosdf=a0a}@pqQH z**nHv;3Is!n70An;_Jse1iW(^cXtE_m@{3=i^zSB*Z7762bk{Wte!FotmB)CSzrra z0p|T5%N>hzXK|i`djW6*pX75h@DSg(my+k% zWP1+h2AB(g)8}&5gSi&?BffVr_W}WY- zKfx?;h;JWefkjo&1+!g_U1Z=b8Fzu#E~CvL4=%8QuK=^a)qEbz0>92ziCN$;`C2dw zJYt&da_Qma37K_~U4L)|^a5YtWmnP8F$?tZ^<%cnoZIEmMK0ZW9Wo}!Z3|q^=f*6s zj;{c-U9Q|Nb1t&yb$kur2wcs#7PG)0-zLlgzs>gmW`S?;J%(A}$EMjX8*Y~y7y0p6 zy+Nkf=LeQFP^Xv$-oPiXCE8`gMRxo*jnET!f$#EFW46nGi(L5cS2J$lE-=Qo1+!hY zTV%d>+{nF5+y%bG7sYIsxj?XuhRLOf&{rhVWp@Oi#U%yv0# zk>8%MmgI34IE8N$X1jd0$ZfB@85-g)@N;~RVYbU*i_G@4Tj)Eu3#{jR1GB)FO|!s^ zTeZ8uY`!6I>~h*7zy0!U&>a0I0^jD##XJN&cRk}X<^tdix04^tA_sl$2HKdo12^!= zm>_b>kMOy!h9K`pN!SU zUG#lE!5;uNN3=NrJZPGSfGh8&FOY`-@cJI+ooez7T>S;|hB*Lyn{O)SA>c~nh^xgN z=-jO52p6!IPx2}7#xH4eC$QiiZ5H@#)4UC6$s7y(Gv5QmDKO<;#uLmg;6r>r!MqK4 z$9;?^m^TAAd>Q!?F#}WX*JHH{7<_;^2KP?jQ+!f(V0!6lVEoQrnt;pKG{9nj_;x6!i`5wbO1Z;X3Kg1J#!(CwJOc>ln7WLNcwDlXH8}O1RC@*G_3H>`?A7+v7oc<*7ewMNT?|B-U zV-|VC^Y+jNkQ=ni42mq_rG1P=xC^|B??cSBK#>)cJP3SwFJo*iv;{u>3~^!>nZDG0 zqD>HURoJ7$r&`}XsM$1HMmUqcqIppHBM5A{PI%p&LZ`hLoeS!CCqd6D|VEb?eO zUZQ=jr2POzKCBY6K#}#@j9K6pUd0cy$Zy^M8f~MV_vJk+(nhIhg&GE58JB5%kY*=3G8a7%lec7nUWGx_>3i>y)lLE1TH7qI>Jnm%@U zA(100i4rIBIRdZY^I;abpze1V(=gj*eeCi&c3B>g@tOS|Wd>j1JiZSx3%r~!4LKcw zOZg^Zt^`j1gU*j#c1L7-?&m84N8tDP$}tO^@JGfI%mOD3X?Ky&nfWKi7TiVF=H?G6 zJ7$ra5!shaOFIoxrlc>#|n@EqN4y=fo%rI06^*ZNO}oM-jP{ zLWje+4|joA@@>Uj3;a6YcFbFVmfVTJucyE_{XgxUPe>F|9LGocC)$>wg@VK&$Oy6S zN{2*t(JoRpeubQ@}DdN3r~r76*QpN;HJI2 zf6?#R@H)`<$Om|CA6{Ylz)G))d?}&*JO_>ycY||i@cN3RgCcKA#{tIR z*+<+5X0$&{?jY+HTKog(yyOwG$T{To&*IiVx50Puq?k1K0bT{_!^`kDP^Nqnj@J71 z^uW_V`^>;kgYswa?Sq_${TJbIi2fKIg@=IJAf5|!9)1<*GW;2+PVyC5^@sfZ8{k<` zeU1ADe*{tVC;0qfzGKisumT>k`* z#mci$m2Jl+a%3q{PN<-^mOs>Xvc)@&Bi$sq2nZk^tW)BRm@`;3MzHWnMvAaMY;COE z+&jgg$~5IgBliQ4$DM?WX(4+(nRD!$=`*7mf{HsSGBG6`JF-XV!60p+$er8Mop*bG5u!H44V;?H}ht}+&3$xWX6}| gWmaF&xEx*XPS?Ov5h4ftEtf%8m%*C2JY1vi3@@Y8Y?Oklt>bZRWvw> zWO_T5)}^$ymD&wkTWRY8)FlB-f*=H}inXoLTD>tUK`jYx_x(QS+*uO9+V}H&|9SKI zOzw8>S)S)P&wkF0UA)Yb=ka**`8kfqvznv)1=aum@-N5ZIe6mQgFO$Ae)j0qInig2 zzF^MW`as>n1=lRR;@ZHKS6p}9f`-61uL>+|ye=^JxbP;~S2+&!j=Ej0*M^=|<-RiU z+1rmhO0J()<(>*v+_UQqH|2r=Aaf zoa;GW8t~sjWPC+=S$&(qE6ipz67b*X#%IQnXHJ$Gf_FVFS(3bStvKR==2 zs-^~RT{6KPr;OblN2+AN|h5b`-e7xNT;fd}n8 zKak_9y~?$pP^Sy3F)~)qVU%0DSGnK`*IYe!c0CpISRm_>B0c*kx3GTUl{`|g1BR;H z_I;JRYW@NW3M@UqRF(VdUgge|$Nzu+GvaTT+Fgv^JEPs`Dj40aL0}|{ZaXs9^S<}i zg14T^^%(IDrG1T~b3L0?`K9UdM`z04@0QQymMR}fm!Fz0pH=_5k@fr20%(egNAi31qx#FPV08V8(Ej#dx6iTC4~Zx0HvP# zvedE1NaR?5GvXc8+Dfgv*XJ}8tt}R;SzGOjJShWME-fjS#x5UGU$WpvsU?vk6&TJj z)zqLcq5siRrO_2G3496|49hGuS~oTvX;?WZIoykHsLX02 z1&B^dX25`FqP0JBAs`9ux@FbkPO-{p!mLsAZ?Kg{r0g z#AjMPX4y8=T5pzhM3aYp-K<(yKgCF%>Bz~!;PRbgav{Azy0AG`w&Seip<^Qz?UXmR z*^J-JTW<_*Gpjbyzo@smJzBNS^j7!SKRiUhb9Q0H_NbK?rRuycd(y!&j4?*uRwHqC zc4xAH&uI#N(9>uE{Vc<(_8Daz#_1xq7M`}dV`9T7$A6VT!LUL@Mpef`uBWSOMjYQ$ zSkHAtUB~xKx`B&IUAWu!=Hz-R+Koh>f=7aiM^Iy?{ZO&cbbQFSaGLbQX&=#n*O6D* z9r&xd&U9ddx^_G8m@eFHe-B1ubX6C7g)-td75lhQ(s*H>VIA3-Co_21^vI@vnPbn` zWe8_LZ&*W)|0!5hOMAm1)G~(}xt~EX6qC4VPUa4z+u8q+bbn9n^VS)^bv@n`zxR zI@a1APEN^h$O~JYj{i5nqqW`J`XecsxVgj-`nLWeFLqYnoD*!TzaD}N~)s_5$azO>vHT-58Rzc|+ zN++rtgdHVRtA-_=NYpEM)iBKR%hoeo!0BG3iB$Lz)2X3^W^OJCgkrI$C{Y)?JNBY3JetUF|k()n`7pI}zCwl~f)T6Z@fL3$f+Wp01)sN?sdFj&8Uf9nR`|;KgX!*t{)#t2AxDM=%U0}7;O%M$opt+-xkyk zBmPvgbhxpQDcAQnccRwRVk0@V&YbL#vHwstZ`~mvFlES7W#j^asWSHXtpNUuF2Okd z^O!#}qYJ%{?&zCOZShS`-)w28D{`VRph(5;XrggaxUCTgDJpaP6ZA{gfVX7_Le8+J zPlD;dM_iqqye{(l~(7*N{>fFVAVbBjMd zl;>@IgDcvdlv96F-&>$q-i}(lNUO%=$f(9kxic=)vuIhneJ?cD_k zdS7ab3_AX^p}a0-Wx%hCg$k#S^FA6G?5E2S&ODR?L;WxZw*uhwj1`#_Dr?W{Fs|Iu z+J4IkS6}U&*h25{O#-vt7kiG)`!~ zxaMjy>OwOQd%#lQN{kh`SY286OuXZIL$SA-MBy;yR-`k0ObuUDUCpm92wfPOF`Q>P zF3+O1ItEp6)BP!)H7RVp2is!Qjc9jH)%LJA@?ps8>f0&I!U_cq%O}Gtk6NLbu`3JD zlIezK83-j!CnS7mMlfpSnn?q0c3`BfJY;qD{aM~kF|^T!aI_WN`hUUE0wM2{Thioy zJmhP{w>VXuA#Y@N-&xcN1AB9=(x?*CCw4|gdBpOFje^aM6F$#08wJe@GwEnE3K^li z&up0p7lQLmALX4G^rYt(rA^d6N@LGZ3zN}g9ww*n6e|setE6mHu@uEy0?}m5h8Ml5 zSdd~bi@u=QOAH1Hd-2{eN%2kkL2J@-x&>?2PNQr`!%5PVd-Iv7X2wx#y|ObL|8jgm z@|L`9RPFG#e!{Rir(TbZL93JDAFUnSQNw9{+1px)*#MUec;kPS@mrl}_e}Fb@9p2^ z+Du+lDExa-L$qpr{e+m+q51YGJ2+Y@$-t$5g7EO8qlv}i1c<3lhubPO$D^&5&6-c9 zA69HM69+*Dv(zi_%881Lk7_$8udhv&E&3xI%klqWtmaC;P*-=$Rmj@ie+-rZKYSUvyB$mfdxm1kL; zFNvKiazZRrt)(L-EfYncy3UxK)A+@x71xTuor-HD#wzBN@P*5ez|}S3gs1tYn>`Jq zVS{!Im)bJnneh$fm(agwy?`hCPubU^&-%2HZ7tEZDSR<{Wl(I^?R3UWv=r+5>ajeD zB-`XpB=Oxsv005J9b)I&$IE%jbUlOCgc`H?#gKxyVj?sf+g?TOW)7od4mXS zZ3|~}G8evS`xcYm+G7j7X@h^cqV28rnW~a(mb};a+@alwlJ}BXHMHntqbua&bvVh5 z#lD4yVgYr^L}4bRSt*LK*khR+m@p01`URxWXtQjnp$LOG2Oo^%FJOS$piA83Tj)nn zb>>&{HX>_Sjrmn0Zeq2A*${F8%o^7B1*K7IiOc>KN)e61`a^eKqD^)`k_9Xzhv zvB+$`WX(o8&+d8ReG*o24n1THL`6m!5$Q{4xaW#^PYicU~-7gCAyhnjJFpFV{8}` zw{yI$c>g{9`Qf%j*Hf}bs*`Hn%0R`ec43XD$ivjl5v&nA*ztcG(5L#NxZ}gr8?Ib* zy|?8VL9uYe3xz6?L2$QTv#eb+!=LfgOwLry(7W_Du0)YNOmo8?`y{4Ls@^mwZ0ybM zPPnbSZE-^TP}6<@ zyL0?42Wj@$qONX`t3T%m7#5ymCPP_luSc=H>3CPN-4PLWopDE;qOM(cM5!*^Z96;h zV?MgRKX(;IXKX+i-*>qcO9@xSlPJ3L0Xe{cg&ic$Zb(_?!<-oFHwR@d2Zdi& zYQn!%djx7l=@yTTW|`jBJ>YZgG5iZB&%_FbP$sK}hH&C}TVCdYxEiDjibH*HAg-a^ z!U$`$TYWTHsUr zE@lcaYhnGtj1x5Dzi^o%>hCVGR`M%e?myL7;wU#8#=}{Y@DeUU@Bbn#Z?>1e1FKHX zoMgoR)yqhR`p1Z~^)|U{?Jz86&8Da|r7(&N!dZZK<9%K;tb+~fx+xm>vzx%EJ zGX~!@w!MW7bi-jrSC^u9s|#mnw5okk{#nV>Pynap09uf@yZ>v(l09l5#~@EYUdN&X z@RJgD5DI&#Uxrf0P~KoDQ-=Br?M}I64fG@ZE-k5zLMnA)FwGIbHO25rO;iGHA>BKq zABp#z+#3^R89UT(Zr06bI9uEf_RP0n=u_exQ*48JOVIpwn`sD(jl z`yyFzny60P$8+JYS-`olBHP}bBQRAwZ^GPPa$8x%PE8^6&mMXs2Y#Z5)S^1RUg1_S zI>%)LDSI1RI~A{)D;#aFcuf`KNBetN^+~`^1!96<8T!~L>+aj|AIA~8&x%@2S7ED4 zaSA5qmo(u86@_igzmW#8gwm+B1(QZ;x`2(IW2Y0rnFoc&H##t7LuOay<(JJ$;ZDuh z{e;Fol+ai$Gs*t#>-&s{-6@oh)|(*(HBHxfhU>r0lRXWvGE+i9wFM)TcizZ;Ubm0u zNF$hej^lIuhw~U@4(AAh-JC=S{B$TjA?4#Y7J4Lz(G8B^Y{NJR-B?2eOZ8R$1*eBD z2u;`K{DURIosP4*eXPf0{}M_t%6d`J=4d&1Z^;n%jl{zx9zx(o;_(s*g?o>ExMUUg zy~jRYvVvpSdu&U|GJW#hl4gB!S4kZw_yn%`0WcHYv9%WUIBLBn8YB_PO64m2s<9e1 zO8#zz8x}?r*ZYh$f=d3Hakv)aM#bT`2#L9%+)TtgoLBM-@-wVl(F7g;2e~w1?Qr}@ z=5z8RWAGzmZnx)|A6FT3*QtZLl^hQrpS3E=L@f`CBn}~Qj}PTBALL(&{0mSaS_)Ig z#N&(8;RQUb4P)}g64(oH+V~$xr-3jtL1m`2)HO#1`i*3=oh}1^`oesn8a%h<(Zp4V z@Wny7fe4Qna)L;S)ELQDd4bcJau2y4sg)DNd4vI9RVrO9Atp;~ER;Mqr(&byzl%_l z%-<%Sx1%!#DBI0ZD(uNrxFc0zFHPHWQ9=PG5N~oaBq#E!XD-K6TK+tyVBVF7e=R(B zNX|n`y3~~lM0D&N6|NXc4HttI=Pt^OnZlVj6eq^v@!oy{%Km#^U!%ucx_BV$EsgZ@ zFV@4q>Fp{CP@T;GM%X)cdL93g%?zAltFSun*e6zSc+WfbkyRWMt&+TBRDIewPZYPZ zE~8^8AD)&|e~I?Nm+dh+oP5(lkMENE-rMez7m_n_jIv!u$EW%7+BRFH9%3P{6Yv>T zTd*KffkfeDL3{G=c?lL)qk0FNn)#voI1MLYebnJPS!tj0CuQgMh*I(+eI8pWO2?SB zHMD6@__wgec$ZIwn@p!u2+D|74b)G_K&USl+C~E(f^|6lTrdc)y}6{4q9vg!%GVz# zsGW?t)ZbsRJ+!(8uM^ZHCapbn7ZWmhw=!?7BL9849=qox5t7LObIHg{mdYU^j{nw~ z>U65XX^lP&3dEX_*VK($o>q;Rg`Uj#qh-5HV1@McwtQQ`@5aLT8J@-?-MoMw^NL)L zaj(C_wc~%7NaN&D+$`V~$KS|JuW#y*_t=O}#9j=3^+$?`F}4PikRs^!RGIyK4TBqm41tkF@;3gC{*hg%_>=?=wLhI-|BbwJ? zcDywz#AqtkRctp02Mp`I{v*<*jgAjTTIkF z&zv0bEY3G!Xc(LtG)D5=Vxg2;YzN2xxPmk!W5oLqC_+0zMVf$GvPj6=@~W~kXRXqK zPZ&g{*h3{kaOKL>pjU`5>;v~ssq|Cslsf+TAG`g(QwFl%{==DwHyn({g_LJ93QMWe z=3iNd;&+mcBY$38`I2m~i2-ajI(qX*sKb9&_cQA7FVr=B&&^o4Q(e32@a?*Aw>^uz zvSn$3b>Bx^Md`=P82wNqonw}5)I=_1s|ekO!q=JBHrw+nj|#*jMVgCo|IjPit0@`; z_C12LK4@Au86j_J{Q&=N>Wz3yt9$r&c02#hUzKus&I(vcsu+DIvw@ST(um0NvL>7)WWq5_*K-kH6Sk_AekDElyWYzb8gHdb9jlG&~taG z^G=!0p+qCj*$#VDMTcp5E4B}Q)r9h^&J8U5;_3hmWcWqdp@ku*op@`j+^;&cu>Kuk zYLJ)wn2+!Ga1Hmj>IRAp-WNi)r_O{qmfrpLgJzYg?Cn7ue+586h z4Mh{N-mZw<%wyA<>XSStCVU%qmd{8uB5vBD;K*|9f;~cPC_P^WfkJF3JwtmbJ$XaO zHP}alT0{jw^atgP00R^)ZirgsT{pJLW6O~)f4VS-W~UV=avcB9QVl#UcA0KqZ>@?j z27)-}0H$7>QI#VTQv5#4`E+^e@P7j!mu0P%JGB~S6z^pbtzLYiqgZ? zY<_e2&F_lLJVwg5hpImI-rb&vG;}E?K^%tMYB=(i`Ee)jmV@I?e&aX0B26M>`Y+ijuo91225ED z@OX7$vFiVSbpJ<7{}Y%*%lhtTj4?6S)15c11O990|7LW2MCaG*&hLm-O`-E9j*ffp z5iDFtSGG4q7{Mjfwy(}--uRY!gAVMc^JaWXA)cP8#THOWuAsZZnh*UO)VO4Wuu&Cc zg5zSj<l+M7~kQ%Te5%i6cyF$V?Pl>mzf=$_zqbL9dg-G1R3cN(qY0T_=x1OWKu! zl*kF8id&;$qoIm*!a{+tWt3NqU3j>)-aE0Gu6W>|&U%4EZ-Fq+fw1+Q+3{LIG>Hhjyp&cqqg-RL(Gu#!v3ghKX0oi}5gUA8`?XLQAOmy36#r+Nr(L)(V%@Uwho z8JVGjyUe`vaKOI?KgET(S6$Yb<3m+v7B`MdOwC6t-K&7Q!?>jbp}Fnproy@tA>D^a z;^2^V7F^`4;t>2LRM9cGE0lL$abg@w{M-Zn*SBV1x9G7d`+S%@oL{mMVK@^kA`EwE zVYn-C_X;2zsdy)1{U@6Ey;w~tJI#pV8bJi+L@dKV7bSbl9NdMhQ0K**N3C9j$TE70 zODQzCD{KuVT31npycyK84dI{&N%&H^PL0E(VYj0o!MRUX`j7V^;xZ)7nR)Ass`cJ@ ztMp=`N2ZD4${TIZX^ErjLq)cL?m0d?f5^c{LmtCtw|Y*YeU`L(LAup#ZmXn%J|ZJG z^RUyOTe!5JK)AkPeIk`s8h=Sy2a#6C+ptR$EQqyZG|6h1%SgRHxR<6iob7dJ?wQhnNT9ijLL7r$d9i zno)-dOj!qntAcQ9#5fkfb?}L29sdeU%M_FTCFfD=vVcrpb8?=yk-C32c^)8OE~zt5t;bHIk<(L&hYJ*P1;FReHkLm!aCl|B|v@?iAZ4W^ItvJh5HQ zK3F=K0SG}sEd0NUAPBI~BwB^z;kJ(e11fqcDpLTJX*oxPQfx&uSu1{+m5zV&fGeio z7rXj1G@3+Ym0?{7`$72)S0Wlt8|7*Ef?*vByD0kJ9VkiMg)lz;O1P;6RTO{|q{sr-3R zY$nIAk6L6~S*KtAJwfeovs9s0o27LJ?S)zh6x43DBgZM!mV0j#9YatXP0qnLcI|2x zwe2oyk<+{071Rzn{$c?usNJVAojeUScySNG2e5+R+1@}bspyF$2;D(mO+V=k=NH;P ztx@=dXf!^P=NHGUE{Q@(+Q9-EFylLmqbT{f`I}V3sNwek>Cd7mv}7yR4i?R-jJcY^ zVUfZi)WwbdJh-~ybct7~(o$^Ms2Yk|S1t427XApsD9zJg8s#F~JpWi%^*1&!+VWQ8epZ@qXMS-j9pK`*A%r1jBPjf5xmIi}qrc zZ55s7^Eg00?4OYRg7$%+((S|tven2VN|>6hQ4A;T7bOY z`P$7aZ^J`!D^v%NnR!~4Y5IG`g8txiv(iL(oc4`m5#g9Zfft~1fp%>(UNn)1yy$I_ z1gDfML;0q0lyn%9mzZ{x%qMOIX{Fw2EQqzf-f+!lw3zse78}xzlf7FL$BEQE#+@U( zCWzzYeHC_t`i~HZ?T`A!RExBU;BHdPM1byEY*{W8MeUs_YF|QyX8c{KJw-?zEgLfw z7MoghaAA#z-&1i}u>PfbEhEqSFl1Om&DtR8F8tKD>@6Bb4zI z?y|#^8J$cKL-I_W_06YE^fV4elLdE5x{ieJr8jOotkUuS6|xb}Mx}T*#;0=MU@eaS zM{3&g$npPLofbWo5IS)h%5n7ZDAdw)kmB8hO)2pg^Pep8p zJt86_{SQdwijdVBR5uU9#FmK?KPl(5Vtd5<=#Zoh5hYK?wqhJN*_Xa7&eX`(iuN@~ ze@q(`3EVTdX!4vwijXrHS*o-$@7Qy23PgVBjdW1nk>VBY-my~ zG=;sK{)Tep93P_4M2?S91Sde`DW0Fmu?v&*KFZW6l?{_9b_OMC@uA*))tz1ggz(^a~V5AGC2u1!8x=Bm{c#)@Q;TlHKXdvg$qXfeH znkXS9anQg~qt@4OI@-(2Kw(M)@LNg)aK%62VSD4JqPysqtP6`#-3F%?qoO%C97(>A z6}u*J=n@A8O~8n_aYtXStl@c+W}}t{dgEeb(Q`||v2|2>UivjBa9yMjhzMBYcw z9v|<^X*gci9!&8*T7A^YTN}P{Tt{2=QH`e}AxeoQmupHB1`rTW=|s0!!Kd96W1UiI zNO5McVE>+!A;Y@2q*4aFWP?xx0m-Zg{|I}`jm&A<8Bi2``r-7PEYoOWGISG7+(@MC zM%?XZL4Tx-h$agf6OpZ;RE{t5dj&+w@lEg8$j+|F`?7!obVc3-T@kN$Y>aha2lD-2+ZL`uOqWWy4UyR{QE}uD<%}(4dVM(0Zn$Z(`oI2>m`s3_!ghn94EK#mkr zmBVBFp2C=fJa%&W8;Ek8wjpFTbwwTmY#w`TSWS~VqE_RntOHoe7$Mzi=o0cCdp4x# zi#^2+vFXpQ5!W@AN(FT;bv@1H$HEgPD|Mvd@Db?mw7>bWkQr5z#E$A?onLHr*twFW zHy%i&^Leev(!`-{(3?Z>qI=;^X$SFnWUAu5>1q!Zr|7Cw=;}>dg+5^KErT=H< z_l(qfAFE(mJ>8YKcuPsM8pDKBWB_1>u+F|2p&%ugGaM;_iGq|Fq@cIMfnU3gD$wfA zb^KfLv~s6M?j&afqRDAehV>%q{)|FFm@w^!DB$>)a98kvk51ggRLG;d)T3K?)PFv= z;5XmWmEM%8)Tb&vOIPZXN+E!vi`AR6M^@_N(HZJdNWDO|fD9d1)!Z%_R(eGTCO|#r z=)ZKHm0@ilW>G^BoB}?RQftX^+V3S$3W#Ky#_re8PX$KZd&==L4*~nTvrEWy6ttv%{P*u^g zCCIpWN93q@TYxIVq~o7XPL$>(zDH5Gh`2xy+x{b=*z_Qgt(Q#_kO z{;6vGYw9}wscQUKU3ft9xITGWaH-QhLE^m9)bbn>-4#N?XI2Kv%9RpxTk^JSI)Q_0 zD1%)Ic{CM!kFD$W9^15+t6*%MJ9}Dfk*dj}N~DmT*WoyS`3Xi2BBO*~ z4LV_+h#l?fX+o|v^8>$;<@{FAT$=}Z+!a|S(uG?+{C4o`& zn&hpL9DFO&=^gPS;k#Z`%+#!$>Wg{9PfK8hTNg+k$yVowGhu$n)GMS1tv%A$MC%TH z=;aVi>YMnBW~%ZA<=qs~NRM^5lo5oEu0qIGF7TP&@PAZLH&tHZv|*^cROtXK>GEoR zG#2S*bl5P3gfSXd((M;+(OlXvD5GuhOAG*97{9v>&Zl${zGw27X70S)XaGb-Grs$3d89az>rD`!Ql2{~}O<-QQ$78%SelIy-sLiDr77`)_{X4}_0YhP{t6>S)=g zlr#r-#*!zGBLU+8d5_RWG;aVHNC)AjDg?j99wnx}^=WM0r%lnkPtB^qhOvUQa(t%J zzFFloir&!>GUs(R8Od7;iI2Gcc9RT)2!JHm~y~ME(W|ym#T}H2b8#8g5=3#hmX{# zy169Sc&NteA8!`=7L!g8sfLo*3D@+tNG844M}E*Kug}2d!0zw`u{p3idr7|I%UYn8Vr;woGU`qG~UV>Do&py7tnTuDvv(YcGxI+Djv; z+GBNSm(%J54a!LaS+Mlr1sGQ(xd769Gg|frqc;cNHuL^Ek|5t=E|qAaj3ys#7I>w} zD1|@kuvqdI=yOyGcSqa=?oNRR^pBRnk^Ld0Y9=qNW1R?=x-bs^i^TSrA5CWbZD{mq zGY(YixLlIhr)W;L+^q2{~gz9!1ejhqRqSt~qG97C`g5>0Q0 zLd&))Q1r<<(qXK|nD^dk#%li`roQ^^L;o<<7<|Jhd(YVRE~!2w{lI9*wE7i2j9G7* zgMT0HTn}aY^Ifoq~PZ`3e4tfm78YyW?rrH)quq+XQAZNo z*j-lWLJg;)!`c!m>n8tVmqom!8-vQ*@+00g=5;W-u8`MvIKL}={F}?VP2ct9{JSw2 z@%lnF{4;CcctyxNHbK zV{3ihv2$Ptb>-f%O+j)MgEjm!C^xpI&UD(%ct=yLYMuACqk)}(4HL-(2=kty%UdBb z1h4RNI|P}VmJ^+~yD648h$pt;5b;+)|L>*f|I@~uqN#n|fRCHWTZNBHirC%^+`mZZ z|I`fqi#tr{Ur89Kv()Jl#Wn8adT(oz3204t^#;iddz}?eDU)KNHpJ84%0@@X=K%t*}Ns-ENw0P+WsGAjI<*QS*R*GS+S{ zeOG2Z>8a>zLHniaWy<#}enp-n7!qKpVLzVXfg2<#VXujwJoyJk#~b;X8L`jFj{suw z51NDjT*hMOQ%yimWy$el$r50GrLqJZ{|yqgUrqpe@O@+6-=L>gBr3tCHz{FI)Ce7v zo}%gCe4qUeB*8N9CM^t9km8vaxrH#02~N@wRAE%?Hp?JITHOa5Zy0Ay54<8l=38*&pTtc3~1N;-iz4VfmYDwiDwiw zNG}&W40q+`9B#6kMA2{L=EdCfUcBXM!+Q%$8jGHjdsDbq@v=--q9#uKw4W)Pa;yEM zLJ2mdD1Col_(lbu_T6cO1%!+V7R3c&pDuhm%xooeU$BBI*XApc77N1iK26`b*R3MQuqWq#l#Z-b)t<4cbTvC%B0E>@>S^A<7>;T>O6 zzI4{NXkR+Sm=pgz>=!+d?XiUTa~ z9k!IQBFoLPr}t?ypCN_m)i9emwEJmgGk;MYnpR}Uta@5(<}b)~w2TF)pD3F-S3Qq^ zqHN~>Kw+Ds7Ln{{?TwFWk3zL*L1yxDq`;1hJtgAc67+cqTc-8-1FnZwkN&F51`6#r zDqM9)xb_KxkQx<;NwKdC?;6g$|8S#X-oi1(6Z6U=nt2Pm-8b{zi)E+dHQ4JnmlS8& z_7Ac=1Ikc#h}Ho6R&WunCBAkl=NL}?g+6-%rt>OXv;VybE?`o4i5m_0EJ2{Q{;n9-lT=a@s=)~d*nfYQ_Gc%cu zSh(;-VFAhMz4qF#2nnD5XUcM~Hf0r$PdSJ$6|a^uhb2hx_&a#~3Al5YYHDS1FB1Kp zF{+LQ)S;cBLhPGtCy8Knt?Y?e^+{KFHC^d%{v{APy*iyx6yJgemCG@sS(QL0_# z%{~3S>ty1oeC?MT&iya-`1@%QR&uyF79_wazGoB}@nYx%aMqVeinbcwQXG$Kz6_-8 zV-d!#X*=m^QVyq?xle-d~{#wyx9%QQce#(JE$5SERTm z|EEwbJ>OgsuV!3JJ1-D(j%6;tmeQL zM#zC0x+GW5o#O6^oYWpUX`?hb6M_+0xzQE2+^w4cYsW4$fNPEC7-5E0!>dj-~IP*}+6Qv5jdWVx!T6%c=@cJf5eU;QF`^cn*CcS&^ zU#fpr(MNLxY!b*92uZDOpXq2~){FMbUmiB(%@nuaEY%omr$sOpz(B)M|1XIg>h%~G z3T`%nn*r&QA>J)EGe}NfK-_hK-F=Ke?DQql^5n-9`#CZtu4Vz(v|PL64VVbot+N); z!%^YKKofX^{bBN;c;(@#zh$H-zlcoIb;70^nh$fp*x*|^n87}a9teGPW_++6rj^*O%vcIo} z-MENJcl$MT-dcQ(D{q~Kny4G=JUlhqd6?l>zZLYEAsCzE9j`+nIQ$ z#COzYVJl@r4M=De69||J5iyG-a;IZE(L`;4_|6W)DmddpdM)wLRx#3zf|PYFe+BDVd!gmo7D3pX#mFL`_vQE z3q!O(7nSy2Y$Rr}Ep2Kzbq5QqggXzUx^7d|P`c}7y6d*?dTf?-9Y@ASRdIkyQGd`o z5|JPkt8FmmG}G@?tu?7y(Slgwk^;!{M++BJ`;+4AjF#7OT0mG`t?NWs~rPn{kMrIeNK%8Q9_D00G*jr(p>3^QcBa-_t#P}Vl=!k>z2zpaW=!KU$CR!l4g=>9wOVH{f(9s zGKsaMs7F%VR35XgtW+*NSIDQ6PDaFchxzYXnMQg+UltoMtb*Wqsp&7Pxo>2Gi z;J)?KI&LeKO+ERAdU7{Uq`B61s^rkE4?X&@LmzrMtmb73M3yI7H*iMl&9v0cshCU? zuA(;44rAnKs_rulB9&I8IT0zQ;{lvoIUJG0R3*}d$IJMYS2&5qWcx8)7zygxCj09s z(#B+!AK!4KmM)@}i}}fOrA*rQ&_PDGlaaz5J)WgYB#Rz}9>a%Pg91#5)TO>l*+uK=HKi+RY18 z#)uyDSN|!Mx0cg}tFU^>v|=qjHTYsxgA3PaXO}cEEm`tZEbsMtt(kAER*-2~4li(02# zbGdll-;q>cr{Qul{-iKaPeUMN-J@4GbeY!oWwmP=%qNI03Kppwek-P8ek;yZnUj+q1miAge(ysagO-_R4Y z>4#WiO(}VjwXP+5m5D`QjZ(72rCXM`d{Y+UDJPxm$S~U*KOU$HVJiiywN0wkl1K^N zESz6*892X?bOY?#W+|q~rg4V!BQ@^6m*hd!pqr{Lvu*A(`^3WMM8cOq!fUfs_~UaR zq400noe?kWK9TU({UBlPfQ)Jw5~j3gAwj|0jK4#;?R%mU4TEI;3~kaUwa((ga2XKD zg5fxZVt?UC1w(nHVffVx)7|b;vnUHe3m=GC{~?Mi8)k@VIZbAjC=yXMvpO%`duEaE zkYXn<0ubSw;8@8q^8aA9?#znnnN@Tfbto>VnN8?%p{NUu9(i5u#zVv{!(Mcy9#lJ8`i0?0{=cbtVFP81z*-3R#HpE>?YQi32;c} zTr|Y%FI=KFNl43bcePQPr*x)9{>h6~npH0}9vHRMPH;2(#~W4eHH?dvy{EigWR^^L5@wjE zwpN*mwOiFFx~4(%pZ|HiW92g7Yx!frwbUer^d$% z*Qmaz&XF82U|P2n#uOnuaj^hKDLGEVdFdux_dyo3efO}@R8TY}t z;;Y25GgApvab_Vzr$Wam z{-x6e(AO5hd(Qw={Va5GN3^>EwQaws{sJQsUOEER!v>R5+cw!PsQaR}-Jvmz!iXbd zX3YA;d>pjPpI2a9>r%hy_gXPunKh56(Kq|7aYFaxSjno%I~DcI1qn2_tBb`eCHtc4 zpN;b*L#T}4V0ssvGSt+M*Z^mP+*TXlyry0`KgR`@ydfLlu&*{B)98#5$lq%NoEm#a zttQS|(LQJD0ma`$S$w(S7Sq5*I1qvaD^a32=U!JJi7X_L~sq#`6(pRbSb?Dk? z=xEQP+b7Q%aQv~g!ub33AVd$*eE(|bL&&4X)G!o}Mlv}-19nvmT@Zsa1k$W0K-_YSY4ZFae8YrbA#Ps?IJ6;8j$H0^KZn8&lfu+DVt zeQ0F00AN>a2q*l?S{P}wJ0pxT{CfkgNa#cO zx9TCCQjA6!X9gw_mywzLFpn13cD0rxl7y=pU9F{@f*E`=@^vUrKBgOSsTGEG%B|-E z*JNacOvNBBj!IQ58KFLLONyf-H9?hWdbhkULFyaC?UphN%C%KM;8|4^z?L+i23=O0 z32$tW_tG7=PAQOiAR4tTJa$CRo_@|T24(rzM$6V~ z%l|q1yo;q|vPrs#<^S9dwdFrD7CgeMFZO=+20kloow3s^)pRMtz;*mdGWTkdxh0P~ zxZJP`%rhB-^7^e%g#Vt}8qhj|Z$y-%DJ&C^Tj!M;iMy9`N}M=kJ!@3;uq7WKg;^Ki zlr8zt7Vrb4+-_;_fSB6>eyrDu_yl`I^43vHmoiUN8`A_))sn(X?U#N*CU`4;RE6oT zRxxkXYs$RTXW|W(+g+`KXMIK*qI)K${4qQYCw?F1v`cBvg{rv0HTod;f^KmY0M_0L zIA+nLbH!V6G~x`r*HK1HhFTsRtRD2P#$EA5GxyaD(~sUqcXIsy=8;rZuIbCC-Yt4j zJ^Q12*4pOQS5AFTst4UX2wRUV&r~SS?pM!Rcowm+H%W<^LU#3(QUnKEfUy>q8O@ZQ zQfej*UphZQO34n`6pBtByJ)Pz%BXmUAZOw0eR1B9n(#v1#9QOxw_yq$o+^}iSC#L- zFk-zB{?dzH<#t(vYniN#f2|_aEC|C9>-ha#8A<vbBs#WmqYfs&Fkg=J2cIhaaZ3nFn2w+45YxT!^OHkZhxI zx^oUQ2!Z?NfkFj1ms5V*15mxg5*&UgJ31q7BXQ^82-Aws=y+Y z8o!PHZl+x+IrZOCNtW)ejG4_N$^4a4O3VMbRGz#LRr^)34aERRZF6bnZBSbN=BT%N z_%@fq&n43n@ZlSV?`NELC81wR0{SO_P1}6(Sn=f*)d{RiATP;wSE= zm!&@-NecVXC-smxCP1&cEWiGczF%u4oe%54lMV-je0$?*F@VC8uk+n-d`cl?J^0sse` zc4>y~l@|VL#9)cfH4CkjiyZ^y2j#1OVoUKttzu|0bjhE^-^nLTtjH=8HJPsfRb@8A zOwx>8r#8b}%r)Pe`+bgQa$b5f%sJfj`l{IsQ@P@0KTPdz_y3H7Fu&B6m}gKhnwU2f zPLxTHN)78EujAkNJ0Ur7q*;ZI|1a)Uxh!TDHA2}$aUQct5}ntqb!4v5W^B(I^0 zeHpybXV^~JNN_6Kizm-6Hmh<>ZSEe=mEPCc%Q&UoQR>x>*YnLK9}}_h zEId6#DyM8&{SG8aqf=;9Ej0XxTt~|WOn0H-v+B9K(6C)Sly4$#vF}Ps{^(ysMc{9u zV%c!2xlH4BJ<;=Uut?+Ik+&H1(?VP$HqpFJJNTCtZ`Ek0GFr8!bEjNK%Z6}uC=kD*p1T(HcJ+cGSNQ%ZFy7{SGs*D?yY@%{CBa-X0RP^DS4G^VQ9|X>PPhj}&oAXPub{q4F z-X7v(3|hACNsFdiqlsKe2eFjV98J^}X3brF(@3zXUg~+UIDIN#3Uf_uHfTx}s4LH$ z!X#4F1~aEd=CmerIy0pkz#`ysYF}=o9bP!Ksize>bnyCoYe*){OdzNDgHo`%QAv9E zF}4VjT{2HQ|I0!Z*&kH)35{7^rgLePZy}@VwmfA$vdw|4(&;U*@(Zha>M>K)m6;bR zjk3oY6SiS$)G`WVRUM6^+2gaj@7GY`u=X-xCg80jOR+MR`mmR<$(jSTpWXEC{_De* zF+Ob76i2F_X*@8Ti2apcPyfZ)b%p1u%0liYePz$k9crvl3y0XIZhK;SPxjV!!=gSf zi9(iDl3aW?2o?LnuC9XiSQzDPT_eI?6Gc*-WTrJEARC9W2cZvO4j75p5IUeJ8C>$4 zUKmcO9llLgD*Z+Fm5@J?5S0>~0{f{zP`rs%f47G*mQwy*wly*>bhYDuDKlVedTk*K zu(CKE6^E_5K-Bxtd)XmAoayhG+un`ly=Srx3ysIy@@IM_;e~*k{~npURRT@!R%am% zJhgNzZ1u1=UUr*$_bb$|{cZc4%>My_K(r0Kdjd1>*8xG2%^fn~x#NG$IO01IY>b(8 zSUDMhq*i0mxeRQg))On}@?4Z0>8I}5@ZAWlT|)DLm^Ch9oms4RChT@MCX}UBAwJkF ztHu`Qhso9zZF`@CN-Z>tu?QZLAH1Go2Y?Jrj&b4RJ59h($AOG{eavTJ?BN-4VdQcxwVC8o? zzZZ*Zz%IjHX7-jQ*hNgwStp2QGqrSbxwnPH3>4N1zKb%Dd0Jyt+unz&-|qXORBWAo zV-U44>T(taS~%w_k<+G2qXSq-wy+kCN8 z>wp%Yj&c0HpUd1Ia{Q;>&&mF5QyDKw7}ia6wSsQFc7}2!Ht<1t*YjVJ(pH(Fx^D+R zF5G}dgb&pTdCd{Ierqa|N7&omF@XNdX=Nzwq`cxY8u;nH8~BVMgY5E^i7{@q$KdwP zz@BwBxo80j8pjG)_6rT!@T=L(iFnK^HVR~m^!yTY@&SwU%mk}62{?{GY7q58CcRbt zHn_DrdUK7u>RuxUO*Ta-IRyeNMSlAvnDFFs%)&GgK_=;{blq-^D7siPtfZ80z2Jmd zUmBH{`!cPB{e18Xx6mAO(>0Os?EfHr_&(etQ7*$G(|wX zk)OKonlFHZc=ru2cbk7mwrZ4+>?lJ>*I<8Owg+gPi2{;MC&ES3f`pZXAf2`E`#|~@ zVK~Wr5`|BGIzZQAdH?w=e6Muzy~@S+YRBJtuiO6zWgz?Qzp$eAY&dLPNRJWjx8At+)1rxcICUs*-S}5 zXFo~F{qIdENzPI0?`GnXLKBaLnW!&L?M+n6f+uU2nWzSJA985t*Ub+Of?Op5#1l(Y z_Yfi~F9s{(Q=1EU6k;c#Hc#;xR=MLp_Qx)uY6N1GEu|>VjcK?8bZR@Z1>JM5{;<*9 zJ^K4bW^!79d89TKN8kr!4Y^hHI>{9FE4!KY^VwdT%%$Q)XuNgiLHaC^ZuH?FxsAR| z>$F97qTBH2Y_wR~lXmT~dk`P&ZhJ+l$~GC0LU>lMH)g)d5wyEs5;zP`ym2KO62{*1Adh7t0+wc6#%)U+NpYn@>N-BFuEu72@n=`{ z2UXSLa_Nb|+wY!|-4nz9#%I62!0m~V)r-@AtLm5)NMmyE{p2S79s||gDRRF%`WcCO zEZfhuqu(is*kXIT+Sr$TW3dOX%UwHMRS!bcRV~YfwUpJDCM(8QwCpv%hh58tWYXtG&=bgD&#nP8&{e3_y0i$omg^wCH!5lJtUJ#(s)v-l^F%{P^@aPuzVimX85VEyVnOS1f^!`Cu)<<73LdCmU@Hsx4hY@lSIauTpm(@eogip&|;RMxky~!?u##68f=!_omGE~aPG9?e+%@w+Og_b}(>54QT1ZY%C zBHfnpTfuK7zf}^g7HQMv2CJDVL76Uw{Hu|FwTvRH1r?c|^jb@xASp7l@na>6&Q3bc zT73Q>j}L^#>g;!cHX&P*`exN#jVZ-oXv&jV%m&9)&jz*Z-$U)31XxW?6Re{qdh0`2 zjEz)sT*lGhxPs$MibPgPi8;W4i7n})k9|W&GUckyK3vp4nTc~m?>ca89i6c+Z6vUF zQz2dmW;xM2!d&DZ3h3*?{A*I%YWU)g&wJZNg4pD&CbdOcc5tw6+3?_CE!V^wzCSED zcs4h6a4<3;kwJEx@-L-GR#>oD>1GO4N*$~plPM6j`YfWUK)%M_d9*-Yxk6tLK$uKrT=h@10deuV z)DTN93_Tu{%fzpiakzR}#qR7>fvnxTtdIC2WyR3yXVLuz-|ToR7ms7vP$+Q*J3v7( zW6^{-iv_%630ljxP!ZXiI?^WJi^?BOQdBH?4sK~#`6lch4i0tll9`+i=O}vkE9lD) z^~=~J9;-0E$7y7{wU~$9K*Te~W(8_tgVg3D*u*Q1?K9qYyk=Uj%IZq#; z^r-ymQ)W>uIX%tXRhjf#?BDeOt5njKKfNHta2JI9{TJ*h>f|*}A$f|ar~Y(RU(i*_ zO}DDr&#vnGRE57rpbegzyYPI`R=-h6bwqiASOiyTjWL<)pul(yl_Kp}}7DA+T^+d&5B? z@+%tsp`JzZZHX8gjOBS_v^P=`75bzF9k+8On`7w{3d~0_(lg2g%)|4NvzGj|D=8+HiK}ZKCz~~}f1TQu z{4~9H@hQT&tIO36#{1JYQ;;V zj#;;GXTmL1;QB!wNwMB%$^!;e}1&5((;+ zF2nZ=l34H0h|Ef?w^b3J{(kuBvTcyWnwO}D#G2hrZLe3?dJ~~~o?oG^-B|Mly6}MH zOqMQCtP+kU6KH|-0C&T_Ki4>3y|?%08uN91Bpa|fJ&k=p2v@T7Q<3J^)wLUGeob-P<;)tA2?_FnX= z{Ci3MZIgc++i+p;(W+d-YZ5he$mOzn+*b})Mb+6VobBkR+4k2GvO288- zF@X{cqhK_{klo6yYE+h%5@DX4qrydvu zych5GUc3{L@^DErr8${j(j-rML*9$!_Y${em&o>@mx?I5cssvMT#>HHe~b;TH*kh( zBN~)7RRmFxm?;M&(9~kNW*$#1R1(Tawy3elMS$_WN!p-cJxlWAig*>R}aU^a1vZPr*W~=#Os1lB+0M5*xopCQY3A1k&%4@tD<9Z<*+|Sl&nuUSSv{SRb|S`~9N=AAK#4Q0mIr*|pJ$u% z-q7ko^^X5VMWeBH`<^fJs4JpUi2LwDdP1uY)kQX>iqyMBeBAHDF+vf)F0v+7q=q6? zA|<*a51u3i6RO~0y5P!G!AY5d{a1!oquMb7|Ibs83*Dxe|DOJ{(p~vc>bZT4YDXTs z?ev$Vdt;?ats7QqHie_sN6Q#xu?r$!X9VgaHB=|kw;OC#Z68#E$zfVuqGu3Wv~wB` zksR8_lfud1!F2Lcs_!2kTCK9glm3WY7Nr6DW!<>Pj?j42dLR3g)dg;Q1D%oHto9gk ztk=HxORBw2Nv92~_KueJ8V)j(Q`qEqDCdSr{bz<&i-}F^pWdmPJ3wE)$)&301$r(1 z>!$vRq1Eogb*YDI!s#^{qYyY1S12M z{%J^jDV&@lkobYymxFa*>UDDg`z&eBQhtEt=3tjL4cJuCV-9xe3v5i)p`cI-vw=-E zDrsz)%mY-U-Hk=`wlbfGeRuOuE)27}y{AsTX`=T}X-TH8KqL~)rj zs<@o&Rea4}#p%k(Qu=isu%7`#bCKm;njV`>%>Qw0@`_!xSL>cax3ip#?1p{N-o*o{ z;(@)3|6{^PPiu1_VxR1-;=AyC&x^GL;Eieyri3~9m^^|cK_u8T)JNMA~)<;kL0fP zk;0Xp@*s{6ELUbt`ha*_D(wT9V9fe1375D0M4%;{aGLr*NfARURyL6Q*(``Gov@4!N zCzmvMbm!ZqiP>tgNJbh6s|I3$Vf7P=%+!xxFjN2Hbp7b|l-sYaiubnnhg3_&*)65gsWqey$!^JE z_-RPtR&yc6J`HLl&tEPr$r|syTRL2~v?IHv_F-dsTGgU#X>a_G*)9Dv-O_j6mgcFJ z_$I<9xq7*M#mjwhKDO)luLr>F2mCF4xKbZ3<*>~0PuExH=)+Wf2uGP^J$h z`q>11b(lVQ^`THd%h6XK-mIE^UmxD)01ZxOx%T!35}6_^Qo8IHQ9%rLVv=#7a#vrb zRqho)8I`+|AVrX!m{@F}zWNp71K+dOU%l*u2R)u~geSkw?@E5R@w=a2H$Ty!N9ezX z)_<1?>SRjpOaCoQ>A%f%C9VIWp~HIQpPE>P68^GJBDsxP2S(+k%JOTT{gN=hU^j1b zWR;=B0(KfQLz-!qhVkFZkq|&9dphy?0ZG<4OK~e!bD_Om3a7%S?7zWM3e%AND|jK5 z5d(yUd?TdRbn|>l>UpdCTqH_Ao<=Q+{J&1P?_cFUm-g9&DC4iAGd*r+x;rDQvsf$6 zS;%VkEt?9}bAw*P&Ev^%Ku%T;cBD?16cWc1n9w71+oO}F>| z&~`5HQ5DzY&yoO9kX_W(Xi>Hr+bGsXv6^&gy9>MIF6=_AqEU-t>jT9HhAa;i3~nOX zUbkYaRa;x_ueDlh#a4^>P6Uzw^70wAD!y-wDB8+HtNZ_+nR|EdW_j3t`secrd+*$N zoH=vm%$ak}ocW6Kn~q!vEj2i zqdhM8ZUMd?>$Zwr$(P63u-M#vKiyo9^*%ikrrRJCGc6}!5@;+hVRi~Ruve1d(?S?a z4k{G3QcM&rV9pgGr1yIM1NON#A+<>f>mfEjuEgdLiB#7nzaoKA39Eyy()ww?%q3JM zndP@A^VI&C0iu57ihQgY>6?6TZ_lG~kBOSV*5(5IzEnFDs_ zKoPiBx+wD9<(&)((shT7$n1!NAs>njUe8V)f(UiBVUcoT(am1 zoEx;~_jO?-pIJtq%<_|O+PZt^nF9s6C?*Nkm1OQWX!-gTZRAL2w|$bV)-gM8S;>#l z8y)e{CPdFJGJ_p}b7V(5^K-*@iU5eg)pIFEf^`>}BXY^Z>s)6SH$^bwzXrLwrQp|bHdQl3FQcK;-Bwb(xLwF6pohm z#JKK3&g7T&B!a6O9|@aluyu;>?Z?K2-ABueLg=gE{5OQOSn(5e{aQ&0^6Oj|Q|Xau zKGz!0%*n~GH_AAP{5n(pA6Nggw`W}#!MZ?{PXD@A;pqHt zqw{?9|2&sxy5#u@seKLMWguO?{wVi#f_xQ_`@Wm|`i5ezbmW^!fJ89b+9q$h$u9_j@>|DIy7i(#DZcds_Qk8WJk))I~ z+W?CVSl|1Jr=)Y>B#*t>s8)$BPh_*518Yg-atjG{V9U1EwrqPmmv(?bxPn`>om(JA zy)EP7h_)N#McX61*E}XKO0@m6%F}ICo=P9JXzRV^L@m{BaZ6}D1y$eau6hqr?W!qK zwUTO|1a(Nwi?mdGzq|fh3hMt)q}mIq-$PY{dJ=fFK^U88U1rDKS;G#>$kc3=W_jX zk^c4*{q`gMGe!SQ(%YwBEw^917QvV#He-4muipPjpJ6zp?;>fiVMBK1V ztjM)9PS$d*h@iWaYi;GP$gdotVb}NRLgtBYG6A?J-pPO0{+a(4-o<|(JkNip{e%C0 zj^l~-mz(iO$P9d$>`d|()ek4Lvi>nZD4RWIlfU~)Dtqff86P>d^-{0>#Habl>{fT~ z(-U$tp=8SXXGjH^eQp#$7EK~KwW10=wboA~5}=*NaUdN=(;oIctX6|R{E#B)QaB^W zk;BW>Vq%sW1JwhEeGPMnRnKhD;J%8-=0`;CZMxLhLZDZsFow?ikI^c_}gC9PAq5^M!;257>^)Q77nLsN8fMv*W2<-;n(KLKG(VDQ5%hI?-w4LDmwby za+#o_@aXdxHuz!~p02*-(b1sZ^XM?tyN!;Z&fIP7OK!G3C+W)}Fr$?7EfW=~-M}8m@LWkM?nHuf&iq#k;`PnD zdd~I3f78_k^foeg)LPNoNc1kdI%-|2zU7hp&r2k49?5U1cN@tss24OIf0?`oB+IU* zJcnBMoD;k*$Q?!J+S?7Y1+Oz^30{*kG+v*V1!re?y-G4zpG?{rUjHGf3a{c5171be zabwmzlMD*8MM%xVi`@&nnc{>>iEZmEEGGd6Hd^sf?MCTS@!(T&@3`)Ga8=AZju;Ds zO2FTpEuh?E#)x=Zyh<`CMT7Vxu?D-n!v}bM@opTrCiukLuRDM>b8Ns!jVs5(mm|oy zc@iOgpYpaYQ9WJlLJN@&1v5M8vp2Pn+;Q`3WJnO`>MM7>?sDn+Nf+k2{;bpW{%mGX zUEjQqbRBkyyI+OZH`qHIXTZty){x%TNoPYvoz1Uy4nNKFdm@^`c$Yau5g#QZD&vtq z$O6`_Pcb;{lsV<;PtzWjSdL$O=1rj=0#$p(gX_JEe#f(6#-P6)MsQ=22sNC}A*zHO zi1LissExU30A6XlAL$Hc$;kSLPX~XoQL7}n66aDO9jV8rl#Y0?1KQBUvKnI9XoVG- zz@=JOd0X&Z1B6n8xRj6yc=HHJz_)TFApi<=m-K^fYTO)>hfY9+u4O*BX#0Gac-BAj zg8vM_J6wY~NDpRzpmOW;a2YZ<&>>87pB8<}EVwp@B!cn7Kxoajr!x7S#olnR9^u7+g=Qk}7GE~M(w9L=LR8Vt>YN9x!B zJkC=MkHd{Q;t7!wl9YN<>Lq0wDR>EyB+eWYW-3=Xp)K6ql;J}b<$Tj~RMVQPP?4Lf z2vZTB0+*pZE=%@tN%C+mOZIU|@^CIo_SMl?NE(|gZ5cEBH=sg zN3kCCN{k9F#4Th&KOC}0VlS@W+oCl1f>wA%aCMV6HMhJZd6bcAXR<$){J22;aErUc zvi>{HgNCyH4*A)M)Voln=V!#_>ODUr&Qb67j5tMS?zTqN?RG}=$B)gQ|EweL9F^nG z0}7Gz_gx`pm;8AtH0tnYvEA*IKMz!^cL9GM%*YJv$Dc1?QOExAWAK#Y&ja_!pFN8G z+5GuId-|l_+hp@+sCTeWZr>Gx3hb6qYcLBK_23+%9_VJ&kFy46{axq$XONx-pLJH< zVTJRf2)NV->JpKFSU#HPm6tRfD3`+cKcLCQEZTcX)WNH#$#yoiqGq{0AL0l$xogtZ zLPQu~gbZ^+c|xtovC<7PBV4hVkionu6Rag9!sYVBs5N@x#2!9O{~J(P0ExqmhzH@& z#6JcQhK$Qs#xMXkDd$?!{on2Ob z&TD#(o2!gR9U8X#hggw7WQ~p5pm$pX`gcxNp@eym>fG|!BX;bmy|sMe${hnf*IN8C zt8mu;Hv8Vfo~drRiaBED2Gy-+3c9r_*R6l_>z0~N!q{=?;(TcMRPBB?)~7uUKjDy<^uf%G4&D$!1WGquwW7LLyDr;P2ousm|!8cO#DtwKlkWOcr zELK;-t-hRdsL?e}bkQLC$_h@BC{3nWih{8BF-dg-(x}gk+$uSwk1Ey>x4_%dLWT*M z{kguOE9>7%JAFy)%3rEI%#$pZS^rbx!0)Z!i zL%lTJ$QkJ&eTVZY`HlkBNYC>%!ec$5wwgYoWi0P3E$JgD-5M#4@6!G!(604{Y#`J0 zMb!a(r2^mb&axW1=-xDPup=bI6h6!4s;7ub@DJ)QuB=d)$GOum-?wF>Gva2fB9_|W zldgMI)ke^29H){5mqsaTO>ph(W1UR8ePa|yR=JaXrMehwjb0)N;GsY`OI;=*QnE0; zZ5wd*o+H~C6NaAk&z~wdgAxeoHl|t)fFL@D+4EGc(^Rh7D=TJ=u-)F1r)b2U(I1^3 zrVNmGg9#+t~waF{pPzM;Ex2|VDtE!M<(rq_Jfr_I2y3-G#d3*{{2i)}h zam)lSZhF4!TyG^*7W>v2<}81L*KSBtfkm3&DLQ6hVfDoAQ)?Ft@>t2irvt zwn)ZvVy3Jasi4%>xsKWTlv4i8P-88Yj^I<`n17~vhsmWrQ!&{ls`u1qD(v0&b>17+BK}6Hv2cgd z?|j3^gwgMGh<@i^XGjJ#As2ifv}77k@w|nKXWaI_a_baHB4C(#0C*%aZ#nx6IoO1D zawYY~&0(?JMo~`TFzT}0Z|j9hBpf$!^zg^|LA>lk{;*uw3q z$If89rqGcejCJZgKNwG_cY84Ysx#*XW8wD%Rhr+pSp8=NW^e}Jl$<`uYY)JUrz`xf z2mfU|!+(?h-fg`%rU?JfNs`$UQ}|D4{Hr|#aZVUE(qn|VH%5IT8$lve64pTZIhR@Q z3NGa`i`2VYjMC4I(tpO%3w(0aH55xzM9>;@&{@D&H9j4Y!N%uZLL{*n?Q{Fl_Vt>0 zaGjKo*#^})Vn(BBvH z_x<(vZfj9Z5&oMbY3Kaq9kFTdioXnW_zU~2cQ^fq{6(9U^eO4y6M&#l0I}&0c}_$H zEq_#_N)#shxaRU%ufrcE7{M=uD|{PkrHzE0GFh+w-fdlTVi6K9lO#dHspZPFJxn-k z$cmJS#(S8GdV|a}3KoynuuTLOtvK-Oc+yIJ5Xh+yRNR|U`siM8Nx=dB`xEBHV&{RS7_}^3#sj8-e?c^ z=9HUjvi=QhTbjAguq0WzSR0`++16+MJ-QhM_`Ob^CgL{Yk1V#-Av!cb_&WjtYY$E1 za4d*xU{NY8f3a+hB=OHw%KXmy8wGHQTQn0hHfzs}3S2Y;a>&j-gO-Kph`4*u-r#te zFbPZY?J&-7tcCgBl%PoCj*17DOSKUeX2vB(6gN8qm}&%IHaG;C0TVe zt)HY~n}}QsR40(@12P~%-?&#s*`yxnRhyWC-cD>x|4a9`D>~+ zcRhdii`nz{hqO>Qe|=Q3v-w-$g5`s_ou^>_3ZAWn#|i{HnZNGx_flTbbsvo>vi_R@ zzQ_5ybN=j%?^D(G_!1F+=PVk+t!6Z<`K{r%#vB(4r$ZsMm3p9K=@1WHahSB4TwiH? zuV>xKkMGy%h%;m>C+=ZTYw zsbXj2TjPRdsFSC7e1G@tU5u~0{L>h+)c6ij0}X$EMgZUA`0kQlIs8+R*r5u9%)kQ> zT>A6Rzezg|wxWQc`DaR!6`{rS(@vNXle!nD3^)JmeYEgh!50#DZTqaMy4nf;-k+a7 ziqR|w#uM_A=cirnZ~rGRnxAeHXdTdhs?+NrR~D1FIk_fc)o~7&;IZZalGPJk%PC+4e!g6ZbP>ebzK2;da3F? z|67I7Nj&^WC9z$JUwE>_P{0}>D#{FJ`{CD!VAijzL@2trvP!8OmkZ_t3<<VN({{WPt7S)@f(r&_JG@V$T5 ze~G)t*ftpAC5j%yot$*7B?UwW0t60x6(T*xy;d_=GQdgcL7n|h75Dy%`X7F1dV|DkPLCG`x8oo2+A;o zy}%=~^jrGAMvx>xalS#|sj%TWM&Owy(R56-9Rkwhe-e-;akk?WiCl?u@^Wj2>_m~g z$cU$%oPDVqB`Jhe5E?5~F&DtM?fD#hH5$HsfG_UAhjVU;HAJ@k$dF$RzC(5b-x{?o zw&AN0`0hGtxA6I-+m7i0Lje$d+%(HD!xi8l#ILXUs>S~pK%fFwxd&L6sN56Fy$=mM6xr_lbXl!I zkojbfRw=nNd#sRMbu2SUs-KT9h|kD!Uy7oItCW0)O24-OBsf!Ara~ZmN?1am;m;N~ z-;ARVB8+B^{#G`RIg0U?8zLkMG5Q5qq%b?Wg%E>8J#F9+vKLf$7j zQ&zd&BZ@cq1@hl`g7P#Sfy#$i$Pk5OgV`XAM1_X= zk)E?5ZQv3rV+MFzE)n!YX`&Zq+oZ`Yb=Bg35%A*p#mOVWTgP)+9U9t8{9_p`>sOx> z7d1P6yXanIUm7@Tc{sITyBb8+f{Rh`GKeAXwZ8^oHpM#R_u-wAq{#0{>eA%5prXXv zvQ}jt55*4+ac3M9-}Y~UuZdOLD5T1JR2`b&4f9e69xD17`Mf%QeF44C&o_8XuEEXF zd*9i5ez{sp#{0DtDsG9G0d8mpSR?WctVQC{J(33UC}J4kHnIs5{jJ8BGhG~h?dVUp zhDUm0qUIDes<%Zupfax-J7LN+8rl>Z#}zHjIol=WW|fhxM{d|(30 z;Vazj%2=@R>&`Y`Rm!JK$dmk(TWCS?bdOYs?TWo6_3X#!h98z~c@*7<`8@RvdqiH* znG#Q)dL3I@w_$Fv`q&(5C1`q%&t?WWjlPjSy(lf$SsrAnqZ)!qa4V#ptARR0N!ie~pM%=9r#U^$3NtQ(5! z{t!5+XI9TTxoZshPaSI=Hym zIr(P)aIzZ_9xGnlY|P1>Z+2^Rq^HS;f)~d0WJjOR(l`5vK-qBk~&S zHNfp#V9hD6r)_}GcoHPlEnYnSNIX;sVqx{##Kv>Ud87b>w{EJn)lb~d=+<-nI85|6c) zI*n2=a)X@Qc@z>T*1B5NJA`ptDZQYjONv`seMgQ)M)qszc6A!1przaF+z$BW{0gbp zpKIw%T1w52lr-U`xyI7x!3%0QBj4U~R8;PseY?25qn+IDo<*!X6%M3leO${af|(M0 z?)$n_DChHGf*fdC5elDmc7cE<`~Wk`#gzy_!US!>fn!Q=z>lqjs@KKRJ=Q(93x05J zLnq2L@3+ZcK#>JsER^d)6&P(~Y?NVxdA^w@Ii zL)F28>L=!^_dkaEh>X;GSm$_HOh6scDad_@lUrwtv@NI#n}=GzRP_!spSAv>PK2zp zT8dkG1Fvk|5&v2JT3V{eqoAb+oZRk~&MLD;<(j(OZc3kzr!X=fvXcmjvz@{je`&v_ zCKfkkIJpa&D!1BI|D{)h?56H3#>fz-aK?W(iX=DwHYw65=+&#Z^C=wUn~Ird`K&3r zDI7Sg@7isREpF>}r*y_2@7GpSaa)%=xeMB=uzD2+hDlrN{wXov7vQ9M?H3+_Y@M~H z7(Z1`MH&A)sGQtg{6Ya+(8bSgE9#;(ySUP4ootY`qG_czFQDlys@*-&^ufh-&&k#8 zkMsjLUJP)|$z6b_fYqT;s_^tjB4Y@0?kmRf5T|s;{|^*R&g7BYo4{^?_~fx(MMx|j z|0-)rzOC=tZH+B%>vpGf#@|F+?ty43ZtHRqyrA-t&2T!5YDZY{zN zj38vi^KBh#w^du*)-RpX8UGC0audRg;zT$vRxO*VvwOeCcei+}{XeT+8XHt^C$ap+lm!W|e>U}q+{ko$Wj8GD+Z#_I{8urf zE;cd~1J)O;CyVhKM+`;}jF7&I5;v(5fZ^zo93a$sDqHW693!wtniA&iDuSYwPUdwtnQ~&bJlDQwc$O8YdpCwF(ic zdw=^tckhv6+#KjslJWnct`L(SC_oFkCqds7GTcorD7VhhZ6QP6)u)@9T-?;XPGRCs z^=oQjaZ?MN-1(-g&m1Jk=5@W=R-R^OgUHm(mFeOT+L<{*yPRI`hR~q0ggLuj-6Hw- z0mLe|^$}~VHm5~&s`f_q9bA0{2M$@=i@l4iOLyOz%wWj+Tg6rX)UFz`zIlA%V4SUn zuwXELNo8-qG1%809=-Z8KT%~sxeZVJx#~C;t#o5#Q^I!s?223=T1TafcoiJ zDIxmY0)!{+x}i6!pU;)7qRF*>OMf8Rav|DQrEkHp7HyB>%&_UE>29!{Sa5XxCAQ zPYoECIf-Al08;>8A`jn}cu~0V`88-Gr*%I<wvPXW+`P{_(pb8QlH!q*77?Vf> zxO&eV{fctF!>3hl1Vzp`(b3Om_CLtj_`V#bC~fDKgfgPl&gv;Y-|#MIkJnDCm^H}A zoS}U2R~x~X8ac)C7Il1HCZ`}$`rOe!dLB; zrV0eNK=TzNFcvH$t@(-?v|mbvn~}1~B?9&|>2OBsoB9x)ToQ(Zk=!7C&qTLLf1Z?t zdfsSQIWt9{jI`km1r8-)=ZLW0WX(~1YiVy9BUdJXsBK(xaXYSs`x~X)64$Yb-{|Yy zdhs6KdAYm|%A9;t%!Q;9Bo<5iWr<^7Tl%7f_TfZh4NnP#BCPPr5>5!OB za;7H1xT(plABDn0W*VihKukTlP6%4eU zyg`^Fa~xwML34RV8s=(P=6k*JuTt{2sIur3Mm;^j#TaIOAYC)eGpLj)eOvP??eS5wkVZa^n?I>YjC$95WORe|x4lH%h(h}k`6F7-FQ~Y(B7J4m=vNY>-fKEt z?lZxOEP<&ex`_)0)8|=G==NCYdgkm)Xw{-yWZrfg>BJ^c{HLdG=W&B!)|Xq)Ef!9} zopH1e8Irh8NNNoZ{Mv3)cLGG^T+w4HBzr`hQF~+Q0S1n3>A^KR(!aN5Z+J-Dysom1vRvU_ zo|;=#lHAL{)6#y|KNeJ*fd~ zy-cxI)nO)51d|x7&qkO)=JeM@{a>~>4%^<;g^H`6%LUA%o=TLin2;$O5DP|2yo*p} z3L-dCT~CDn7>f)qTppK= zZQEEFFVXH_437u~9^7`zNzTJLaCrDCgR9kBPo^Xx))m1OdO2*8gT$ex?6$Y%dGZq|M;a#)}+q>=_Is(-_ zJu8LxVylJAwR>Bx;!`4XayF4U3!-=?Uiwnv@R!9uBgDuP7o{gTHEh;1>K&2bYC7^% z*xV=sxoC!1_GKzg@8213i-ZsE8=;~0UK^ITVa|l}wjRrSLbsp1`F?z5MTwPK;Zyzq z$c}Qmhq;%h|sINq!t_{hspwGq^DSNY-&jAiYm;xtHRz$x7T+9YAO~=E@|@~nPg3y z-(%E{FXyxm-o8xLAM{bPtaWoksjI6>qTZ!Q81s9g-pCejRdPeis;1ejg{+lRHX-cG zdbr6N#dP%g4)FR`1Hj8l5Z;&{UImKG3{SWs%h7j}o!eMpI8B7H4wv_pv1->FCQ%X1 z0g^hpG>9T7Q6V%Czq%RDu=m|NEMNXw9%=<)0iP5jf?nLCl z8n_6B^}#!PlQP!p8;~c1IBV=;40^lSuNXVuuKQ3C!CGU1l-7PIZPA-Q9wtA1$5=lAucY%%X(Btm{fSdD4EtdA1@>RQcO;L zd#~99eh5%BSMGp59cId1CLC>)XgQ#&TKHSc-7#D+=2bZB7pGoJ)P8KWQ3{d!${{=-66#O}xepnqyPCtwUO3Bx#YwPzG&o%={RU63&HIJ=)ry96Y+c zcd1mIStzZJ-fX^FbFjgg6PXe<;*KL{vg#`9uxk$?pW9E`k?RxjeY2hUif z08J0-i^MZ`$+PTts{kY5#DF$$mX5N!#s%o+iv5s9y8&fCDL*)7mQ@<*L6P)BQa?U5 zIS`HmAGEiZ9M9=IL_k<6=$NL&I@!_8lmXSodTbnF+i^@$_tQ_)S#QhVsZ#0k^L7rp z$+*dx7;sb{HZASK%4rb4EVfLdDVo#w{A?cJMosxx54m5>w9tH)lLuX1k(ZoHF>WV3%5QuHU9?q=! zdcFho`9^2ckkbM7{UwXNODBzxYqZPoV8w@FeZss}j}gwmiSq1CH8M)urGkSMJ>2QS zK&@0@cJ(RhY!Jv9695v(m9NetrC-a#{|L`}gg?{|{&!pmm?stN34XOd7TO7f zw}#3FB*fQ^N%T@!Ih-XizFpJIiVJIsd+RntO(PH^m}3l|K_r=qu*sH+d%o9 zeoM|XWfCZiS2gW#*cL{V(a+|MjIf>`3wa?3d0XegD|0~nOtni%KP#_tD;#Q=8^j~f zwok$)zNw04Ce`$xQ!t7KMV@CF*&{AL2yxK{0bf@H6a&9xjs zKyPV^dKFupkYUHkL?-H2gKr~!s&aJ+PaFP@%UK59a_*%2?@5Ko`xL!yb7_Qca? zR~WSy%@uY%VJ=H9q8I!y-n|_&Zbcs_;n3EdBFZ^5bg|k42u#<>E-scH6wbDXSule? z6POy?@;u@x4Y}gCr&g#jQwIiRddti%bco_SPaGN8NV64A8|9oi$mupv5;vnHZWf`i zE9yhI&vF{GA|XrBCQ%**W4_sYFO&4%$50*_W~&7(CVlL%22^^`rPYJs?H#=Xo8R!D zRBPa^*Dwcpmts5}Uo$$}vI?Yc7b2b}5;Hj9T+Gi8%=3s}@+>!()&5pv{>MyJo08(@ z@jYU4Mtn|3E&jHkJQXD=^&vW;H4oan46FO1AykMiH>lcfY>S(Jm+ZZ-fe%g#)w6Y9CB@AXVwr0NywR`J9IBV# zsZ0%LomhTAzB^q!FAkg63Ys9UZ!m;u^^(tC#)vr_{uasJH;w8AdNl%!G$I`q{1~JkGo2q-7?AkT(EdvK?u3NyK1bh)@-O zzDRmn#ra*K5hb2B{fO&7{oGcH)~l{2`X@PGn-+V z&u9OYWdEO0|GoR?YYs`xLjApEuyf5d*dIqjd*&F51&GKQNMP}ZnspAxxbbb_7~Yl} zs4`}H|D#Fcqt~+eVNv|HGjAXyQ99JS=S}^gH5jwJH?_BgnLX!((kJC-&k#Ak4jWq+ zG4IsVW(CtGB-7@l_hk9Cr;R7vVKHqs6;B%@-B=#p`s%^$mCbEqnK(b*$;1IeGI7SL zi4&99C_}7!)~kv08>T}T+dxNendbJV5HpOl(e3HhcA*q?CPyR_JxS)GQEBE+E~pf? zy_n}aV&)fE3TtncS?0a=1U19dEHWH!e?Q!P8Gl^ltBQ5=+{4(B4!3W^4|vba-ZgaM zCiv2uGIUMfg)#KlHY?6wZ$)>)#U-fK8&npNh{OPN5j83?s(^|q`OESNvY&zF(*YEhM+FE?RXb=d!{Jnvz? zyf8eU{UXWU_miEi770_BNN%y1TyAgQcD-24cU3%GDwgSTD;!;V6E@FgGdur|lG$?* z2&19M*mZdQKuU6MD4TAnt(5Z zvVTDWsY~wgG#%D=kEuu|Ym-iDJM8voX&mhKKEg2#y;gbkT2&tITy!_M1-bv(yt{=? z{%_`;VLsji0BpeA(~sL>q=To!8otu}yP`a_^|js2Ke_3ycmf{zv`j$RL{IBGSYr;1 zsrM)F%|#RWgp%EI64o8wJk=aOjKLMl;%${1z%Z6nk@wshTYDpJ|H%1o+h@L>7#&Lb zlMBB&?p;BQ4mgmUB*IVLC0>ty#SyRBz0AxL``zep9b&(iXOz>IRy{%UEL95*ZEhSE z5x`(FYpqYkGb)R-6{rqw+aj6frJtTni)j0rt-!QKi&f?T$p2FUswsBhWh}y>TWWWecu9$tW zD26g8QLnD5j;icAVbne(rDwz4Y+Xj=6ZRGc2rK)<2pT?u5pvf_P~~cTqiY#!C~`e>55@`t)>L?u}gKH6p}{9GO8b0 z|HG6NnI!zYU}7S&@+vtJhJFrW;7lp13w%}k_99Q#KLR8;+B{xC#E`>>>B{RZffJ|K`Ga(7$Pl)P$1%xjman&Scl zFk(CNq*WT8k49Qwsb{WDw+iwi=~mGyTEBk|?3ho;;29acX%@Bi#g_>(?LDS~^Js<& z{h6#-`|w2-$acrlgTM$bvEj4OqUYzZ?3bt56IiJnb75IvE%#P7OLJnd-ogC!E=BVZ zt`X&83-L|0a?v$^E$lYV{QZDu7K8QHkWPV{u*0*7pm-``#yV^I#Zu{b_%wdBqWG#} zS60oiUWSz~NgJnf263&m@Dl0j(AgNT*}v`Irkrc5I7Tyo6ExLnu>h=BGOV}d0we?L zcPoop`f^6TrNcVw2m_Sy?~S5P0DES6QTFxyvLnFEkyfnnRBgT>bbMWhl;ZA+;>K?1 zS6UkT3;H$5eqM27Kkk>EIiNg3t(R6v6RNc*?AAI}QUMYV?^j$}n{2nX?Wv-!{p+Ga zOem->E3WowSHAq%Fdq4F>6MmuH!iW%)fFYPFO`1pQ{2Kut`?jzIR*ufWdD9K;K6>` z1x>L1_151MN)?)JZWj^Y>OgjuYV?+3WPf^LVfO`F9VqCeqE!mX1<38}mt7hSNZ#+| z8$FChCF!W*))w_EF0DOjw{~W6YYqLf)0$$xQv1^iBdV8weNs0Vqv3Gs>S`q7m(}!K zZ~d_tZ>9Yz6&QYC!|O)o@{kbboQFp;V?G2ZY5=G__AcZ=J)xnE&v>n+rS zQhr8pThsex&r$yC3OA~=D`-shzZhLd_A4%}-GGuyvVWm)RM4}}F6h@Y$(Jmy_8C{c zJmmvhp7KAEmUmD2PYSlUBzNG)pf*Fga?FS=X|}9diZ`Jkg)iJ ziQU25`XsMN62-E`BT$=Y)^0_!c859>Urs`1!nCaaLjX~C@@)|L;wMHL5oa>;c0+l3 zCYt3?MI;>x;3g8OspEBQHGefRZv>HOY!$AxyoT&g^(1%m-e9CB82nB%(!UT0rDru_ z{s|#8V5XD+=FS&%QiUH4bK%a)h?If7DDN9rsFa^BOFx1YJ_ndHk<7F?(?OB4Y2GM% z5Rp8P0P1Q+zoIn6J%ZF}`C*yq$q!2t&x<4GAQklXdV$KCw>M}sEs;$LIiS)+nI192 zftcsOsHe^sJMw_Yk%Jrw*;G>p_i=B_Ho*j*Y+7nZ-DRnrrnmKZKIj8_t*aHba+?J2 zwSOZ)8XFSM*6kky6c49}Y)xTiS~eE^z`YN!;-gj;?gRcbK;+@)9O3<^kY>XC*(CepSbP&p7J6%kKGlfRPSgTBg;t^hG(*7#;!Y#N&Dx;Mc zU>ImbObnjU{8-oLfoaJqRk^ZewDG*y{K&S$%>EV=nHi}NqILJe61%N~kems%Ub;QI zk{GYLtd5Nmr}GzMMDmEIqwjGKM?9U_ zmKd@jo}QA8;xUt+>P6YZAhkE0V`S;(4H)AH(y2 zkDs{b(F%FXNK0lwQi2|?VdxMy$23v`&hr_wTGDIxDoCeIvqt3~UXbo3-B9@t%BP33 z(UP!dep%F$ERT7T1Op`G0d2;(3$>_0603_^Be5nP?AH{DM!eS5PIiFjin55OvE1-9 za`Da;su3K5iW?!ZxDgVI8zHf<5i@!hTD9n1IO+tiQ-tKb<{v8ViKe{+l=1UwJgl6V zvERSd%t#z4fu$yG656pTeBp{aAf=Z+NTJ4^tV!I_w7wk8h%0&mLz_R`W|su#cFby;lA1oOf?Md4Z8RVFTHgyqMh zi=H4gC?y8olH9{g0t4ssE}bv}$6s6`S}JRR9)FakveBDyJ2Wq<=Hnp*%`gv3UsoC6 zV@w7+b7Q3}Z-SN(di=0?s!!QzSV^2qFyLN9wSFX;jI{oj}A4Rbwe=sl1!A z*5PZE)}G)*kLk*SpErJU&cJx^i^hOl3>^I2yJ&!@-*vljn4kdHdnIcfCr$&|_Fzk8 zH7VXjJNVxqpL~y%vU)N#s??X)CB>XyE%bA=0E452;_keUK;%d;-6NSgkM*6G*^!bF zHB##XsyYcNY=+TvpF`>Po~LDkLvC z0sujeF={XLhC_}Z~MB4(F$0_b5K;gKvc`aF<6t6Fm1 zOrOUngK4oMze!No>}?JA?n^u5~+v$>+*E6*>EmR%(#Fj^YdJ)L&QD?oitbUIQ>14Qd%c`jGvemm zvXK)<EqkpmSq&-2#K4Y#qn#KXqKIbGZq#lZ_6T;g%(66ozx(oa9Ldy$CaHMySOE! zRJ^pmncKfk;P8fA7;g^kh}RCEH7L%cliqFMX3)krRTY^L9dJtyL%h@kioO=F-7i_m z!OH+4t$ku{7q#m^oTO}T;&C{|o+z%wu@tBLQXg=d<231ZW>VGYXRLpKmAJ`hf0Iy9+bWv@+VRH%V}czS4AtrHsK#OyE z#Fbg2&jzAwR}6mlXN_{~zXkY)8T3Q;5|AB%oMlMOFPtM6hWRS&?4YRGsdVq_EJK1I zWtU^zOnp9}@vS&>4}-644H1y1ORwDefk||AC?P-RRhLNghy%jQm?R);FDvwd)Prtf zb7N0j)<_(jWgcr0JGFw@pwCw+|GY>W%Ho%>I+&lO!`wvD!M=CWKdA}7U+#UCCtTCx znSmLWOA@i53}(LuukbBHf`RUGKDI~?l5x7q=>L=xHr9#cHtfTeh5AegFNMIwtrP`uoVouKqsG z%YW0~zf;cX@54M7_qWvP@0%YK_V-tx7WDTOjXAY3yo{Q@Vt;e?OB~yTOuIr7$xL*zTB1{M7?Hv^2-a8VUbARzsr#U13$yXTh2Po=b;31x6 z8E8NrVTEQQjaN%G!sK|s!J$F%U><7B@n!r}n}hftr&AZ1M443?{|5tYGWl~~A=-ZO zuL89FK@WgJTMf-Gj%PG~;v{61V_y?m!8V^Wvbokb@Mn8miutzycN%Y7Ig2<%u79TaB9aMID?Zc_xMf;ymtzTL%Y{RYZCRA%8)KBaJPldaGv&nUL$`rEf5?(yx)PNb4d< z3z^LoF=Hn6DC(O|sodJPm!TQ2`4{er{?%&I=7wg79%(f+w{KQMbDhGc>^ooB$O2mRpprHr4n)f) z458kNo-9vz(F#3&?-RE2A_pTAUnM5|i+9T6RteP#C3SOMnVXWD)lAM&QimQBg~Ta< z0}^38LPpoG*JPydvT%jqOUTGgL|>VH{`|h1e*VJSe?vcaQqDo$pLpH{{d~#BL($LL zcM9pJ6@qc`w@y70s_SP^-Ja>^EBE)&UUz>V=k34g@82os^!H(&chTPyoc_N0c42>S zQk)?-)GuiCss3I>b^pWuu3-gEu#fQ8fo_Sk!Zj!n**lJ!4I zf-GRNsQiZu*!>iBkn$kUJ6pg$_>6QGusf*UI&iYQw5S)gfOW`M7BJZ`%5O8jvWBsg z>oshqUc*?*WDR2}lQoQ`OxCb^wT9uiSjkJsjEd-0!>d~Ssxh4M-e1I$mF$f#XiWHz z@n#|aDN#%&H%MVU=4z0he^(9Cb$aE31r1%JS&%GYtYn+`DOkyvj|Jlu0Q>#M>vzfA zcri^%u%$U1eTR%!GijV(p<#Qxq$%z*B0au#QfF7_2FeI3+x_Nw(-1o|@HD4I?<$j2_R=Ofy=|&vY|-G(XeKXca#VdhF}< z*faDaNQ4q9_#xWO{`~BQ4vs`4t|n_}Q{iOY&cM5vajiz1qJy<>+jJl!CQDJ-DoqFY z)(RcSFc;8)BXUCzUrpZr8ZG4c)giq7H~h*^IcKO3;&~VR>RR^B3I*SLy$}VTGM!y0 zIN=>tokGFtw-gFo2J~IX+h47}qo^+5-*55u-}Ltg$~pZ#lILCYcd76j;m?WJ3i~^- zu)pIq`c!{ke)E6Y-yqfH`};lK{+s@OhjLDTYkA&9f8SLqRQfyh)x!Szp=sCno~qHO z`uqMH|I_|Xq`G{66TJO5{f$!2>2HkZUG(=*r@xo{qp-h6K)|m4p0ClT`uo|Yuhd^u zy=;;C#~Tl*I|n$oiPv(*zaG9wa0uBqT79p4nT?pGq^=@Wtv5X?Xa&KW9aSJWKZX+6o;Y*#q=gjC!3V3bX0#^2~P z`#ZjxuZC+6oju-teoD2MB$K&3LdY94-dWDw>z!63*(2s;!SW)z3cg!sN*l|=!9yEY z*s$Kp*Jw*NqvDy`6R!cv@)M`6iIdqOY*64mC9-uvTn_%p5>(atW-XAE~%m|3>6qM18Qs3dnWmu z#fV=qA{Cqxm7AFI+QZwlJZPd>f62Raz}5|wo;Hs;V57xv82#>8)LPg-Qda~@)Hx;} zNdY6XPwQ34G_7(5x7*0fs!%bO26=rkpLh5qt`iLhyu-z>!R(<&>J)a?|Ng5o|1#$E za-p_?znA{7a0lXHCVj6pss|ZeUScC!YAuZm3Ad(oV&9R`{?UGjXw*#*rDZ^ zk@z)?GjxKO3(^zoVGTK>Cz54RTUBc;lMsWSIpV`YGEL zd{hcGu1P&AZQ`SX3ygA$%-Lh2=9gp5me0ijM&`45?{cZbSoj6mLvsch9h*y$7|lNm zIAXXPsWhPU{(wi?$zop{lL?b&Sl<6$K4tygjGVT0d0TEJ#pu$aAuxc3l$`Sy5mlr` zBUOX(tF9Z%Y1qWt%*EP~G``>I%Y0&z!y49A7hsD7N!3-QCs&8sE;}BBVhwJPS^vye zsKD$D)y$C4A}fvbB2{I2$zrKFt=-AeOYYz;)VAbqop?V#=_SkfzYpWM`Nf3vW1q{| zOD8#LhaOFa_>lk^u;je~dH;4=ty8 zXwTasGtV~8dRy4Ulr&55vZo}KC64o81*e3?C^=Jio?3U&bX<#?*G!KC4uK>t(DjRoFQFFEM zoip)S{tD$zQRT$h@C=3$2TNh5434nYJwSGlhU|J_(OLhMFDf{%7q$ zWBQ{e*8VG6_1Q#?2vXiP#6r9}eo*r~(Xru}I8gHu!Uru>4AdO>x-gdXLSdbJ{;Pf~ z*Kcd|n@_*Z*KcwZpPaYrw}5_2>$fWXcBX!-)^B&}Hwo85xh48df}ikqxqdVB+pqOo zy?(RXYtU~O=x@{XTUftM*Ka}nHj_6QI3;&p|A}C$&~ePl=O6} zS;|Xbkt5X7GFZ6JVA3^i>J;CS_Y&1koh&KpyT%Xd82epgNSi7ozs6BrmGeEI%U3(^ z8c%gK&b!7`UC4RYxT-Up_mD1M@4Rb#)ipTp8e?_SoOg}0y6MjQG+lnC^RDq$H`jR= z%t5!!GV|;%37$w&+}}=WR!NE>s%TF_UnrJTs9D?Oh!UoqN)Y%rqOR%YQAMSJ)4t^O zA?z4FPCkVK=ZC!uHhPB>{DUJ^L4@S7Y3w z99+UC4m(<5B!U%-funFkSt2j8q{$bA1K=af2O${-NJIwFpGatiwm6cDxkiEs2axWq z*D28Z1ph_Ml;V}?d$qiXv)e6F87?+z>SoZ=iKD^>5;@Z@`==V>J%UD7YQ&*9SPfXA zN3+~<+B6LjsDA5H1DXb_)2My*4xlEnLGe1EWhoRAQ{m2_?sz{3bn}}I%x$Ms2INN4 zqiwKc!IW^r0;bDk8sIo0X4Cwg>ga2xIMzZ!@H$rJ=>4K_#mWpV%nU?T2o>%A-W6>} z08+9evM-0(@y6}JZFfUgbkjkP+!iFs;%^Io+iSv}CQNWqMY9;(qTBm8@)uPkJ5*&- zVNasngiJ|Kt+r2Dw_9CsPC4Lsv5andYE8}5`lc{GtThUmg3DZpuPN{b_xbR72XPBp zb;9fl%b}q4)@{(Y+1Y!T)MT$(NH2)=48$H{gBf`NQ&vxkwFF@L4>xiSXnY2__K;$Sd?yL((#8l1VRaYgOXb z$=um!-SH5Sfu2+}cdwGV<4E7aG|Y>93A59D9{F5eMYt2%y8kFermZcih;~!bKx(Xj z_&luUntsrB6KuGgsJwCS*bXp}+?yaJ$$@ws_LYYJ%GOf@PCF8}0$?33A*?~3@0LC^ z;pvp)GLm`+sggTj9mf@SA_$AF!TZ3cssm1k1o}^o)X>YmDld=O16lw&EHQ}(y~cEE zyV{8Se+E8@OBcgFac6+nYQS4gVi+`e0krTtVaIB1W5dua?*(Kxy1nlOcl<>OM6d6t zP7YHc(aW7E5aJ*#PQl*P4o~tZ!PT|H*#?YYlxZaQW zD;3Cj^8ERZg&zocih3iY+4pVQJ=+|3$d7V1m^;Wqh;^ZQcoKIW$#ydHH#xPyl9d_D zaWX`PD_ie;7 zzp2@~q~vkw?k>vTw7c@7U#Y!ab`!I!q;#HM@y|tycwLQw4^QIk@}SDs_^y#zBoeso z!^RTzBc^(e`~8D0p~ERR@w_;0J|v2g(VOMq4PnD`@zJjDAz3xU@x{LoyF$4Le%Sa< zzOwAo`O1DFm2p}hpPyQ_|M6aX3>$f;zKHpUk3^Xe&y4vCs3?h-ei0*a%DXxfzn^$=-iPQ~^OcN~;1}T&#mCce) z+Du1Np~l#@ab*W($CnwNQ+)=_&5Q-7ln@>Yr&of%MFFctm##gKlPb7XQV*UKTHj%$ zf?2~mo{c`K__~fajP!UTob`mW;j);AJGpd?2U4LFjp?Ean%XI-6jU?F6RH`Rma$B% z#z@sltsSaX{LRBgEFGp+R16_%C4QW)b^l7LMOR9-Wt9$`v5Zk|)D|3p(XjHNsOAtg z^Ox;R1pn#1<{qWB5Msl3lw}-IkUtoQv+!B?973Ev zgbxJb((yh*5N-QdoW4O!6}lJq#|rnyP?X77d$XD;Q}OmM3!5G?9}qFyr81lZ!$Ua= zL45CRTckcvQb{m(Y0aI7doT*}jEQBRilsX0O{JrV2RAg{r5jQ@w{sI2LecAa{l9Kc z8c^*q%Pa3-XvG*g=U$;to4GAqAxtkPUKB)+uShGS7W+JxdT2$)1< zyrN4DvzoCBc$U41l!}{A2%{16f3G?V0Bk$6aN0_Uf z6{%?;%GN%wIWEx0$(V6fX0uRy^Jkvqx#otgEib4|R#O9qk|d-N+dQoU@p!4Ojr79K5QrG{1<{-lu3d z7;Sw!d4joqYs)&_FBq4W$>Pjjb~6b1*R50B|2X=D2LVdmC!}cON`iB7%+=d+J?N2Y zMsfC1czJORL$_Q1B1(ezABcx~wsC`;k3B=-U&SMpVw23{aeKc{t+2S1pUCZY5})T$ zb%a&x!&tTLs4#{sODZZ1dC&`X!>QA%EHx+ZF;(xV<)V0IsX(7VDdv^);n}qb)M4SR z*ZvzeFraCC!u&qb7IdRqrO_YA@;rVQje-rkQO?cY0MRAVOQdY;=Hwsq45+>{&w$=u z0Rsw~E0C2MTyrSrq=>h5xxTXJZK1L&M2U)i3r0Pn(!5p826IBi=vCp=>TEQ)y6Kom za81)mMCsq+{p%SO$?~ugi2DX{&L(2M6m5OQTuvx@qVjczgHI)&iKczAw87+(06!8j zWke;sw1W2l?|chY0a^KSRHPF|gk=g04U1GZJYxPZ(1&OF*s6%RSSA29>%RqR+Qe1lHL#yOm# zXWzC(2C{S~8S@H9?ay(FD}>hF!ZT(2EB@yRiLBb|^&RXT-f+Ly_e1Po_3fyG#>#cl zp$)k!0X7(^Rh(tQU?!1;VI_QdMO_xqLu#@bg-n%s-AZj{;f!$0T4iSW9snvUi>0hA zt;)(maa09K6#!zS^5~V->k+1y%iI!kn_J{l)_*#rj-r9geqkOUfH~T2{q&E5t?hjX z9-qh3+zey>)2KbWV)i*|KB_1NTn5J`AiPwFV>{puQ6Z@2R1kkE7&9pXw>VO=d`Vp* zKkn!3WZ}>+YETg82@mxJw!IirCrljZIYy>c){mO0h%mBz+*a8Pp!s=P(~+O2V|ZR{ zX2g2OeGlnYd5D>1mFc#2DU->FVwzL=*5_4Z%-3-)77u!?Gy>bns&@JMA|bq?XN2-w zd>x>F_wk9*`Slw1swXkObL(DxzQt>3#cA#w#)?a5Gb0@T6 zAK7&?_X0k9Bq)}RS%Y&<$Q`rztHc-BPB3+LVr)OTA*Ms-z@*Iat441{E7D*eAp_~6 zXf$8jDayS7?Vx-=)d9Q$W3}TO8jt3Nw~FSgeI;=$!wD3NLi3_86K%IC@f6wECw#FZ zCpGjrVfP$4^+?x1KF)q{ao^YCoXrf?T<*)&7Ry()mGHx6ogwqfiw+Kr$zXE|N$ay4 zM3I@p4+pcR!_)1M?Hn@Z&RZoUXucvw5Yd8)4!T|)Hgr1-vcN)W?M*`xW_R4|M9G)k zAYEkFbhbdn9)huv4vPAItf;;~r;#~+y&=eib~1|uku{<`gY`IP6xKejFsOvJjXWEk zOB?beJ<^5W2lxs}7ussnDKY7jeanT5f3oeY(0w#t=7^4W=?^)4yfKBiwtw@L7_WzA z#$v?b9d{@}K*Qdp?cPVWaBbVwa>R5T zJip}2afcG0f{jvPi6^C+;$-gRtCHx4v2xL=Dt{w*9#90r*`JHeL-9LKajL-ahk7_x z)URSB%&ItE^2TV66cY#bDpc5V#(3jzgkQE({%Db4>6EOrS5_MsunnUWCHFAz(us$f z9Zm-Ze%aABkzN2dmpZvRn@6XLpYDk8g=E7HRfYQ?7~IE#W!h^RE(l}|Ua!{RtChr1 zhs3Z^`0sC(L`d>MeK_dE)s5 zG(S1Be|OAW9UH~XAey;-jV6Y}&|Vccw!ZPZ!sfmc z8`Z8E+T}2`&RGX>*0Vi6svCy37Gp5%NMbP_Av9{fAYthC4k(W0%-UI)6df=J3z!&} zM?i33V~9{4^3Y{U9;#yjyj$cUk*bD>P{sM^y6QfE^VNrzG#$n>8Je|-P**LJ4s9!k zp(k;h#xKSA6*mbfUsoQM6>go3)^vS(&z{>7W~lcNPA$7eZzujZ>fS#T3ek2w|8G;f zAQjWEmurtAZOM9~b4JbgBI)@Jk=lnvw)bAU6=xN-K8MYJIfEF>Wb8qVnXiYN=Yt_f zL7tw98sEXt8)!To486niVginQDg%iI;eM{{;cmm0GpL_ShrF#1fuPQ4J=>x6)~2M) z!-BX`-y0$0phE>Z!QyIPkhJyI@TQ{d!}?`c`O?MJzWcR&wO4|h4L|`HJ>)02oP;UN z;Otm(2)PLN#Za+jQvnF08&v$kXir*&AGE$zT=!31bvq#c_IVBRS+gYAz;hBvaNOlZ z(AW1Xsi0p|+{-FgzB7S-lSe^s6+%B&aM2I?O&WR&=o999n2PN2xRyIvDBW{DoOiv| zP~7C-50>vv$i}XykSz#9SM5gFdI79olUsC?+ZDpbF_B+H9k$78noajr;IE57FE}3> z5;W`BWT*9`f)ib>6Zq?DMAK~3w_~Y~JR%l41T#P+ilHj23i$OndZ6>iawD;1OundM z&^md6d!4oEpnRw1bC{U${;So%52hcn?235sC)}RI2|}n6E0BVgIoKcCDR~f7wP{zi zIcTdDv`2wo2LuJ~RFzkOJFF1gd`4C2@vcgzx?w$&4%4-`c}XZ@o+no-BwfPdO6Hd{ z7frE2mNRJvb2o6Km3K*`RH8kv*zj6O!Nm>Ei7 zx%`YSAkg*JnZ;EvJFp1ewHW^d-V7L1mQb(Ncu5iHR~Htf#ETQn>lD(g)IA&+U785t zkQEL7T@S|XkeXZy{44OcJ8~=N_`&%`pf(-1?^ru1-jUFolNowjV{JY_;GQV)`hJW0 zLhhG#;`KTC7_~LlN|jFy(R`T_XsgwQ-J5AFrngnDMS#84R2R{0dLU19Z}ZGbey40r zR7?k9R)V>Ig1LS)UauN0WcF9Vnk9SXrA67l-Y>h#cUy6_JN!~bG4DcAHHRQIN%m;7 z6>Unyw4s7c72N znN#`|WUrZ#f|_6Muui!f15vgs5&U;!i@~KO3fjmt=7UyMwsEZU{`m20Gt-MUo zkXkX*r4L*F|CoCd_$rI*k2@i`5`lPway2e!tf7X+)>O0&4b(((^@%1mE>v)9T#MBz zLV{GGf+3dc)7xUz7F(^fwUxG7>A%`W)S3X204_mWjpBk!^})~@(LzM!zTe-O=iV&R z+J8U)&-=cC`^*TAU)Lhd zH1nZi6bslD`9Wp46xU)`fU)&BF}1bU|LiQ*U+R+c3wc_Q1)X?#Q@EQqz7!c)#&0>l z73_joYU{fK%fK+3Vave!*n)6n^H2wYvxf&b4vFuTiXNiqIe} zuH3aoY=sH#5EHFAV#+0yHclg1kICUAb(zVjT~b1uUPEFmwFfVg>@a+wVWa42*pS41 zEj5)!&ZoFJ$)mN4ooIW9MPTp^Md^nQOs*Vsn2%e!_7W+Aev1aX%HOWdHTCg{_iKS5 zdi@=HhWem``NHmqxN!j5G!c)pa_6F401NX?AF|i;XE@BWnU(+mvG3eGy2Gf7*5NK8+b}=H) zX{+&Xd(@`cEZQ(ji6v(x`?s%y!@PT}aE1_Y{1B8X&l^~kbWsY25KuS-It07XFb`qi zR;wh&Kvc)-;u)qpvY?0@4Ps!9IGxO;tJE#($x+wY_1iO$&1@zUXC|sRC2_(TkRkzbb0Njf%qOY zAfx@9n&ell)VO_%R3P9Hw`J)5E~J; zayF~Z4-xbCn9Frh{^#Owtny^=qYDR9cGr-IVT>Y6WQf3BW3c@-cCOXf_ySTMByTq< zw;!vpbKU4+fi;DauZN6DLo%}85PDQy&dOwD%SD5!-;zH#+adMm8|ER@KWwn+>JndU zwI#k}Xk~du1PmcQ&V}_{RUcmh?Nj{wKGOPH<@UANjb}UlZiBxcYsl+{(59T+ilXMC z95N&)hYW$v3M&c83~M~o#qoES%p0Qyvp~t5m`SE}NO89gfmSwh)A&nwhyVH^F@%37 zt749%X+!W&8v-r-d1?F`EYCv<{L@2X2>)*~`1>vxT;P2p2j^D!3)1*6-5vh5Lt+U3 znHl^`hu~j21e%$fu#vj>BpVIWySstyR|nSb^ZKJ%OEodwQk4xl4`=JLZ z)+4atIm4BPAzY#TJ~vclo(N7dM8~V|O}F3Y+AV$Q_WRp_@)$3=dXa&evbu4l@nfRb z++*!`KfVvtG@~7ZZ+?IKJq|9Zz2vdB)SAp-_z|}B4en(w8bT^#{eC@anW46>!ALDn z7vPco`=>&9E5DfrZ{>HWqjxv`%}M=82hiI@RuQWD)LN-!RTWMxyYBMstcRh~*7_H< z+9M~@72od}nv1Uu$;HV7bMbs$rkr_fg`G~@8*|ec`84`^zx=H*NTdI}2mY1(`Tghy z(0@O<0+;rG>)1(p9iL6<(k^UK9XmVaQn4>BH_&sIv-K@ogA^{-bf*gTsd_-|Z=7d# zd@jX{a)@Z8trw?aFW#k0^l9M1;s~o0Q~wEZDjA z=bGx4CmS!(7J)?@)DKnd=!&cw;1|9s zOI1#nzn)jza@H?=CCZ<_vHh*QoQ(@#O=YFB6jG40KKiGPyl#}2*n_(kEBv=oZSkCS z6_)g@mgX;~S~am=6^+|6Re@%)S+}A^P+cowO@|Eav1Kwa>>+(2JrZ=L@iRPsHum1* z+>xB{k)ZyK=9J=~cPIxOFRM(&+J4Exw4QBZwlKZJ+&x0sQCTQ{*)x<}sa;tq7mrYC zvATV2B^y|^#Gmj=_?2Z`e_i79S_$L;2trevV8-yR zYE}ich*ZZqW9lZIRnZ3i8<7F7SY+Rt4wEI~FY|ko-`nL=vKueOau<94|AhH{em~;Z z!H;K6mrO*BmSq;aHVe2iI(b#O+VqGwvH+` z;A8(01A)3{akQh~)V?W7N43R|Vut~1F;gMO*2p$!D;TqgY|6SY zvnibIMZsPhVOYpg4)-?}IhHa%&PBM|GF4Qkdp#X@s~!+{_l>t~pL=z!zaL9;ZX0o- z)>e+FV;c&ayTypqP2~`x+wD(e$JBcE`SPlKaNQy;m$4)}m)obJRowMam8hy#1H-z@ zew=8~yw^8hFd>{|&x~ydKKHclspZ=9W2?*Aw&m~JGeUnU1rDG#Fuf(O6tyLSCcgKb`#H*3u?D_t|(-=0M@z zgAWvLM16;cR0a)(h084T{X}DLl%PjvIMtz{=Hce+_%$$I~>8?VS>9Ob(7A5r3III zlAH=XPI9X;%QYcXCXvQ0v9)xSe^CF#KtZQ|2CVzu zlCyJ?-l8zE9w5`M8ExD<%`#WT_geXHW!jG14Up;YKe=0(9@v9a()_Xmf>LiaycJLmH%M>|){EJqD1$l9}1={5J!jxvKLZOZffU9~*HTT@Y16Fsh+)#Khe(pwHx~e}TDs4p35_O?7#G&=WHyg&Bub?3g>m!ho&8f3NkY6x3Ao zay(_N>N)DqX-u;}9*kNd5UMSUWl~qt&!XeQ9;YszMeo7vOvOeESK?A4iJZhWR6^ER z)bnt4e3}R3z1$?$R)4kU94w$^nc8iol!sngrx9FFvVr_n0sRnfqu5>*UG(dIer#`?0-?3ZG5^GmZEEWpi_&tnUq-Ij0RlgUrv(dTo65C5 z6?@^f3`d+`^&wat5-j5gyBip~3FPtMw@I1*fq<X)@A%3J{PyR$X98k+L%%B0ZTBeg4!Oy?@A2Q-D;rj$egcVNh zAF4kZZrGm(?wu#yLjGk93D6DOsdGmKd$R??OO$zuxwJT(qTb#Z<@7uVhy!YO3j+wiUmY7Ob#VosVn%%5)AZY3mQ~ z2j7^hi%~e2$MMOkR;G^aOiB?Zh*1^T{c;Ee&t@fhZ?7G+Wdj$j0)8Zfy3$=T$lz-POUTtW36W>pbw;uOur}$jU56WL>v*>OZzm=X zdfZX*gl&*-{~#wwvOMioygZwOTinxA`hnN}Ch&CrW1izJ6G)wRtH}N;9Oa2B+A!%h zj4CR<#Hi;ps`!9yW0j}5-k_>PJ%Wk?@)$^Txf|6_VGls3mprw%`Rcq>Qw4!d{|LGx z>a9^?^4LnP0}JeLo;by8I*QF;oG`7WbU0|{Q9AUcQ>gQ6iq2Zi=Hajz;u#(Wc3DgV z_N=0{sw9>Sp6SZd`BVzIKb9R9fQiqWDtii?l_uDDfjR->U!7l2SMmHTDmC4MHcjvn zM@`Q9+KT69?Ol^8sI6E(t6O=`gC!fh*5Cfd{ytMvy58Hd#jeEn+0z*u>*aJY zFwLt<^{}V1rY1> zgoNu<7!ig0RUnJS!ihH(bRQ5K{rFaeih~k+@pTM&t{s>A(F`Czutk!Kf$uTvqPzML~rzKk` z&*|~HRT|aYN-(jV&?{xGVJG{=$Ts4_!_(iB?nZ5me($$fOcr&(&wo1jnbE;^qM#KdKK;SIl;XlU{Vro z{ly+QN|k&(m}+h-gbu)sAW`#u7V8*6^$nXxdv1FEZ}$*4wBM1&VVWvL101RR+GX;B4Oz&i17_+W}|$GMr@yUd>3geS7eHZtVws z;%tM*$rqaF22j^(ko#<~L`!ZhscTt(Qi3YQbXr?(9`3Fw`9I<3U6zyoulf0Vo2l;4 zkw2<7NR4+V;Wf4&Pp@oN&!~orB-EB)Ea+;Q^}Z( zbo$51H>q}=TV!n=oo%&7fdQyX?4^`WW^Jt%&Do@5U)*KJqe{)+&dX;0nx z>c7v@s=qN&`zV;C6|hIYPY;w}M=mPWmQJ${AZJ}i_2Ufkg!3z;L+eb<_q^t|Ue*43 zd#j1pndryq2gCM``37Y*(|>PMPgF{a*QaW%$Ea?lE1~(L{jyr-cYE!x@q*k-s*i8N zyjrd+@+#h)Jwc}n2F`a+>GhH)k+gag`f574imt2jd0s_NDL2~#W zf!RDi&i>r>ygXj9@v1#J2T2$9@D?9Shp%BIbEGL)qE5SC#Im zGvKP|Xbsk=#YT}{j)4fnwI?g+;M#H+C1waKI+~7DakhjlAxAo`OR~NElzVj1Iv6OGv`9S!8nMauHf@_B)%R;@#xjS5AU<20gCfL%z54G3Y=>>Wy_(I{HTHp(GJfjyz=~@ex)cC_t>EZG$)vj68u2tZ9 zX4acEyt(9WTB0rRRfEtf`y##mIP?hj$ev2CX?yL<& zZ<+|A#`zpYZX88}>iPS4iIFvly}2NU!nMSkS6hNL{+=~HCl@)JH`*2r3>B5JRH@Wf zY|>N;RPIxgAT3Ucoyf~fZ^4`p3RecKC4fxRA)^UQMPLhbZ=3I z;K2JSYmIxAgN^@!0|`o1;j>H6TYNaZDG5}9>1)Y_4?6?@K(K`tu3 z&LyN`JE@ai;*g%tA#S|xIs33d@oN$Vsz-i3Dp~-R5#Gd+vzR$%2tl(%V^Q&V(@%7) z8G%jb#8oKNc&^0}p^T5y`uQ2*LCJx&C;0cZ=Q9Ip&o`DZIfD9iRv$TG0iiOFGd?4$ z7JMr&tMMy~+M0%&m7RflK#%&xso9Aq7R;IeXjKS07tq5TC{}?o21HW`LFvynNv3~P z%YxWNq*CV(8&(9Bzw6ybEPI>`>e&#q9*3i|9ul<_f3F`$gYV&>#-(AbLk1rZSAIs znPaU9xgpq^7%#E!mh;S$zKgA;rkRDUOJ!Xm-MaAhNM`2N4WRyCX&RV$w}k!ZwbBpv2qkjF0#&F3OafG& zXbkCnkVjXatf^r|jgwf+o`3&4YHAMdsp5^xP@cGOgXVop-U110o*^sj*+Zs|OQIyw z3px>*O@}e8&fnPcSWV&p#<~>fN6{8GvT}C9I&jlE$c}4`Xnu6nh1#D&c71HMzPi0e zN{|%f^&B#!^)88i?=f2y+0#q4p*GRZ7c{>*+;nR%hk99)XAQR2Kj`I|J!yXRe5vr9ik8i?NJ@Nb19E&!pjg3TB1~&f;~xvcQ{8&@dgKLdz#Gx$L9#lRyt9o z69t5ZA#B?mDWe<;7aiq+j=8qmn1Hq!wsy^JagGJ0f)q4I~xeMZPKYAw_Z zimjxi?n)UODW1^lXQJ+HECIsu5;BAaoBkw3ek6fp(&^F4$xsw~g(yaoWB92aA8|Rm zxQHG{?GFk_h)PQRFgKI{5&pvlCd70I&3js3Xa5Y^R)5@k6xYx)5K-)p^=A?k#+!A? znMffkRO`{+YdPeJZhk8dub3*>tyvE4P^svgKV?a6a#WpG(ly@6S;uY98+v)mas%^& z?8mL4;&Ep|^!RmkIpAp5?p&m2^%M&9X$fq_$ zS@O6Pa?H37{j*}*{4ZL^-^nfP(t}U-b2rfT=gle7tCt&y9ir~i0JT5bymd?=DdPo3 z$e{fm79%qwOCv(rw~4q?hI{L^JmoYTs@5^8TGT7*V1X*vq+-=As+FlmZa9*h^eF5= zw2)?1t6{T;pO~PAB=!!g5^V_*Ib!w!Cnj?DIWaK?QKE#h2SV2Q``3s7+9d_mOw5_h zoiw~qegKu^EB--L5{3XHXBUDZT9?@WV_;m^$+OC}rJKMZz0yCxQUHxq-Nb<=HENI0 zer7`(tr)0*gNzK2YP5BJ%B`r#nmrsUzLJpiXV;n{Uo$a}Blxsq2~}R?b`Y57v&~7@ zd3gTbhI^Rh174&R8so$Znzv4%;ivhqfn)%(sNuX_+8CmLh4PB5)+8&LaG)7P(00}! zyU1(c%s%F1!!=kMcg|xaAV-)?A~kTMC>V#S z)@%FGNr_QU_q^tE9O{=P$Rw{Xm`rN=R|-7>d5a{?31p=NlFr;(d-WfPrFDWPo3!@hwaHvoC;l4TjY!j%eI=QDtXIm_gU<#(HlLR?Zj&Uf=I z&~5F_N&cACC;8Q@``C3OjchOLVslqy_Z&js?RxRtquO*G40RFdbNzSp2W^YmYLZ^6 zCR*J&ps;Hy*3G+VVV^Omu%SlH&}r%NRtrRFqCk2gW2pMTAO!YOeM0r& zKwi5}NhrKq>H0Gn1m(Q@ZG^1R1p+|~?bZ{UB`Ez$SjfM5f}qO6ojnnGILHM70|pNV z_UrWfQr8A*FniQl6Hq6@G!)n^-E~9g=9Ttg+EbHIj7kpcj_e<(3`uKXc9mA0u6VO- zK{N{sFR-O&Idc1MwBWp4K9}T0OV+$w6`jtqVaemNctv>cUlkocJBXG%EuR8BuA~nh ze@*wh(P>?{Sm29}pVApES!+?t-i>o7@hJRGYUOjgf2(xZYP!<8our z_({cJ*eG!CAfgs{#j?-H)wiM0uw$#wQ&{B~`OYoTciIv9&YbjabTn#G$zR`%PJf!; z8h(#kEWsa*Ypw7D@nkv#yP)`uJ~bD|Q#go9HWa0NK$HzGO2H*Y2cYdPUcq@|6q#XD z33+#sv6GTgE4cC?B{_h3)4DP&RlHHrl4r#wxG3HU^kunUOh+m1XCn~lxTHA^c8}(g zpORf0;dp zWkB}&;?a^RonWzIdkO;ZIBmJe>A`l^Qi9TD_asSDO9)y+&;^TSdj;D`t!XMKQe#?J zl*%4)s9ykx4-}{g+Af52cA%1jOQWxzg!38{lh^v2v&UR}pj}Ml{s4aC=5X5B)&222 z`by#j6RF2jI{Ack``v18p31cvF6)_jR?n>h=kY!J1*`G}@+cwx*)QTg+N$|XRr9l- zfsPHz)rM%vYUc`nIUX8_*AvFVtFbbmQ zZdOWbjH2lu5|fH$3B%gyJJd>m$f}@nL*UuHM9{Fv8jub=0_;Y~Djb_(^Vlh{8QH}D z8Oo-HqMNQ(>(L2q9aWREvKAaGq^Rp9k0q*TS511RKbfdfTLaOfuT)d^sM4vioNq}~ z9f69UmJ`Ha`dT1nE9J+;n%AOdsKJD*qfaTr^BH+=!{@J9ru|cxPAMe%Y0F_SJz$uQ zs=DL=nD+!Ty#yFF?UGCrx6zlVbv&=QI(o*tg1E9A1VSK_ei1<5ERA9MTpK^UFtCFYK4TjU?YT#*~?pSsf z5pBbT{OY;)>syBT4MI|}A|=zjX0F6u_xF0qyX}fZIzl^rPTcS@_uGmK7)*J7drZXQ+>AO<2wlFFY zZ-|aIYS<6`!Zu{Y}Q!Tg536oX@U zGr)SeiMg;7P_Z+`5}BpinW)x%FQI4h#(HMq5~q)vC%=AH{P-JwFQ$O%)*y?j5gfR^u=cGvp)(6`@#1WlM7_e z6Gry9wY+OtkKLLM@X`U#yMX2CfFsiZ?JnTubilsp0F$U>Y%kboX?!G>v0ETU!iT0k z1LxWF?FarzzEiPfGX>q*%zdQ$G^g?!hub!So>40~w8xRol^iamRa<$5piXDWCz*dR zs3@9XuTyrVLL_H7BrBQ91vm+?@I}Da#AG7b{geQb`ENsWDi*_`?fk3{(^7_KRcyTI zNA?_@oyFC*O<(u(znxN?*l>I*^0WD>TII28CC_7DG$HS>C*05TBQIiBv~$zGpZ{gz zROEN#LDS6IQ8hy+fQeKf$Umha31y)UkOA6BI9{EqqX;W#uItrk_>o6;VW@SEGxW{> z^w5;jTN8&C{n|pSc7s;J29H z5`IhhEfdiMs4K!YQ?{ax_;MA8E}+QL&drW^C&Z~AI$HB_>Qh`trc2G3u$mcU>n?L0 zI!i{bCj-gT_J|$=0}~4XXYSKWE->5Zss)USme8*+VR*7EI=X5Z?iIMX@UyZFcN^|5 z++Dag**4t7syJM=?mdgoltyv4{MCdNBlwuXB_hEy{r_3(naf z8uPzOe6e&IERAore3Iu3KLes~hB3YT^7s{qBmX5gXBanOUmseJ*r-anTQ*KpR%Qe% z*K5B+H6Fxuxlu<%!MV)mVGzFEUjQ=IZTsG{Px>EKTkquZSiB>An3^Tk`tVQjW zA8!jc2PNW2P`uD(IdKk_bBb#H$oDNl?tTaRNbITD-_#`d@je$1kUK?;)pkEWFv8-M zpejqF!g5Y|Ds~U?f~D*ys--5)NJse2$n31(=M=4cXUbP8T`Ax1T7lPjYD1UQE79AE z8^{F*l~mb=oKB)X`cyl*k;F_2eJ-1dXBJUAmzQvRvW^N-6Qbj12GQ|zyHWZpQQF6Q zt$gZHD#uS-%BNv5pXGCD=R7~MeJ3fInpH`qnebFc%K{Qr`$iY9!}WR}A=rAh8mEqy zoAKbkF}T4kjKlXClIjNWVJ7{R;>$jl({o7X0)Acmy7_J97x3%l*9Y(L4{J1%rDATm zjmQf~prd73#&#z*{^PeiW4k+JTZ(O*h2EF(>%x|YA02*c#&bnzt4`d%1S_&~Vc1OQ zK&EX2sJFcU!PO>Zr)%jy*()@D{-5?#b!;glqB?7{U>t-Sd_}@%eptwSi^WQ`^f{NP z$1~3v1sJK=7p(8NMSxYzGv-X~jh^xMi>sqocv;3x;Z%Suu@hy9UUAH@(Brc; z&SP?R=n?4-3BUhM+i6;FV=1IHClgS)e;c~u?` z;ornH$S*$%rY?o9!UOQ0zl2H14#4FAf)K~kAwrNvujmYM%z1+0*R`i?#>-)YNYYVE zGY_VbBhWf)xPk3=g9-YDGB;!d_zGGu$rLf zoT&nq7?dER%W2yFMzLXC!Oo@Sn5{^-B4Tah*XboA6$WAA`yL_uM;JcDP;wnDSxA|) z>T$ z*tLf7r5$lq4H3G8-!k6iy9}3~e+0lz;k40Re8$SGN`aa3S=FlX4)D{Y2-IVlOFhM; zhfcSy;!e4}^ksaKjlQ`4vn0l7LiMgAewm1+E14;E3i=n|G-})kPSy@f&}{Jrkf|4Sg!kkw@_6?@N}$zPYZo&6ZcnR#c06jXi=gIgt=vA2$70iKsG!-QoV;`lb?Sowv}P| z&8=z`yb6y(F=9Ts3@?i)2}%RL!M-9~so2fmP%`=djHp+<*V4={ASEgU?CXeUnj7wwcHTjPU^79waI+1X#O0_zYky9pWPJmZdM4?U;& z<`1pnY|c?W^4m#G-PAOkGCmh9;dAK%=6;4~5X}Ejx%0=0*ko)@vl4D_ z<*=voyAD4u5jhZ7Di(J^|A13~Uu)o>$8W+qNqOIUKW6V>!~+yQCIrN=0u?I~*#?KP zQ|_m0r7Zt$0*!|#szfbwW2iqW3NJDA4>_0*)o8JyrejJsEp@6UaTfbTE@giq$KCu$ ze4Chq3dB`oHc@#kl@3^yfmOV3W3utFwI+E|TX4sb&ht7v9UwER@&Rs5r|+ArH`_EkevsA4UVW_2+RfzbbA%@?Z+!kMkh*muE$F(3q(}&nuc)W5&E9H z{yDk4Rkxum+*(Fsg#@)BG-A*|Z4d?(zeA{Lbh>P+HW^tx3>N#bNoM3njO4F>T>_ob zS?Wsat#VWeOre8$1$j)xCXg)}$I4)3nYEWa_@)8h1`B1{YJejE6sGDBQ#F{LZ~#9t zfS+A#n5u)BFqoG2fe@1bf9U`(H^8|7f;NFen^W<0V>?lM&s&~v&n+M5C2n6rOXVfTY7zoI z9c@PbF*3Mge=pvfwrCVyd($@VTg9x3-n^FUK!drxYA*VabkXS_GgC}l>qjobWG*36 z(HlLXx1&WpMDX~JE9`l&1{7#D=fS6!scD1u&KRXBK9-FB^YI)V3Yy$O#`|D1jtrdaPH?#BCf($vIRu! zQepM;s}1mYOKI#~ILO;l8KrD9`G9v6Ho12cIPzx6kLe~-)FNGJks{v^G{@Xtq}(pX zPL(0AQL<}Ic_$9?+~tgJXKZ#x;EY~p^f|+384@jdG{;8!&ynUT|K;k5yL9Y4q;ZsDS_^1mL z(?g|sz>u&bzDBvzc13j?dyuS+1?f(nXFdQeL(5pEH^N;j2Zk6F#H{yK5R)2^U#+lS+p?5|sQt^0me`c5MN(3p!LzR; z>hU(U{8s**B;DotEz9um{M!YiZqcSy&e;qOafRs$8Ls5pNHE(HoZXN~Z~^fItmw(u zt6HKK9|FTb+*{?x`u>$bD`I+#vDsG~s z%^BSoP6ajw`JtG)#kOSr6(S*FYY2#Y8b>(ii!dYZ;(-REChlIMjyYIMFgm`a0SSl} zB$ZTjd~&9_Y`KuI1?Gw`!Men(b<2QR=6qM+y8^2bJX3YKqoom}2uz5Wd@$xJfc73W z3Zqck6iu}Xou(QCZC%zZ0+czLYHafCXc>iVlnXAvN>X?V19}&>gRm8vZ8Kj+HA*0G zMz1r3vZLia9LBh0i;?N(lt>d#sH6B=j3B5q_yWFMVeB#{AqQ`fGc@e#Xes6k>Dc7v zzaSR7j4pA;0vUc}(GDtlrzF*jeuiy0qYcc@pNW6;jCJx|hi~$`fYX5S)gLv}YzvNs zR}I31$|Hz)*1iOrFsJKfQAd1?QNZ=(1{p8xh>Kcf%f*eCs6y@+#vE87XCR!5C72gV z!?=uzk1Uq#r5Q>ND5-gFC4rd`_1Fu{u4y8hs3Nw=##Lki zVMEzcf(v$)$B85o`bB%pabTMM#BPRq`{yV@Tqu!TL;`5r_uIVt4O2gmi#BP4bs;9 z=dp~-r27f1`FCi||1xuza(XIdYyM}OyE$cR{wF)&J}vGZHqnr8YWyn5t&OL}PVXmJ zZMcgZ_!Mxj7VZLvo3qCs815{G`vb%M?pKWv)Z7h0N4at_b~(9#mAA_1SBnoDq!NF~ zfH@WW_L&BGg&;$f_)?1;tF_3n$uin^<^+!&rrCyf zDQTsA57eLocMs@S53n5G46nQti{p`_9NmYd>8|LGUgt@G{MfNdwqHlv%vd=U+k3j? za`Isc^Bq6>K^``cx2G9M86a*>Z_hA)1E&h<@n0rQqwJP4RTb82E&5_;BlMl2p$!)L z4N=Y=mvjo65%9kt;M;l8inhS#fZG0<;!?3gEjPy)pZwTa;*&BA$dhT1U7|Q~%&`jd$5uiojQzZfemNU& z_n$C<+TCaOj^y8%j$J?kz-J2{Y2!wQ*nHU&k?nw_6FWZ*-i~CG*qN5fvg1OT3kLkglo&*eUdi2~ygq(v6F4CR<)Z`)af6oq_|J$y+_nuMxd6RqFG{aL# zva^+jAIr7W2e51P+abH2nrh@3U4HB#BhRsGSsEnjaG#*AKjvSD3PCE8mj{bcV z*36OF!B4i?{L<-F={Y8kn?adlT9OAa9TZh_H*-vFFA-{UObdpKNj8x*iA!YA5htZ$ z51fwbaC$MDM@%UT_CL^e_uQ_YS)=7ibJ%(RbO`)ifEP^uIT*8>8mR_z0rE;Wc&+Wx z58L$LU6B*fqgcw{W#VRJ`2uN>2nR zZl>YBen8l7k#mLg-6MCU)<%vX)leBn&sV3y3T9*&0BX5d$YzQ{WHto9DpcIaj>|tD zDzXMgGY{BHGUQ~XLB-KXMg=Ae(aDhYd*ltjG1l9UYWsDnUnu;W41ce4N zY>+_8|J2NTqGbV|ZVe?Xt5Ou*V(6^%Vn9i#{8>oN*wllo?KEB2I`D+8NrBAz?Q}04 zr+Ie0I?DEpTK~f0hMLkfYWN*QZQ_^ez&$@+!qY}YHKmWbFd?Z;+*2qc8Cm;xb;uOF zl1;@@r-@v3=Ouq=tfdq~I#U4xtjbDuep~IHa%=VwMp<$a!^Fit+IMa_Dv+yA96IA7j8z z0POjk_#XRuk8)7eT8?5y)JpV)VC!VrZ;{=0?s#NNiMsKSQ#9@-UMlvvQ!P<8F(el& z(ci^P^dFYPt|oH8*BWp(z)T=9H@?YLjPg0Nmo&ehYTUDV@^|q{@xV137spgK&wDmE zYtG4B=#;-b_u1@dd_y9CuHryLnFeNMY*09BD@il?l>aG1zE#=JKNTo0SRKQ>6Sii+ zvw1Tz5&4|ZYrd#_*7w*98ySWeq0LtpS9jPB0LXO`qkfm-TH=hQ&RFh@70zgNMjJ*V zc{5)GkuU#Y8W@Sb%zPD1i2caD&ujkRA@fx{{%2r2ai{<&U=@J$i{R2p!m6Q#Aem_h zZDGAzLZw^gB_qo&6|wy_w{A)BcZNg8rt}=7uDZ%kZBb#}dNQ;CEyd=~qN|Ja+Ei?Z z??bWG`VY&&Bce7nX%W|2#0DZx*ffzwc+SSqgNe!B>)pdc?R?J@- zy}50|I*tlOtGV$Y*^Tdnb?9TyWo1o_^y1{Y;Z%Dfd7p4FfFr};Z*?xy2@@@waZ!)2 z4$}2_nS%ZNg?OZ5zYr6j(y}`lDS+;v`q?vAtq+wAv%f>N=4 zPf{Enc%}M_M%`g<_va@fe|QsTzwd>-hec-p357I8eysahLFdm0Diw=<#W+!(4!qX| zHdx@cIz&1??QpWD9JxWNANdh5i8v$M8RSBId>&|P<)$)R$uG9JZ&=){l5|{+tqbAW z?+C?Oz*HAOpg+hVi9D)+Kh`hRv22!2+j+UxG@>a-}rUdSC!67j;(=w`e&{w zbj`9Nf#=7D=Y?7vb|VOH`@KMdibRj_P=A6~8sH`XnYKJ6OpPrS{nNsmu?PHm!)f7g zPoH)Tp3}vzJA7MFP1aeLSd$fho$-7hMU#~{C&0ROIM$B%V@e*d{7DY1?makI4=s*^ zE0q7gDgV`!aB$c2OquWftnTTx2|P$xz)!P%bA<>~D>LAgpkpj5kYR)CB5Vj?Hvy7E z3??r53Q+Bz`tkq=>1UPEICn*$aKny+}FGP2%jQ~ zsB;^}6Se~ZpgTc<=L|zVGeqs#W)WM=qUhfdNb(lZP7<>%i5F`^DW|Bsh-ma9RfdW! z0?4himb0Ef*kS3U`vi!>B20s3w20zc6q^!5W6cXG>1fg168~X`qR9qzXv^W|t+R`Z ztV^(jl4ocFVdHCJ!Y^P#aQ&;P)Dm4q&iZY9Ssf?$X&iMpH2)xV=wa{Pz|;&Xh0P^; zrE6`J%$?}=cL?i;ItDQ51P$BMx4mX27uMPYz#$gsbL$J2_5d6;cPaZwci99$>Uc$N z-enU2+swfkSpSJ&Ej1qc{@%DxUFkn-+IQt}LY3Yoxcor#f1MZK>Dgx1m9BAw|0OtG zArL+j!3e$%;k1a--ZkVb1e=DO()z<^70MZMR)#~)s)5fcXnaX4!&xU=riX1mepPH+ zMjw0)e=tVqd59y~N}hgfd!rM@9%TH4T#MK>q4LK0N^B zs5HopBn6fc4rEFQvJX1Yy?_KGehCSQ$)zrMD_%MjP(6zr!iO5!xeQ; zHSR;a%hK@In2h<4x~w8{@7>mW~kYC zBG-x0pKz7FyTMM_zamOg6^doGq2LOLQh< z4@Y;UIFC(}T%>B={7SZz!x^b?gK;9V-14(%eP`D&a&G%77Oh{raTuEZMkxE{%c4^; zV%Lc-KmROz3`h8K%1_MOcFx%05A94^l9iS98Gf{0%P>0Z3t945>Wt+Wsn`#zLFVV5 zv6&#~^5Euy*y%u-|6#OS)7)v-AC@TFMf}(zy^=VS#~gLhn&_fSbIEG06j0m88!d@^ zbiR{0>*~~HxLpDwg!D0%J7a}2TAk7DjKwniNW${$#t?p_2QpYqXd}#ZpeT07x=}}| zvf_{GBu>P|G3-eLMhjhP_+U+R{C&mJ>geE)IpI8`Ln%kcKUCa1OwQXy9V=6Oal@Ef z5Jc+s!Jw5Q4~|r9A{m#8Fcu8#bl4m{<0&{Yjbo0e645cds~gu#qwHHoXRnve@md4O zlgTX5+GuH-znW%NF`{Gi`bNChtgG1H?61a6N1nJ{O0n<;qt~(~A32iztGs9W`9)%d z)&dLiTfj{EDwamAt+Zj8Cky8Y&+VchZUzAB`sn!h0-V1zC%5a(jF#ljAOq3ybu4zp zmr-<;kQY)JB-^#=Jn}2%BfMnfYb0oyr|VNvUEj5vIyVJ&bN#aNgGzn7ZAPJmAy(#*6WWG6Cyu>C6s!r;k5)tvfAQx9<<4?P*uADOOY&pKh|`yYTIC)^1B)sm$azjRd+dXJB??Qm$%U z!U?H({T3GYJK_y4m6f2}e>t)(P3OtLnu zgo=7ZX>D;)X?1aN%erq*YH;%*8c z?(%LCr%n#iUO5=p<~3gb=0)A_!TSyicV@0l%Ah;vx76hesx?OT`|nu$EVTs;XsPBEJ`A1t+})3CfyvLg3B{ zrY#m#$Q{$0%T2_d4i+82nLR&0pNyy@KZ2{(3w5u0RiV0Bt}EZu`e{3H7A9?TTeYwY zH`sI#t(==4`wnc}m+$G(7UtMf!Ee{^O4CY>pi5wvZ$Tkktp#!fYn> znCxaPz@&vWvm`W&WTs*dfl~X;NEDzVSf*qlmYXV5u|E<@F(1%8Ke~5gv3$ude$KW; zwuzHpc?Ow(B-n|-^|Y$Y>rRkZk2pwF-d3z)aQlw0Vc&6jAMqA3R|gZrKgniZW;{?N zG~2ucP1k! z`Qcv0lUG6XIE4p4IZ&i~hjXj=W>EI^u#od)&>~-yv4kz_B(&dqq?f~V)ky`|f^GW} zoP7{S+&8qF zjMWM#SarbeGIqCo|8>SL`F9z!^04!^;X4<0`R>kW^F2_k%W^87jDF{5qMjQjYf9hu z+EY2)E8Fx9?YJ#{Bir?OPidQ%(_UNAPNTw;J)OLP!1YoagSvdB%#$B2Pem8qVPxc>^7U~om)ATc zkHT^d4~1k`U_#k9hfsD7mw5J5CNd!m`UVpBN0b-dnLFlxN8E~0A`UrXmT(U{Db4$G zgNfq38Cv)vuD6neBiUoQ>8+ra8-kf22;N4Pr~zE~li)Ro#FPS>I-Z_L*A?r1*Ke_(= z8-5H6j)kG0z>j|^ufH82uTD_X@;Yp=ybd#w93-#DA`e^(I7nW9K?lY8%n1WUV%R6j z>qnGmMqYp7ZCRI-%6{aYml@an~~OG1En?3NZb2=N!s~Fnv>RCe>iO}vO16? z6Easmj0sQLsGK}s>>yX%jRoJSD}(~bOL&7OWX8@RB-O6l#%>v0J4@La*1p@ zUcpMb%f#2@ulyg1t%hsNEdOf!()mZTjf~1fwfwKJ{O9k|Y=H9LUQf|%9(Mjh&{~C*>{40K@{0HV=2u{x+7+C(t z%XdKehxt#J{~<%lpH7?nm!

    jpM&9|F638fA$H>e<#M^@~86&%ReVLw@BH~|1Cg{ z3%ANPhzsAB&*1XUzrf&?|5LF2%luEuL>-v_&&l^c&p&-T`9JbM&;Ri*|BpOw`CozY zZ}NYLGM;}EKra6aWgC?LtK>5z{|6ep^1mmxf0_UH9!p0ZnEyBA`=93@rGWf@?my4} z7hL|YA{hvrf$?whuVEMMnMkr)Fj2Na`9D%VF8|dWLMzfqw7k=}cu21qNL{u~E%y2^ zt{K#Ey2r?L$d>B5uwAttJ?^tinJZk%wU?aDITww0+#Vv%sVZ1^7TWDLky4AD%b?bL z(s!ur|DlxQX-wL450b@5MjFsdyu==BkDr&d_V_Ew1!39Z$-D-U&-zA!=Sh{(~xoV)Gi7SKW5YPc})wJ^OO==7K8Gy-f{ z7hQCVrFL4ufE_ob8@(NG(S_2PWH#Ll(%(_cFSOFVWR)rI>1~bfnxq;F>c1vwqPa^m z*CdTMcX&n5K@PZ+tGsrrD}0RJA7}_?LveQSu6NcH z^TM^Km&4Gj;tAgJ41sbSP-!n0glp4{HmGcHT9b{+sH0!ol3+%G?X zGs^A6YYJx<>Gebt_zgjk34E%t$!orbs!+`+uje>+{WZO99(oONmg-;S;{NeoCK<*P ztJmgGu{6LK=(|Dt26-WaLz&M6hcNiL=y1ggQPzgi=+@O>L_f7(k{G2m|3 zfcriJV4K6?B;_TiG3I`4vzM6C*OM2Fk(b}?9 z;PEX6M+E?721ie6_9U76u5G;J5~>0mJ=(&^)D|~VB@R*1l&jeuc<25g{vI3h_f1k0 ze_JdU>Du7sRPdN_wtedC4YH%$oSiIxZ|%{RR&}q=kMtT z0Dn71e2l;M$!qug9Y?VY;xF&B4&v{yAmr~K2J-i}pUhv$PiDU0V{${j`Xo2EfBF;h zHO~OZ*V}u3EMG6k%kO3#V(l8)XaQpPVKedJoOLR8JY2!^4hueJnT%3iquXV<)XMHH z4qT_GgDqO9OXf1+H}-NtCL&kKNakABsoWByMC43)4v{7|-}rCx%d)-Y_nvVszyGqH zeAfWTZz@k|52`0y|Y-@CE(=l82{3(p@}@G(neSbnY6D8ESuZnFxv@*pF* z`+)pjEXYLU3K?Nl%H;P{dH%QgeYp4&^2>ans#RwY{dj&q98SuX-+e`c){rj_ge5VD`X7L?=Ky=<#%0Q zCcobnWFqn%8AJ2?4SDW1zjPMi{*$r^+Zia^GX?MCS5#GA))cAs*Y=FoO1I|6Yf^2; zl^>new1dkj8T16Vtw!1CIH3^bO?mPXR~1w){-WJsIlGY8@p!SWEn9~n=o5J1k2RAK z?$ZC{^v5Xuh)chj7lL)m-d57}{58rcL%PbK|CquI3SwrV&(#m_@D4IF%53bMMu}|* z9-`hZQL#SFS1*A!9-t>6e00VQ!5s>;Js1u`j~+DcwGu*+lP7|+b1W_@td**E@=~<* zu4u`6W}NbXVgy@;5sihNa;3s?huG17k_V*e?zYIM=zKi)4 zBC!SetHDCt!3sW*HHxXaZ_51{;k{u0U8&$)eWtrOx*0b386Pl5g1zNBmJo`;2!8;3 zmf?HMS?Azk>z^gTgfv_U1JJD{!4CS2U>Kf0gFw{SeLx1!29Gt$T!w!Oqq)3)q(geB zkX9bGy>ca@w-4>BNr6^8|=(?9{nQO>i>55hM6I zV;wR)qBTX>maPupR0l9p0F?lAL~bKC<(AcxSNX?mOYw#c!IQa!-{4w6uem%6X{IQ~ z;tOB6$&T^*ptM9ZTTw3zTkB;yMtCt zdzfuMAl_vAAw{gqtn`in3H+Ae%0*m;6g+}$!c%OuI};N8)jfv-IK=_x0l1jp#}>>h zF3JM-Rvd2OYhXiUZ#oBmBDxiZ-o_eS=O25l|GpcqB7fs=xTCRYZ?C<*;Nl(ZC)w6C zY#Vp7tSi_ufpQ%v`#pz>8C9f;us3~QsSOlGVI4vLjPoTpaxi0$bKSsM^O7 zlf{vna0maRihmA+D#Tc?k&FrIonue7d&IH!ymf=-sN5o%D9zK`UJz92KSvFZe&<&z z_6d%*QVzczPMyvZ>c2ZbVZ9o^QSy7v{5ZPM^L3is{g*S1h#KIpdY}?4&6|%c?Oc&v z^K@tJgjKa%N69bXwq=>@^D3z03fIUzzo#00ENpcCFY!;q{LJj5^M8%0?m(SDZp>=99V^qPyiS==q`nA@P#JhN6 zRbkWVXjRcaRgLkI%(dinVcmoA`T`EUuJ;mSifbx1HWvCPj$&zb)(DgPmMOO zD(QKdwGxfs#OUU$i?im-HiClM19O;_TcNsZ*H_)}T(F7aneY^)K625frs3XqV!v8J zmaI4&%!0`q@X5O2xyIZ@Pc`jDbGU>a1|xy7LN!Dsm+t#wcTrl@Stctc#4aFW^DD=C zPcw6|Gu6b?MjL`Sv3hvCiNk;SQwR?}hsT6zusmV!^*L+c7miORMKY#hNx0WjT$f2U5Vs+CM>~<(j#jf60zHzEk zoP8=c<6>#$^Rmxyc24Ix_hPxVtE#SmXYPx<3D_sZ3%m*KUbxZ$ro(VdNGDdY8p_-0 z(C_)B5@A23x1$T^ujGU|=d9MgL}_=eS(Ro^Xd@rV{FZwGNR4|YCTT(EliT%`XKhD7 zn)7mlH^hMEJeCoa_X%wiBh_F#UZCG6y5FxPre7Xv-2fqD3PsL;+BI}|~=f|Jb%+KC1}ZG>aLX7py-JGQ1N(HbR6 zYL(0{V5LhMv3=8lZHGpRekW?USr0a`r%c?kJ!O(RMO}yz9ZngHc2VAZ73Y)r{TNjaoz2-dG+}ekQ^HFV9aFjJNuCiXqi6Yn5GI_eN{DRrv z+}v~!cjxCe?dy9*$%UJDUiJpB)Xs@AYgBn_cH?l6B)l#V-D**m5J zqX#{tLerSkvL}PTeu(5eYlm?vUYhkcr`p@(B@$`vz2qe`h7ztl9In(?hT=+m60UUJ ze@={hc5gA3ec#VuNyS>t*|~Dh+$=Gp=q$K!oBXsK1GCGWgdRiw5U$}5&;({ZJ|2!8 zeh-5A8?lXDd*#%py;NcpG}D6;k;qR0pr-GTk%-K=+d)=hzYMRA$Z_~6)sDy{Y^s7C zk>f*KD~(>3cWE04^S`W^9l6;`x!>QTBI zR9jxEr1?w5c8n&v{|5=e1bYa9{dx`b&&6 z9s9b8mg?XF`FX!5!OBo4$?#t59?ASewQY#aP|4Hs6mf)?IFT)*US6=5^p9j@e+#!+ zd)ICwVA`LRj6^MjkzF$K-u^;DcH8^gb=EF~zukg%ga3E1kEpL$`0nsOYaxdJp#k`b z^Ucu7ux`tFpmB2U#;?i9$qHpUJCW@_&j!Z4;LCvURAOVkse=;sHC)*r4y(%~W1&9d ziHSwQ0V<(n?A~vQKzST)X=u-=InQ+AZ)d`Xr)emV_GS-_BB({p7X|cFY{ifKOm5>&*2a>_4SNjBYCLrEOc@76 zR=odVs=2S|sslarGW~-M(O;~dJ(B3z#QZ)n<0w`8FMK5{+N*{q+K@F%p}B-!NoXs* zLsnwyW(aw`qHkU?`>#csD5~p&DD9MA`P!a_aTswX6Q?S=@|7{OFYk!l`BTTMj>u23 z(V$zfX3fYSG5MWcGUdUT5Fe?R&j_7u)v+`+nEH-?MM8eScuz zo9(;SzDw=*J{v)^go-`UqQ|6%rZbe@~9K4nEVIt6`KonOVm7f+UM z4Din~NrD$>57PPKJ`2UqS2e%J=wV&e_N-jq_-x##ig#7(E~wAl(OsLf(Yv6t_OOjL z7p&&#PD*1+0Yd=R^zyb>Xs?FjiXsF zwLcy-Xyu6VEpr!M(3MrHoIREEkn1d5ArBnCHg7GcMQ^TEpI0|wQ!Pz#VQpzw zUG%u8c@F%bad0-PuHxx!v-i8WYT>I{y>-c-WO3hT6>Ej^ZhE`cCpM|mW34IJw)(Nu zxW#)(7PS7G8dq!IKY}Gp+Km43b87QIf9ZNRG?3z!F&sBi2<}R`bNk&Ta=-WYFg$U( zwnoEDUUl~TsWYf3GuUkz{2_;lOxuYcc`#QCs3!BCz)ur=iNT-KU{jTP*XV3}UWf~7 z`%+rb`30GHY{)|R_sy?=CHY!&m!f{V2ac{Vces*#z60(IChB_B^h9XK6fqQ4PAjzZ z1qZ^%2~RQFy9Z5TiRHIm;i+-wj}p(w|1{) z<_uyNXsTKZi0`IBIXx=uE&rgr)7WPS`;PlF2*7VR7yZO};cgJf(Tgm-HrWf2ZT!_A zZ5ie4o8_P7>_zf_Z>zx=?*i1{SN5gY z2jDA_Ut_M(S1$WvXZK`(p*52=ivrySUp)YR+5q^i4}T2aa8|cv_%Tg*kM4IHzl!_a z4GJHFo3skWi#%sHzI^zI`RhQ>mR@I{Dfrj1i@-F$|B`n5H!L8WO-If_sPV~ob;;|nOI=?O^5^t)`bkTFOLf27 z_;ECDXl)~zEhS-CNWX+^%X@EG`LB>YaP~^U9>nS|@#^K@<@}e(z6N_3-^%C0r`@b3 zuuBzQ>|htjUKqlr(?h-3a^X{CHI&stXE%P2a`t6{DMF@!(S3K|*qXvQuMThwPZnzd;N?kbdTwMU0_XdxHa7}(lJTt9Fw z6%648TaII*AcLJDrRQ(-^MX@%(9DLNK_+y;e5QlgPlIDCf+%#V%0NaK5HQ<}akMc_~Pf?s?L-*@p6z86y;rGjog^jqiII zd@~e*KjEtv;9&YrS1|aJ!nfGrJ4IlA>`NF9->~4Pnf#V10)N6+D8K>udRW59c1vT4 z$fdW@FkEN%#pZtvJ8N@In+@6_`2YF$^}8A2@ppA--Wi&I4$ZcQUHILhxg<1ah2~d7 z^SIFbTxgC7&8*OT>mi4yD>PSz=3l~ewuWY3Xzp>3gFh%V%R}?j(3}yP^Fs6HFkYJ7 zo9}n=J45p~p?O1S&I-*7L-Y91JRmgJ+~?xY2+e~-^YuSC|Jy?I^w8WZG@tq(7rr&b zXxGlYLDG@l90^`ZGtXx<%~>2&^*mb*W>{2dUQ#i2PVG`|v>XNKm5 z|4(~o10Ppa=KnKklBVfflD;KvfobU*(3feNmI8%Lnx<_aZ9|%tz*Z-d%%mBbWQLhZ znz9N3sul$dh*(&3vj`XpVsugKD_KRYF8ax0b?g2SBU}A-{?{#@viOR{n=GHB?=P&qZ?^h4^z}CWsKslu z(syUYm7R}HK5g3fxWzA8eA4257JDr=Tik51++v}{u_H#_%ND7C&V1ki{O0 zA&d1EeHNX5-nU%&xhGJuML8NBdTfS?^P4L}x9GFjY_Z?sutg`ogO8UOIn7p{LqA_! znD@Ri(eL1`rv8#|0)GAB9Oab=PIf^Dn@u?bvkdyaVQ|Evzs1r^Onr(hdV@CKMFt&u zOEBaOc63BrgULv=OR0f3@2ky!k=h0`VC)ZI`lTfKk{RvPt58wYU9h=O?v+iO?smW(_%^bUWibeimM=BHRvSV1vQ9| zs3>7CToLFo)j^uHuyI~e>Y>DSx=1oPJJbPEI*}0LUl{3LHF5M4Wnx4P<)u%#{sThLMTH8kOMv{7oa*2L)D zVx{b}oZaeNxs__v)(WdDb$KgQIkgiTI+m*`5B zdrE5?3q*7)CsjLf(KM)gfwW;5FD)Z28Y2|N9@3i9D}+Cy?Xrb_u!}kv=kzO)x)(mD zCz$w3_*QCLyg8rb>!vj6g;J9Aj-c+TAuYkka9f}{8VWCO3@4Jy8zY_J<<-%yw#eS) zjnQPV!z&c;vhKts%d1i|p&@RIbd8@-)w6f}gl)lIldvhAo>sGjotcypcV<#bpwV?( zxHB3*;I)c|L*8W@FIm1h9u9jOf?a#V-kp(K!(J_J9J-dZ@e=84i(B2+kvV#~`M)u| zH`2B7Kr)6+t?lt2Scf- zsJbH_3dgTr99q73WqI4V{z%Eob@}G69Vq|*w`W~6k#yRTORra(Xn8i(w7PCp-O|BG z2aWFPrzYFxGw7Kao#`dj=&AP3YM=UOv_nd}dT~cbw|8@#c_!6%T4Zf9+?k^6N`ymh z!pfB^X>y^^d$tAlg?Dvz!%`oPxALf*?r^Tw)*T|d$(@GZ9F3~gfu!zGPDNI)vxLND zzQuvoXlJLaVw7^x*SZs2^wmCBLizY&RGOY4#zoH%qr7HUme0MN@fR#`nL1Q3|?$MP<2D_4$;Bakq4$wEYZ2%08CgyR*9`(Hf8FwP0Y~I#kmx zlfHV*8k>+s*L9H$ne-HU7JYT9fJRlLc5-#CRom4TuCAi%etbhX(cO_utZYp<>jt?# z$y&k;i7UO#ecgDvJM3z4jIyEa*@It*XSuqk4NtGy6UVcnT~7MuhMF4T-?ja^?K^JR zt~T!4yt$@<7g4uWZK(-3R3~0nyS=80R42Zpz7fVc;*AGlVWG>I6J$iiHwBYH35%;{ z_LK#$9*MPGp>VIE*Ri&SH-o$@L02PrBg^@Hw&l|^g&~*$sjn59AR|~$>P9TFkQ`5KLWEDRDy_Zz`pAc8uWELC-VT zpgXQ?MN!uf>*Hb8L2*=-%9@P(X{+7c9s7b^-ptEnG`_qVEfU^13#E97VWn0tU!(Qc zQ_>o)Lfy-II)mEUVO!^DoGy}?zKI1yJbJ*zkeP1s%N!~KnEf!QrwIkv(VM6I_VPi?FSN()?mk` za8IN)ygV3-><-5hv^I7aCp{F8^n~M?EZU}7a(cpDp=i8zla+EtIU!x#xzc4iIEx-S z|8`a*wrgjWUl-|&BooU!gT1DkrR||-i^#+G=ntGgo@k0cN1VP=O> zmfdW=u4q?vv^^YWZBvzucFHW|WXjafcK)s5U@S|av+|+y@ z1Jyek8g_9p-+^d7ezN6msoSx!sxGi?S7S|6TH2ZAZ>!xNFl&f_Swd`SsM@}(uBxH7 zkxFwFu^llje?&s`|jTs;0nBRCh6^`8L&TuG&@C7(f;)6V1Po3SO6)!_E^m zT_ICY&pzfLGk=-R+Yr8`I}#6f($^E^G^m|fHJ=_iZ_1`NInAFQntJ^qUCxWFP2sj6 z3pJU{G~Eg4^#tiMbc9cLG2yS-U0Yofkd~5Vl-ijLCMDz69XlHXyK8sWZmiSmGaawq zv9+dwzTu=-S5B(d zze6>s8;MnsQb(+sP@|>>Sc6uq{f~WmSI0|+WwuHl7 zYz}wGoh&!L$#(88%oOZMWB=HZcb2|&4sj8=>k?_KJYuq;%$iWsOeH&l2cq42Z`vub z6^6LJSMaju)6GJ=4I_lmwLRDqiN?K=HWbBJon`}ukSvtFEe9Znz2^FcFz%N=>$R&d z8A(0+%vxhpxTSk9cT&-KQkJ4E!Pb2>O?1efeXf*k;e;&VB|+$Vt?5d3XWZEyWD{j2 z7w6>4M65^Icw0QUSI0YpEYxvz!2@g;E?k^gs7;^fj)@9kIo3%#J3A-3<88rKEN0{T zBVB9C11YMeqZYF`)d<9OJ#_x6P_UCuxZry%r&t%>Em53ba~EmlDcS)j>{i;PSQl&D-oOrFMPRRV^@w6&LqgoZ)ybeg5n%L zNgwM>%aeQxK6!ynoWm!0)Ws*|4inGjlRR!G`TF4VT4EmP)Z=7Fx?Eik_?j%AyWDKP zG4f^e*=#A0{LS!rEk(+G{*#NYM4QTg{`>D&f_{9JT9mAP+f9Dy#V<(QO_#9a-Fbv2ajCbojoTLycl%P}ZvO^xx6g=>@J5!;2Y{rvA3d|q3G({H4I zo^(AY6uZS-e4>ZQ8;`GO96l>P1(JUl`CdzL$`yLHe*NR{Nx8XM`uSXZj(%2r3WV-I zaAvu@5UDD9rKcEI0>p^NY}}D2e2ey{`S=~7Y9?;3beZG0y(i5l{VztmKTDo){JuNQ z$LpJ_2)@yA_?ldNVy|}MMWPtat>2@`v^@Fo#t894S$x8=+{GvI{KQXX@kw+vk(MX* zI7NJG7N5|kxcHC!r^!EiGKCOv+XQ=KZ~X137@(p zN1YJFIpqq+K^LF&pTiQ*s)yu%(KYY$yK)*XN40113EAx=x%s_k_A9Z=>3h@q>Gr*E z{CX@unC27xj*`y{iWB*c{j$eL(>Q!0=|o?qywdaJJ8{pM<%*<3&>g4kcNV_E_hj-F zz_;FJbm}1-Zax`TN6F_Ohws?CGv%Eip4|^buG=osj$7gLT6q$8=BvTG)AFR;W5mxT zPuj8RJo&um$#?3|`IS3#o_xLVW%v7RyGT9O!*>`I=d{Ooe5c=ae)>y?jSnp2Yj_JoEZC zUVR71mtF37eC;ki$9_^`wq2IF_(YClmu$XM%&*zyioAna^X}ns_{3g|viPjV zD9y)D7*##-qcYLp9C^awdw-fw^!5`Uwj_s7@Z^JOJ`poU+z*O#_@vyC_oeydKDY=z zxBZ0w{_?l~m7s72@b|RmsC#ZQ`JA*V&}ZU*M%vG^(q#QqAWB5Y^H-Vqra^YQ2i=kX z!f?TGj$i7X?TXeLda1?ibZsn%?z{tE&QT-HIr5I*8SD;0_)odgPrJgxBQhMvPjBCl zupcjS-OgR^FnP8H=|A|hIZ0`DrS}Sd>zVu_%aQFaU*x-;!y~j)jEqc4Mykxb-XeYK~eKqMuV{w(I*!j=p~ty1SfZS-ef~ zIWpaR1IQiE$rvnXOQ*$N$WfxBICuV|S>=qvCr<7O#YrANe!WW;Q6+V9oLipM)9s4T z+i`Bb{&1Qu>FwxI0PQILbe2v|e!+gy#_KoZ(S%>xz1O9?*l-B1+l#tN{dw0CN~SwX z+IaSE&eGe-rRBQhIC&-Ai!2}cc<0n%PvKTq9w9pRm$2Jj+4h%oaU$`sONP5#=`+$l zc-K_(y8nny4zIf$(bMgYA;V1(yVc`|U1hrYg}2u&gR<*g_ILA2SopWP_($Q-wuw8B z=-lqAPp>O1eC~FWxYK{L+s$1scm1>LHA4S5-@Ym3IZpH(bl2+zV?`NP?((wRbyV_D zr?c6tbT9pN9D6!#DdpFHjlKro&^UaDzHamoKDQpH;2+NkwU=wB<3zSomOHa(vt`|Yn>{G)%7epG$s9(j#% zk9qcEo}byrW6^j(9v1B38I(;n4}zd}gn3vK=t-3OR#etjS}>m~=&<&SCtDZQR_QW| zGH5sHGD_XFM#JZYrgf3LjfT$$O}=xnve)YJaE_|;IPpcjqRZ=#Sl`6Yt$lcT6CQ14>7)*q#v~;^s`?z z%3ds+xAboNCfo$&N6gB-ZIMu75r;V8_(Etiuri*Aw=VLn7E&{`KGECIwBABLALp5u ztL8P#iOp`F<@4m{sQmIApQkWa6;|Y{!t(rB*;J3pWvrDh%u!qwg-J&TS%WS>ghJI}<8 zPWg)8y{KyZ2CWl$l9{x!ipgq8Xr5YFF;~r|yvbveMkfxZ%N(#}^1B*c&K(!(V%27^ ziI+{sW(_$$u}?3&MO0kdXL`Px9-648W5?;`lVfGe3)F?7nQCsu3^nZ~Q!A-cJ$W#% z&eKnw{1=<>mqBq?FE;VwvV2t*nxbZ+Z+<){Cgm+7uaq@FKEWx_{7Wo9<4)%X=^82s z3Xf-Rf!fHeB;2%4X;b}F`=lCtfI>y1tO7MpTnmzr`P z0mTJ5i&OOQl;o+BmT9r6%~R?pBhxcGSIurP7A*VgIqK%YIiBF*Qcu&NGS80Qxt=ZU z^E{Qws+`M1{+v}6l{puA*5?$C6^!POdV#^s`;MF)zq=cqmxGTNNJwn%c+;XJc!$&Z0>g;k~; zFDUMvHMTrGK4lD;@#&d4k@jDzW_rA;AU3f%zh0>Y1zdw(U!wXqEAO1eAl&g1b;R^H@dXotovsQW$dBncD>O}c zufzWMA(5TO+}b!*$M1&rd0U>OyW{RWWs4@NlF)4C)meHDog8C7QEY~u#oj{nn5pJB z&QPTlC2Gn`1!MW6xq3cnvNkQUHk}WSZZr0py`9Xs>p_}WJ!0c?c9`_C>t`wd;Z^FY zLq4^xceN^S=Utm*m%236sg_i9sJWgtHD#=Dv|uEE*qP6Ll=l;wPukIE%N(-uZogg^ zr*6E_%HL9q+;zyk9J#&7-H+TJI-i%@w&GfJ*Kl!C-TbHPgP5ko7B=!wOU%SQ7!dsP^DwDM`wRPS$`{_EE=}&?TJ%?3i*Djt zv|H(xA)kz`(cL-f0c)!mb?9eREBd_WW~1L#omRh%Q`KthvjY2Eau)j>Mb@p>K7H06 z_gL(=a_XW+?pIkEiCyL%F!8l-x8>9p^UUTlwWfEuTGhTnT~^V?ys$@28=Ep(I8rd2 zAIocY+PR{ScIV{{ad|?ww{n zyW%c0pVfo1hqTAWZviI||Mzzq{zLCF`TqclJ8;Ou(Y^|0Wou+`kyG_2y?>2H~gQC|P?>F&F9x~~k(n3`lDp944#f-NiRr(1% z@?&M?Q+fTQTrCQ%R*NcpYLVyCGy0J{&n~N+O5eDFzOkFWv5UUZNZ+W>=o^0Yi=&$8 z=(Rd-vnc0G3jNc6YxJ5sVDwxDikta&CjD|y+%-0C=H8{_&b_799?g^yv;2?Q{E;V( zys|B`)Y@aS)rzBY)Uv@+b@Aacb>X48s;qaOn$Z|jW zlCO+>bICW4eDle70r@T@Up4uz$;cOzeEz9wRq|?eS!e^}@+#(qiZkYgv0aQIc*Pzu zYoE7)kklbiTK7Rv`pW<){qPyk?R?Le{C^9I`#vc7e`({d+xV~SUbxB`2WpJ9(<*yB!wNWNiP&lf?FdjgblPJ&YI7%2AmJ?M5m%9H#bvgLjol>FUC zjh;UOCI34c7{1Fu;oAZVUn}SVZwJNw z4Jdk;>*+*&?_#c}`O5nu*J-@s9h(J!JMO-}8NeR?5i3XEuW+xz8h#dQ1bx5qaDjR_ z$$FtNwRUdy@ZN9;+N==wHqcY|Ht#4L#Y&K6pT`;TOSDq`(jR6b?Yb4P)?qcMx` zU4`mha-Wo(jLbr0BD;b6C4FBuN$$B~#q$f)d^@kqWL_zF$y~#Gse{;|XqjoJ`Jlv? zS#;J(Vl#36ADH?+0E+t}D0cm>rT;sq+hx@7HG$F&@3rx=nYpZ+3gjM0uOaGf`A#`S zKQ#HS0>$;(cy_*Jpa*=&(mxJL$rDZ* zzEW@k@f|jPHa$)>8huACU-^&7gnKI}@_RtB`~9GB{;Q3j21Wj=ADi@SI}BO5^)~&l zL2*Yj_zP3_DTgfoSVnrfJ(_L$!2h=8x$fr=y_LPiQ-#;+>9wYt2e`D>- zbK3>z;f|N_rd?shR8<(yZ#T3>RlUY}=#%QqX3fF!_GwmyV&>4$@vEP~Yz7rJp z94P(uWlK-(>%{Y8W`AZX?S@sQUg>sR!_NJo}fv;f{M2 zU^rsOo0cXGPw zV+?%=zv*_VH=`SR22JK^`bHl8qJFwZb>elO(PL~P{M0o&4f|#A%6(rlU&V?%>VtUq z{#7Q8dwUm8G2b;^nw-ua+B9_m_l>2r9piH>cQj|XRQitX6EQnqQf#V^NMDh-IA4iJ z(+5Ct1v5N)9=jM6zTzbXtVavgg2qW|eno+r)=;SD{~{~DFKa%oxA~q0#r*)3(C-|1 z(iWkq=uDqxkBe)Z=&WU)dW-4D{VthBGmRcmP~7`L2@P3#@tepSM&5|!TQ$q%-vvr2 z21@+R{7Swi;>oRqiLmehHU`ysRe8mgYKh0E=8lz)&K{XLtap)E8%zGJ)&>LEK-@DH-D6uP z@JHlH*g**&{S)}z`Cj-l@=1P2uOVB%o%)m~-s72T?7R+?np_Wx+iB_jpj;oHwsh|W z9>s`J6(FNV?EuAHZ|V1elZZcM>GfWZnnpYXP6uxX#ob})UjU1UpRn{t7JAey;-3UX zZ?ivAOdE2qRxw>oc_6(%;&ocaZN zUYO3jFipi4d(@Zl{dd{xjw@a)eUI|zQU268^UrABh|`Zw?=kH^u*As!11RpxD@?hs zfKqPpk|~VO>5R{5jL)euKAl}gJ*GyGBYdN-x@bYGO#W_A+-H17?r~7deUouDD7vjS zd_hp$-+>bPyrq||o2-^qT*z~lnd$qHm{`Z`bMztO0r*EP|K}=58=h z>56nk`Wb1^6tWev-Ewsw;2LxxZCFMd$~7GjOg-NX>h_VobC!Gs zqh%Tb#Z8px~OqBeQ%bU{>yaVAGPn94Osq%{3id$L7o4iLUj@4 z&ZFPXq2Eq^fLxB9opX{~y$rsnX zhxyv_AE+_;9|X@;ZYlYPEnjl8$@gCHT=~X$&u_@`g|?V{w}a=(cZ_@kmaozN2qFsV zdTRT!Uh>Qz*S-Vf{StNUxBS1YHRaB{)}6nY{ENoT-%Ea$aLRA_J@QZ$Hy=Eg-u2|I zw|u$VOumc2bLCq`KELIA-R7IV{oMH!`Fxh|Bq;8U^T>DN#vJ9{VdNhN#l2?ZW*yG@ zn>9iy>!8wdvkos_%sjyU`pkII7%TUY!eR4NL9XRP$dfvb*=y^BOU$0j<)F;&*Mg%B zro8A*Q{G`vLSF-g=NF({CkJ*LKD}<|{z9JF>U#^;m&2s}3V-@;bB%Gw&o+-9LN{?E zHyF9^Ycl135fq)5+-TxWpu{~T`8;djIYY%vRr1Tpu?0n}n@`SA1&t-DXpcv&c#B8H z_|3xtWV++pKA|$TprTaG`epj5tJtU6+Q(U=>b1mMO?f{EnDWO!2~BRcdQ8h#(;i4) z;}#U~?1uHbkMzI6pB5d2#y+gXX9ZbL@Y@beV;@E0Tk-RShTQCUMjn~d$)k{^vdWO_ z+oB3D`~0)Vy}qrdZjT{fY}ZUX3rhWlTTMMCg-!jg0wuH?6rRXl6F&$_y!hftYVKpR z$9P&hQZ$@?{wVeJ@X<&)a3n{b{}G5X5#&L z^X>yWbC2hy>2)R7@p)Xwr#57+>9!(&C~4$f)otW|4V2KUpw@3G&nI{%V$Mrv&L>{- z3s0ZR-V(ouc&x|d&)sk2cHe5`_JhKc_jWU%{Q5RSFS^~(%j&19%H(vOQ%~bL^%R~{ z=b!PMdWd_yg+*BPo2&4wqvC0as2v1HPG)-Gwx?EfZq=+fbNca z3UXCJd9JKBeP;iKJ&gBx)N%ZLekUjV?znqz_%zQH#q~XC>gD7gCO-51>HHaSxrc8o zR3oG>dMM445w|>}q*r_(&Et+|?akTe#O%$%KlL#BYu(Y0z2!_A`*kiJkCeqa+U)V@ zJ#w?R#XiH(BOdit{Gxti2Y1|4R-oTKDQldp%GfU{i_eOAWKB$8E{;!)O=+H7@0nYu z=CY2R%i4S{*TT763+Fb>Hhb2rZDo(pjFri%|1pmm!uLyATBkxC{|dB6o$?90ETMk@|DctX9nZ*{rHB8BN4*8_=;4lAJ({&1vUf+B`duyfe}FII zp|ZX&ML)5j(B8n$7drhtJ3ivd=b1D?O{&OM`dtZ;ot&WZLX6j^J?aOwLu4lROtN4bo+A9yLld~SbB&Fp4c|4JFR=2=;FL%8p5UI~*aX#iG*`6`=Bd`h z`D(|ZiK?o%fal*llkv<_o||W?n@{nRoD&{(?XetH$q$Yy26NS_!+Gk`L;1?vJ5kN! zo}-X^4t@Q7m3eLLMAi!%Subo*dS;@($UR7JAV`3thj>Dn|XdVDC@`ff|B=|FL-p`29VW_dc?+Oe9@%mKX2re zd3oQdVma@XT&mk$=4WmlRN0<+>betaRrRrT>YAgMt1AZAtBS){s8xrqRG0Q%rIxgB zPzz7qt`;V5W1sKs?7O~AE%e-~W>Q~~;YS&5lgE&CnLOKK|D}=rUiL}~K9L&}o77{I zo6t?_GivKN@J*xF%b>V@-?DmMGg+-jzNjt<{Uht}Z>ZUxuVy?mUxuuOR8(Y*Sy{ga zhiv|bzH8)7{AXjY;+IXl3zR(e*}$kCJ3IrBz21>mJZco*{(i=e%xlR>I{tHLF^9$) zUE&}A0rP;+&^bFUGE#X+cgro)?|01gtO%8PE~zN>%=MIbvfryw)a_2T8${2dOHJRI z4@wP|fkLSOrTzV&w0|8aeWwX@yP_YO`u2k2E_&72BLoWPAOF?#qr^$mPrmmf(~mCw zvFS(O07dR)KQrl@Kq>cnQ1bsHD3bpb6!!;E+gHDzHJ$gfrm5+l;Qh%Zc{~RVsac+u zRKFWVz=Df%)kVFlQ~@ut6^(Jt_;;@T4;j0<<7G1_|4!4_e3Ya6T8o{VX}jj%c+_+F zWgn2d2tI?RGI+`s%lkbSv)8eZb%|HW`#sW+$BL6=?!V2^J+dOlcgbjI4 z)qoB4Gh=hFD*M?54d$7#yr=zQv61_8P~0UYCO%2GLt4g#)JOFCJnz!KG&4ti3*7~U zR$Q9-{;0$I#;hE5*=&Bt2(P$FrQ9dW_-3D|^Z|3f?C|b^?;yUE_wEJF^Vg~DVNOx= z8}zfAn8W)W(*KCR5ne4fmo{F=^Gd5@v@}Qk4*w3)1wA?RIoio1*DBUAQ{(ydCOk<^ zueeath7z2z48W!)Gz>qgyg?t~`yy-A+WVe67|);}nq<tWnWX=jYZ@+&vc=y^lR?&``}^p^QQwc_cHf2)g{%P@MDw!BYH- z7iRJa-I2d6ONOLLU)l$qE-=*u`@eJ?OuVo$aXqtA7ge_{Lm(pZGL&gRgHOo%f#4aX(oB{cqE>XP^zO;Tq+leG8fo+E$l-uR~jg ze(f&W%+JFE&F7-6gH|MbE_qGDhrA&dtruE9v;i0GFtk2sj9`97HG4}bYSn6bnE)bE46n-e!V@%AjIjj~@r`o>6C$i=|> zXU^%K=Ac}5EYjz8T2j8v=RQi8w=&N`27qk9SOS zy*ZsZ!e=DD0GI2mbN66}LE*?~9yuNUK{{DPEnlb(6bM^DPO&Xm_7Y;vwVQWIU9B>bbE5U{uV}S zn0LieG;;##wrHrk1NujxW4}zDcA8U8H%H^0oEEGPG2WdntD298a5%4=Jk0ymDY?2m zc1Obd)l{K%hIBWOFEKc+-UD6Q!U`~Ce#u&yl3|Rn(YjC24n5fIJ!K7 z1)WhLrEYMS;pnN&wJ)8r(>d`~%=;AdGjy=6u+tnxt6QNpoxeU33(Fb999%0$^gd6@ zF8LjBILkDgu)|U?RCjakNz30H?vj4fl%`||w#D+!RI05x{hlAt zr&{%sq+H(}?hfA&j3jGAbhgK!oKZ{7vN_lp={Vprx4Ogdnl{orO8vbfk&jv=IA2># z@?OJNYdd;s{5?5dS=OS}!C0^*(!r;g!n$*Z;kCWZWuhFY&IrzEV5}EN=SX*ZbuCmg z?y?$(H_TF2a*|X{ikq|aW9^aFgp-s|vkkxWTQkk6t5e+LHN!2W)^v4u%15R6P>39! z-6}nU)=p1qY7nk7>~C#iP0F}2HhI?Yrh2JXLtE^;^#(Z* z+S*h!pDZcNh4Qs9cTI10Ninmna5tpnqzo@)c0{?ViinJk$m#AWe)k37IjDNbU=qIf zz}`)|Cuj5?+Gblyf>W6tX-aD94MMY)blQNYtUddrJ(()?T#-=Kshno=_$=`TMrzWg z_vmlVtXRD=Bwuv;X)aT@%%#zea3hUfmE@C_9A_R@U(2CQcP2wyxyr~j>N9z^oNb(Z zAK{qy9W6KWSxmJ5oRhMXv&$X2`jq(^P`HM#Z6t5tx7T?)JHp|ZdP7RoMBejM)g949 z*cLHUO62o7aWa?>ebv&dSLb5I_yNAk)h1U$h7?z?Te_JT4=CRmY5KzvbjsgP*wLMg zbtmrLL=K*j`h&rrg-4JExvclw98(N!sJi z`MsfN*%3j>2I{XC=hQ{{WL$SV5slxVmg-n-YqaYIwPivh-{(kl$TGs5mCmkOPJKAM z&qgWzlWJ2W5ep_;+tajra~kMniDWol-N9v|rWd|jtTpM`q3C`gCADVhToY7P=;rQ3 z(r}1sHNCvaDc>qEB$<_wv}nwI zOqQ)ZN&luzy8_!Sw%V^X{2l$b>m1!@ zd_!%thT6DEjI}0hG_@wTl9BXZEA;KyWVKVfb6Rqj?U#4EbDd|OcJh9Qwr`K_XRbf+ zCwJ>@lpna3&Zw&MDti-*4YBc`G%}0`F>T;ldeUZ*!i=psw(P3GOEJ+_Ws$T=;m#F)!OeKxqdT?O~|WEhC4d` z4B1RWd!47qQUO+D^0(=UpEzp))QFJr+v-XJY4^+ouS+l=f9FpnuaZ>Nd z3v5gF^4$@dH+mot>7uw~w^HvgbQIgC4w`U;i8jTs-RKsI23q1QCR+JiVj!5bANdS~ zds#lJI?H2Za~(JF{ahQ=U1^!>(|J+>Dw+(&lj=Dei?EXF&!}^&eDju-QeTG7Ji|6} zML+dPoBQUde4piR!`a=X>+xhxWhsMF!%qnZ43ZQR@nzown zvO6Z^j*AbkbVhr^+^b~sv~pvn)Ww!gT{aHL6^M27@kG8{qc$poXQU-e5)yh<8}nN$J9~5_{e`ALEGS!0n1bOF;Y)nnb(|sF zie97wrX>Sye5F@$w;PalwXt`rw*7oo4mx&c5!A)WB2&+hWp&}3+M)>)gF37*S`W(? z_V~J4%!xq`Dfx<~q^?o{g5iCUbrtH0vulm=@o=&`PM?DdA$C<1KrY|W48*7#bxbJg zpa*-BZ(^^jz$ENU(%5YhS8ASOJxRactI~brZb@&A9T48)GwYMG`)$f;C!Y!!@zSz< zd}=QrT9OGPEIq@C%ja>Ut;(+giEz7^P?{jz8>at3VK6h9B>8w4zj~uRs2g9aWLJ-` z@zK{x6%8+6_@>hay11)m_Tf9bJxZ++8?i$|F}J3Y!d*R)c(hBtZN{5H#y*zI$?axg zG)StLKwX{=2f}qN#wm1`)ZW}QJlbNswWJt=#r1ZEmbx!IJBx}yF$Q5gKh4%-v5?EMtXq68oi%yJI z5kY#pDpS_z*iSbHFT1;~746bRCW9@~yEnMBvnet*Ba%HbzLkByLpv)@!wve>^bN$V*Xn5r{KTYKEBqqp*3sUF88g! z91C5(w)Kj2;kJr!dF$n?Th^>zQ;vPubJ-Lfx=iZS;4$ojP5SBChbxjuB)%d@+ z!f22L^1I<1W;DZMYwN-iC+4*KD|mcdEm9IV(K| ze#M66UF-Z3bB4ALOyD=cBWHns7hg|!6s(zHcn%Stl2A~@PNa3vDFB; zWI6gk_krE`KEfm5kMSZ$K7ueCuiG8`+*;oMheys`_Fbh^bBw&;H#hM6F~V|A@;7Su z-GGF_`!-Y0Tc|JSoOSFz(^$?nehuFYkKmH6)Q7O(r|<^}4}&wWqrC~c&m(r9RV-%~ ze;q#pk07tJsFQ>R@5jGRcmRBP8~X-v>;ryu2X-SYX9IuzMr=x0&hLF&0NWDo0|&$C zmVgJ`z8Cuwt_Sama2`2fIWPC_PJVMEVemH2m1PT7pD%my-Mpbh*ax;f#94RUv<3L9 z53tion5`%k;T+XLp@Ut2hyD@Bb=4}6y*}G2Zx@f{Rj*G7%%NO1}=HV z_J45sr}+*iJaUHSkN<)G*$WSN-%8EdUpC=>@cTAA3f}*Mp$~w&zhe4c zGx!_4lMtIXna4tH(L(m+%-k`MZn}!bRYnco_@*;D`Phn+QKR{e+lJKA-VyMcAJC5p%Nd(bjPiRL zp@UO@XzE)8-i43dif-V#SLtJf`@!GfC9j-c89vGPK%vJ#Id4+vf;B&8?)_`}8~A1X z5Men7^1=TNA7MH3vHTSM{B7s}K7yAyWB~jEUbhSQ3%nN|_t}ne=HnLpR_KB^;P()A zpW`U!JHCwXgD&_w{z1a(C+RaB@DSMlmLYz4}BhknBDa}VV_M8C(QnxP9G%;C4ggxzNy%9)1; z^E~P>biohcj}msDTPWul?#}n9pFtPw#H-uU0X#O5-)$2f0p;Am<%Hem56anupPuGX zP0$79T)`M&_xXZyCgIcggU|)TMf~=ju;4!YQNr$X1?3FF<;9c@U9cVhGGW0k{Fv~A z3rakCEX(Z4q6`b_c=E&{)aKT3E6ymuDou-t)t!0oe5UO8JYI*0REpvS-` zOQ|PeIZN)Ph2$kH=dN`v_9*{d@PN))YJ#^dp)Sw`AHqLJSk6m3y~3kjAgoq;)bH_f z=B8TZQLp>Z;UMAF{8k*_Ojynz`$L6C4G4WLzbjnFZ+i(Bfxo_j--*5h`jwv4IbiN{ zz2xk#;tkjw9>FnuKViXDS9{bmgngi#;U)8);N#a|Tj+8=*MpVV?Vache%sHvEQCkF zZ&cBj2+LVm$!g9}AS~xw-Moo12+LVjpQ%B&cOerzvKgBbmNTUu*+N|i%Q;c+<_Fs+ z2+NsI*IY|qIz(RuH`mcu2+NsFS8hXRp@WxhM>oQ9cGBD(=yrFGJ`bs|9^D8Rf!}GM zzJ%r6qR;K-x6Opz=MBj@L3fV#+-C*J8A2!V zeeekW7JrzqdaFlW((F-#gx%--xX9HL|E`A_(sBlzrcqG zkAcs(+BO1@x1$4eIa{Z!!=p|ScAt|Y=jS|x_uZ4D@gsyc5f&`zL_T4`*GqGvufnLnbU*?uTNm-d!d8Q88d=M@iFLvKfxa)Ea*v^uwbp8f&WBlcJN<#MoZ<57dl+~3QFpNXUiu7SKlppR^i_2q=d1KnHgqreLAx`70w2pg%lsT=d8&2D;*D_2Rwqk_zi^Rtd%d|>j=BgRgv>n zK8J6IF8DS4+X%bQRgv>nzKOpFy5RTl1BBh@s#JWyqrQtDf-d+f{sqGBGgIVzl@0il z&;_r;oUb%Nk<<{OzOkAL#0X*yBUgop3$)=--;~0N6i(?VtQ2~w zE{dF&a`TfOwFkPOoNLlc*nPf<`|J}r6Q$xKjB$7bEAcN77WCtf6BaDzToRdo+-IN2 znJC}DpM*zH&MuLUXt>WXai4D@=bk*vxg-_v2!0V?M_BNA{2s#Yb4%omlixjq{?G-V z{wQrgSg`0>=5)e>b3bOrk^3AIIpd^qh-(u(g16vbBrN!G{3zjJ@W{taUiY~ra-PZF zPcSCo5lrIezmM?(e){jRKjC4p?>THFbnu-=sSDwL@Hd}jybzWT+r1q>C_G^FdHMt4 z7&tg=#=Gwy(B}oNi4P(Z{MCyxKFRw{)-CwY6P7bLp8gi?LRikv_|hx%jrU_`&^Zso zeWr!`ybC!8W7a7B7=FPe_#wiA|AHSTESUR4+K;fDM{&i^kxSSQP95Xg^bmB=Ir~BI zo?mlKgf93Md;#Y=jDU^5;krZEecpqd`!Mf67=zF?65d4E2i}KoBHRyN^;^ao;d(Id zcV-^*g0I-{C|LP>V>dsT#LL+42W$Rd`0K%gcufa?`bWb*2EO)&p^t(8rkHQYI|jby zv0?DF92*8-n_$D>f8=sqMK_g48{j3cU@+f=W1x4U2@7s3;P>#%+x6g=@G{SgfKN{1 zo0G^H0v8q-gVGSyQ+k<0U)>9-nH$qu>kEOn3x*Zn_B%gZ)K0YRX~e z1aM(-j=rWegOAy;oI|mzBuCAMrwQ!GOI;oZ12armzT+6f>-#V8ZX3P_oG^TqrkHsd<6eE;m5&}IoO8qOmG`sWY&WZ<3-M+ zV7N3#Eqa*sFz6|xzVPRQ%kaV@xYLFk!4KQ;HWz&#Mke@aywHz>W%H2( zeLfh&3%w6KYQus#7nnB91-IL9J$Mi=ItcE%knbKMrwRNzzLBt;9WnVL!z1TA?6ctk z@KwBA_fLYkUd9dlf{Sce@FxqXH*`7wn7hUQr*1W+`S6AKsrVv%4L**afxjHT9ba$p28(Y2C*%L({wG<0YaP|W z_hr_~oVi3lIfKq|<@y_p@=ejVvs}t{8!CG{JG?!-GRON)S1(8{4jE7mXd^1^6WDA>W#<9TIIE+tCzL~I}+ig-c>0%8&;jYlntvgblR{gRXMV6SmoG_NOoh4 z7x!^}gI+XPI=F1mH@JS#Ke%kr(tqUe zk%1$FM~03ZJu-ac*pZPVCytCBIdx?0$mt{M>4K+=o-Tdb`}DG>eNV4{8bz9!(}DJZ zzJWsn1ctd50s|i&{_v>}dmoQIe)RFt$BUk5dZO=%V^0)3>3g#NBg2D-o`U5s?$5Bm F{{bcfFu(u+ literal 179200 zcmdqK3w#vS`S`sd2?R0Drm`hkkhMmQ#%eTLL!)(is!ive#5mn0%Wpthp48e1TF5C8aTDi+#K>dF5PhOVCbJXMwM|u7*_POI* zvm(zOf9d=M_5Rwrn{KH4%8mYOzw-63-_+p$>UI9Q#;^MqeBB?KIm>_JP4lihJwN}5 zBDd%Tcb@g! zo^hj&@_6z_c|04Ow><~xv{yX`cygYJ9_>V@?jAo-6`=B$?WtLs%;e_fty4(yCh6Qa z^61$X8|67oRgmrJk$d+jPw{va<@|Vg&r?ril&3c1_3+24=qS&9nJ@jLJpE%bUkrUd zz2Um123}Q+p;OWz>ATyHWJxF`8lCPW^YlyG z*Ajmu_xnbqYdgKxNvQfrUp=2C-RcqP0;k__{epS*6il?IpLj{K?hxtf>g%p0O2JMr zT#~dND&2L9ZX#h1(H>x`(rp-#ZmLB8|M5?cyZHzSPw$!1g8cX7 zR*+lbv}e52p7Wjdob>71BW1^PfBxy(^EIbEvqrS%a;H6Cb=tGgY0okzwAg9S^_lG{ zwjO&U%fle(ow>h!)In7}k^jHhqdf0xOLD)-E5K9S`}4h79yMOpt*P|w>GZqY^gl_a zZ%m~>HvBX*9-Zy zeeV@euDRRVVB2=M>=`}YP~?x8ox1ra-ORInznAFP=F)4fzS2cqscu_f7Df4W^F`g< zs3&IS(Vku(DdXd$h$Hm)3|{HvH#I4K6Giwl3((DrNsv3~v&6RS(zWo$%+jU(R~6++ zbN!_HwvdWFzv#%;DmCCLeUf!pL$KSnD z*>3As2M-C4dnI}#JZNR_cw*vN(m>rlann(w)W9fcDf&>ZLrwgYYU`$vr?))Q@HySg zva7r+^_I2SJ38NY;$==7n1PK(i7ibE2$D@jg)k4RSs3}Q=ngVY?UfJ zrf$T95_idCD8B622zTqve3fxN&$jOhxRP!@s|JSl))REwDz`NJBhpy&#fX`un+FW@ z8Kb02uh?0Cw2{d1>hVhAQV`cGcGNG9m{)oYv)d@yX_%XglJ-bq{FjW1jrG&?#8g`z zjsjQa0xR1$Q|gSC?4FSrUl1;9BfY-UqW?zrCVgP1QSmJ8i)fW?k&2CmR@rSmaI}Eu z!o0Fw5i>hN(b=8WoP9FxE$gH%UfKH-m$EMSC6Ld=;k0vHf}Vgg?%g_}KoFZz0gH0%n(WL2s6)tWA$+D|o~y zco?>)>kk&noQVy3>t;$z?6zSIc+e>`)qs0cINg9dG8=GHvH?$70HL$8P!r6E-CpP= zAVP|#E?YN0*O4t4J7oH0jIWx*TdfkGz_v>_2W{WvBSv|awKW_~Df6Y2%pnG2;Coa%M++tNpAe5vhg zA16>~oU6C&YXI9+R45T(`u@fCJ?KQz_RK85aNeG3nA5y;T#rw0ur`d7fuBilsElsq z>79sy0@w?^eNe94Um^m-A_VcrfhkH(A{_+)Qh@>8EO#Kl_y(`epk~ha(%(fzg_q57 zJ8K5ZBZ-@9gJ!Q`_Csrg=u9bX^af-5vl@@kW4F!sG-m7e_9RpqmRQqhBLKLCt~Tbz zKF(^G;G$W`+@iYs8&DHi%ywU^;kA8_Id3n31{6vaexgNA*UfuZsvhQ*>?Dps2IiGj zak?a<8WB902^^H3&_`XlKSv#KVW8=q(;XPlZ!m8WV;rAO)=ok1;MLa*0f269o-G|u z%>S{z^lepZXIN9rAzlq&Y8koQCHGT$*=9W+K3CWu0HQ9TUZ?5ioT|vRqh=&>vh<3s z`ia3rz>be%xJXd|q0NF3Y3oMyZbuB#V^25JD^Ft{Q?B<3-bBplg?eIotue(TeSbhz zZ+=@qU`UszOV0%a)1~jRB>?{OF2dNpA25GpM(1hkcK0r#wAizD?>woeGkk^+pm5o~ zNWAgn(27PNq_7Nr?PN0hwPm|ua=JPDWZm2fk%yLMX?L%acZnD3XXyum({%`wI1zJ&k^;Ku?4x7_t3cU8@XFSOL6dpDekEZ#ZK}Twul7 zKyMy5FG1$QWBf9ov1n>wo!=Q2`j<_$W>ror_u&(pQf;I_5^KRc%Y9ahk5dl ziV^CAIJgA>r>$oAT4-;p^~q=AjnQ*1?!w2hpUJ7e=Ij z#4M(=w}HTgg@F}cG7!}&HbVR3F^8&2uk(y-^Z`?WD_$Ucv9i2mYpnfdU6Hq{c-|1< zS-3NFOm$yGX@_228oVqxXDH3GU7AI0wR9@?*_BC}b#lmj53)tC8WV#_)`hy9>4hkKb6Ns3jjD!v~J0iTIG-!78{$BDX z3EEg)C>o0O{|QC&2eqfRr_lW)&{vObw<|h=T6kaY49bLny}3b2RPou9JHlf;qS-{p zK;|Y0ooAYj0cVAnw6Dao79}D!JPZ9Ub4I4 zEUC(Uxi#Vt3wDU%Fr)7`UQjXdH zmGo<|=cWH<2hu&$JWsp(9>PZA@;ss6w>Cs7Hq}pxn(YqV9%BW@N+D^u^i2{Petaap zc!B^i+33)Ua);tk*RpL6olHfRJ!8a=1P|t_EYQlyii(d}ab$My3Y9khc;pe=_o3}* z$49HsI}!?-`}$6V5;qoVPetEM4f$BQ&m5BR&V*mx#fAt=j$Q!O8l$LI&ZwbO>w_!_ zr>WL!D(q6N7gWGhRO^JlLA5%dS|5SlI;sw1VtCQSoDpt4q+%h{GL3~S@osuqUTzt$#cSVRM&?-7M<@| z+xHX?fqk7>4Fwy#YATqL*LrM^#5V}Tc68xPO6I~hWnaLgVD7gB-yDO&DuvrStU6Ul zCP~ihU|K)c&D#+2UNkBOZ#`S@40_29CAqE8TQ?pJs6!?SGa*Gv5sXD2OTR#cX{dBE zz=g&dC4&w5D8yOVU~FF)xH<}5{C01h4@T9IQ%-JJ){qi&Du%s8YiDGG@C8OzH@_o8 z8Zno;mr0kaBq*%8whGK4; zkj8T)GWl{CX+#gUeLn&8$@VDf_y@`jmEU@^wrr~mu~5Xz1S{b|P`4hVq|G6Q|3j>i zs8+;KTYeWI5oAB>P{VHPET&Gf+!Q5j?8$6SXhmu7ULk5i3ExxHEe#EXs~+J*M~Y0V zvOx_kMGd=!8n%)?W{pZ#(cnL8Pv3EfX`>)_w(qY89P;=9^vw2cm(cIC1PpZ-8Hr#9 z+3QwhFP{mPX%G4MBMGO?5$~w5YY*vB0qh}XAmeN6Hua5Ki_#hG7sB_%3qts=Dylt% ztad1Rkv1eWjxMuJOn)7IF~3upz0L4>4kF+7=Kp6%cw^?jZhEjzyqGgl@6eI3!bR+0S@xIFX-F zVk%lNi1Hm((I7@VZP^|o#MB^7P!uXcB(@3&VU=T6;Hg+F5Bw+;zLW?tHGBvPi>ghs zw5h>QfL2!qB&L`LiLFj{!OZ=|147WzO~u53_Ft^(>>11w;@oA;L1D+Bqgr_(FHx08 zOVYlh92*W4^bsd*1W6CQ5-W`uBoAqiG#O9Fc2OtoJQTVRMPz>lj+_WTVer~-9c}xbR94q6Ypm)@EnRtou1p*3%d__VNqW%V z2lu#U?dbF|<0GcXS^d?@#h>TvVp$cI$z^OJ*j3Vc5D(>9WP^Muv${@bs~BH z^3;x;GMMn@a`lEGI_v+@v{fey!mu9lKa=G>o@>bI7BTDWcJuhPigrjhoG$%UHGGpATN%;$E*VJb+ZpRYFA7B)u^x20^d*s7q}o9x z?#hsUZSW%ov3K*oY+LX?W7TT5TDB?|qhMlDQ4>~B5!lAP4mW@#6i3YMs5DB_1#Gk& zJ)ISzpMs&WXKaYFF0-rrnycp~$4<)Mh$}(n-{F?Z4#-kj4KvI7!E1+H4~of5KnhNy zhAOssil+fmX2?=d^LtrCg?Cb9#<#vdG9GROG0(ESw$C6tgE^EU$mnLpgN#qRq7#xn zc3YlD7BOl7F^d#KCE3nEW@%K7ED~6 zRrZYS%c5wRvb980MYx}&tGG)ctJCGJNtU;knp-7MM1nSXsO6zc9{fCXFW`R8vfq!( zU3l#=Ukoj<iwZHJ{K58j#)QhOWN*uL6MF>f7!lJplH;X?r|?F*`y3>ETpb#_ zwc^EXA+0#PmY?WqerB%}svk(&KPnSBXN0tZ**ZVbYJS4=wSvV>I7|FdRFim%&^*23 zVBurLng?Ut{SJu7YDd{WI2dYn5|2qCadi?O1bS8Nu=nv z>Y3LfYK+@ZMXlDH=VY*BonC=Uz_2?7qx49{VEv>tgnGw<*T~&<@O;}h?_DNenVqt? zlA$PAL9Y5EWQJkAaGQ-JL|z}qQfc$upb?ct^7oj$0Yf;s}ji}u()nL|F&pkGHK{ob)=$QyRXf7+7)U}UB<%SFX%wo9=SX@9l!`;RQRL&_)OQFD>%3ylt4B`amxW4@UdoD8Z=q-U}kx?MC}sV}i80 zY-8CjW1wF*-|IUzm0E8hvx;Nh?A6WlkG==mNiSH9m|j?ZjW?o&;J!2S><4sK!_n--k%D;4 zg~Q^j)oS@)1dET-WCTl-4LUd3ppn$l*6C=;79(p~u~9M2zc@#c;3?Be)l+GzeTj<; zWhkl<%x&L8Ql5;79_uZX5vx|iFqr^!^F&^)%&!VnXSUm7S|Ktefpg81b9OQ5HrttMzH~xDLniZp$P4$iqa0% z(;B1X83(@vV-${AAlF92+-Vg<(!{cp5H2Z7^YSjoS}2?uy&2wAkHzqIJ*XAe_w#dm zPgpCi?B?gfHhyNUO_~zt2Cq`9ASrggEihYDh76Xk0|QnbTbVI1`#eVp$j!ek$*EU^ zjp8-L(Cm(wI9%XWRc>cXful zn~63|7H4r$!lus@tc8|0_D2%)5&mxk&2n(*P?E(s#!&DL$zpKWrBlkyHLi$%O?1*j?6W ztSFXVk+m2HlPqIO*rO%xgv1(18oiA?bcO>gje5#zbzha=v4eSkC(iHmWbASXI<=I_MJz?4(2q@?+iBy zP4Byg>`i@_2N!IS0ag;kg0A$yt9-$D)YINOrmRcpn**=zd@GT2Nl913#9+y`hNFTN z+ZvCdI8fTt=jCC%JdELCG!GPC8(&P zhaokSw%?6Uqbt1y^h7b-2p@2h+RB|c5Sg7`R zAqs(r7>4n(Gm5QuSW$6{!Q??P;0|<!X*8MK(s%K|tL<;)F5EazxbO75 z3O%C5USk9d^H${N2BACK@fX8C5|)uCR<~I^cZ7as`iSjD* zPiz&{&J%PyY5h1A-Y(uH?KTV>In%m;X&kAj^fu-Rcw^s%a+u=Kc;pf(2-f;#&70Gu z6^ugAjdbDJTQI}hcWOGv#VQ94plL^o+i$s6AVM*tNLp|=R+(3*+W)50{;|^jdtZY$ z^!|iCM#Tt_YP?~N{PNpJs<#&2>OzdKqnjm8_8PddIMAiGQn1Yvq`C8Zsz@#!4* zsT?%mFpW22)A9;slue+b*qSh=f3|}{OE(MiP)k?v8}#2ymXo#z3a;YMRJJFA88v>a z0Y=6P%}wD21%ejHOh}L7{1i3!2A3*1EWBW&L zg(3j1ibi<#rl3Q|%62&{Y(aYpX@jhoQMWGaYlhb-!LxN>uaSKT2L8YI>v87mh_cM7 z6N42~3mYfIr{^I1j!2;1&~Ir#a6wzDs*ux&pwotM{HUNg18Op(FbF{nmbDM;4Q5|b z7@vTmuwe9mnrjX^lG9hjf?@K&qM|N@fpX#Ooncu)U`bJ%y>?YJ^)`7+8zk5b&h*DVadkDJER%GGV85 zWOkEAUgWj*?X$56HD3I0TS4kPt6W-f|BzPfbX$Qu1bH>Gu@n5ME+)iKf)1F<^~aMs zNy|QMSvmTW(8?O=wbI2A5>g!(fdRvPt*Wr|Bimk}fG%U}Oa^BzC{7!1X{Os|7r1cN z*5+5U8m@;TM?};{B7C$melD zY+oE>W0Hy|c#fJ^`(;iWQ?j*X*nd5t1)wvPTmSFrsKPk|M~}xWe}R zT`XlH!S&LIPvWNtu)rEZ13R4*q8>eV-Walm$`NZs3o^4BPSVZsV}#etj3g=xn{W3z z8_tkV+8nV&DSIbMYZY;4_2`hv9Gwq0P+?_-S#+mlF+#6G&Z}tx0;HIPj>KC;I3`M* z=x?t`58zUk?K?8rtE2x-uSyWhP2k}O2~BAs;SLyYY69p}MvKro+Wg~6P^Ay0mS|km zQVA4B?4VLGLh;4hktEQW<`q@-BHDq9*tnw=e?!$51^?MkhaQIcoKf=Z5SIUM)p>Bk zNKD)WQVhcX1t(VB0ZwEiZh;eC?XH548Jps6vLF@v8TV2H(77$j~SCd@=B&$4_HzTQ?h5^?iv&k|g#Z zt55;qBr>&3Qj9nDp= z;z@BT`2^Y9A@Nv*mVk?!6^^hJWuBk^4F^vq9cGjjdBDKn;KE6VNtIy{`sgFE&>g4R z1;U=_w(D>RThbz=-pC4L6$&e>P}q+ms;oj{z!psEOk&bX3NvEwf=S7-Xq({3PTl;o$m6U$uV+Sx z-c~BcoQX*{I;N%VyG2bFmCBSy%&M%GUQ{S@C(}I0LtDcUU@oguc6T2)GVN8sS4+ys zM2=NDmRwg(%g>IR_@biq2xuPqj_wlH*7b|@h)-8^hhgqT?QgO!+9U?q@Q$*!_3&&S zqpv9T5?;1XD`43ly(E&jD32s~9EO*R`$4Hza1m4_{D2m2C%rA)sH{yZmJzakuqFT+1|1_f_ zC2QW9UMUXSv4jJpo>?lXs-U%;^UXv(6*U{;)UGaJ4*KSyeS1;qnk%X%=}gg z5p`Y21J1cwjnlPt!Q<8}X}I9jjVpr3HI`#rEykrw;eml%8nnMtDC+xUH(l<*F4Bnt zA5>IIjCeoAa3FrkGE|50brGxFp~tsdA>8?sX_)e&Ka(+z#BXLT>1GxJSA#G(hbX() zn*ilqz%O@}XTml>m-`r2Rwi_YKPOu?0Bv{-pbekPn#;+o3zGjtmKT~AkFmRAUHx_& zvhZoPPv?EeY-_#Y`s=R`4p>Y)b8CC=0!y7#5aCxjzl+b9;g771TP0a{`Zo zK=ec{mR(ik?+ot%dLHZKlN3rn3d;k58Xu;QPiqC0;81vnR&XI`1CsLsBr3SDN;ZLL zkW9*aSeWHAc`{$aHj1%1tI%k-Mv1H41W+U-vnc#Ply*&pXWArx#UMZu+G8|1^+MiJXqQziPhA2-oUn?!#Y!A(gQnF3ZI! z5bURaC(A>>%hx(C)G-8d!)e+k?w7NXB{!enb&-ON!nb*r(z-AbEQ<5oj_hw$~v%DyNuo ztGh|oaG4a$U2IWGJdKz;l*e6+(SItVjMXfWgR;(r%3JY>dfiq3@ogBROu$%A@je6=_b#eW_En zzx-N-O9pY-QrlmCsKPN@ZGZWelXy_b*-O8br6pQ;t5ja2mvm5hcB=BpZsk|1EQuUv zInniE5A5qum4DeOZyJ?%Nad%ia7n)&JE$r@PK9F!RpkXv;(qbVnoH$L_f5ZHE-X%N zx505;4Xuvj3d=F+(UoRkEwT5a|dSJSp;Z?#rWNroI zN0%QZm?-m54W)!+DI>^cjS%#biE;j;Uzd##OPEZrMdI_23lq1>3PJgFZ&V9ymIVS{ zU0T$Hq;EeEPzwZ&vs5#vB1{e8Dws|{3@{jhNK;Chl#^$`oXw=%VV!auY_|Lcuc3uL zaBcQZY7yQ;V0Ez+={7r~#UhI895}+_2pcPv+~!)!^7q{c<{pnI?^x!-5Y6kVg62md zdO^n;knr67UB(kho`P>nBL53LKzH}kSi{)=aDQiL1@q!*FJMWgXY&b5itAv^^jk1s zp;(F%voRh-1J>!c0O8oXS&h3L!AHEc$|L5DI*ND1Yz!DB+YNKG7~0ut)qViBwb8Lv zce9mSi|wSMXyU16Ze@F%`DU0CqKO+o^z%PYpA7TtBB6PQTl6-Bm=O;NJp7{fG5v7e z#GG13$5n=gecP3e+r%25u9$~mx2d)pu}(%8qrvtxpgy4ocSXT(EE7c`ZJnYhKss9T zPjVOo@8CdrDys{0I<>kWOH7KIzaUn6&xe3=q1|L;AB#MG|VWXxSb$AU81} z=X_`3C~S0%iU~0w(+bP4cD)cT%CoEDWfCI@6Qsv_WK9q&0A zkhd9->kc&__;r4YhO0vk(xc(bqFQE6t)c`x%ooZKDS0yu1Y_WBBXRb4Jck%3D5oVWt@^_K;I%E%M#MZ9?3uN|5nr%eOe<`@ zbBJ{-g+Ev><|wHX%f%s6Wnj_Nw*g9GPE*trEMfqVe=rLsu?Q1ZRvI>1%L&X>fu0(Q zXvv%OLyi4;jQ!9#I{s-^j7DXG?C+71UB=G0BH6p}{TW@>hLWV54Bh~4QFDi5rgqnh zGVJ0CBxUX9j$la_?hboRO!8eQ8QQY%3r;L-2SS}e&3g>jSG@czU@xNg=2Cuc3xqXq zu!Mf!@_qqMu~nMWeU zAUBHab@mN#upI*kXlz&sh-Kv@tcg|1OC|%YhdRnExkIK_v|0qsYkw}1$%92AYM^7m zQR~a`kpUtjePj;r9s`veBERUe*MxofsL==(8Pe%c`9(K#_sn*smX;dCisT9z)Lw}a zRgbTvUBCx&-ffNPSChM23`zm3_v;iM>&t5VhZr#-)jMR#g}MElkx~F`Oir9dq(>|ncpluJ(Zr{lr9B3#U&;d3{@s=UuY}x`YMb{1Mlk# zUt+ZXri_UuD)0F#uXCnwVbZK9Qc;Ev8agBPa?u$_Cdkkk;}-o*%wpoEEQQy1DoY{c zUjP7gP4`@DF~C8?{3u$2CAZBOcsH6jdx8PI7D}XOPk#fp*S{t8`Dd*)djT(YrjWIGyaPkSe4hF9 z!L`hv$q;hecd;rKV{Wxu?4gah;v%Qb*(d$k`@VKL?jux!A#Vd7h zlws!G*D!&xl2xfuG6^z9DI_LUtggGLWw>!?eI#7YtMXju#EAF~WWiQN8NV(;c0w z?ld&7lW%1LDJ^RNB8D7OYHt$Ei+z|?U)=jsD3R@Jp-Prc?{sdrI=5QqwtySB{F*3+ z;nhaT=0oa*m6DiNW$J}r4({8m^um)QQrzDMjf%~p82aQzq@>4)eWVn_T$Ld9kx~o? zK{O^%57WpB(PW35ed!(BhKNJi1kk4o9EFUhMXod<8%oW8C(4AG5b`Mm{kso21YKzF zp$Yn-$hg>7^2GjyetAbxVTLaH2tLr@62x3^%gjBsDoL=#HX*jkLD4E%Z9Ttq9MgK#-SsVe736(SZFZg{Yw zGh8~0pnY!*HBMQ)44W!E^ZvFei<`x)iXd^noOKw*O+Q@S$IfG%sj0KtzE_mn@L6r) zv!0Q%YnaoH)|!}IP2uGV7>V#|dFixJI$o@(mvvoFvU@Rs8S}VCTQ_rJtT(G+k{t7M z7Dv+*%PDkHowM@or#i=Bu1XzCHM3<~=NStb%GiGLf0^5XJ5jZyAdgm=^z3&$FS}6%T ztnbI=UH(_oNv#f*Om$__3^8vAlav<8^1T{&HR#e5x5@Kf^?25w1Z=E*KazQnyJsh4 zQC>j&m~*!8s~d%AP5N-IgI3jLXt0aK_XRIkkc<0n!9VUj3cKQAYrUWRt;h4z8Gc^I zM+#zq^^N*1h8ds7z@& z@Wq_eK(zvwQ$ylToSxO44uO%BjODY|JzyHi!_rC^6lA%+(8&U45X_PmidC>P%w|#4 zOFWZ9@x>2zhF8PA5MOP{2K5yNA?%@C0l{St4yA`p$dqJyhMVcQ-(?RX+F>kp_``ge z2QrCExyrc$4iOg?hke!M5H${m@Mmy{JBk`$5onySh&zgEg+~a#P&}fVNvWrKgg$&y z3IwEH>YA-2HR&NDEx=Fn2oyvK`mNwgP7lAQdPt{*Lj+5yhj{51PBAA^ z4AAIz32fQxG7t_)eylo$)Q28C#Z`8)sh^?yCW&1>uEuY6E8}w?1B1nRH2sm$lu?zz zWFTb#8O#UA5;bh-zI(W#)0S-)!2G{6d<$Iw{T=YDzBoX-Fwby&%pHX8mE$OeFwG&+ zN-{U##_X4@>ixbfok%gj|NkaZoVME;P3Q-M@qdgITYlyggNs2XQXKb>G*bNJvQL8) zPrRs*qLIW3Ddd<7W^nLNZ@5#n>d^S0Cbvv%h|20=c=4_OPvgZ_M&J|h;@$rMFTVHR z!;A5MaR!wA6KT9iiIb(DmRS7eCsIt)*D?sj5wE6k;#-$~8k|`3g5-$SsP#`KxtOJd zSHbA~n59@VyB+hCI_zmKtKy2dG?3@;Q0-g$X{Xv8m_+7+^DzFYTm)vz{`oc-Xh0!a zuD_CADF#|@-NV`gT=eI&cqPj3B)pc?rT~s{7`q_=bVG?5?Km2Bp#~qdA&(b{8R*Ru z*RjWwoZaH#G+FONHv$OpraUfhXP{7+S7%_Nyza}U&mB`IW+@=Yzv0*8-+<(QqqJjc zIj*t!r%<DV!>mmQ_V>>(I@M zl-`%~keaABIY7XI{H*mAC0aQg{dH09MNB>1D5)IwpBUV&oO!@(EtlDP+_#axsv#TYLcHd~$Z6*Jb)n}k9x zeOuW?I>WVuY+ZRrr0vhTnS0Fz^j0Qti<;4iC))sSD_3B>tmVG1{~Ap?9tDNyW;`0; z(z&X*YRKL;lk9?BX6}!i$5K7fB4zNlI)emks72@uC5)_40-~TSVO~W7)=Px#oAAHt zkRvDnXV1>|iR2BIkb#UD<^_58y45fG*azvB?e`qNhTD_T=$iUg&!W3LfL^EZ9 zuNYWod+Y~l0GZ}hskMTZ)x^yW;$!eMf76XBx=4E>&FTc!ey^~Z($~DoFPS$}rkmCjG$Ja>vIWvi%kef;vhf z5E|3NWeKB~ulKf5fSx zn2OtyQP-)c2Z&NNf}%`wXT}eu2(9D;6F^{gTT8{tdL_Gm^UtIV>&OExQ4~VxW!6>; zO~f|Ma#GggG=|J`J*^^^NGVcLUfDj5m=Ppp;>#R%iynI#DXWU{} zl3^9sYOb|OYPJTSsX|Ij63lliJ63CakInC=~7vhIkB1wGV-4x`M*T|h`CRcyK;`2QS0@Ql7q=(N>0)f zS2NLDddMm>z|7yTlWQL0mciK~i7UNP&K7xjX5zf3qxc_WI6k}?DLG*5d^eJPz-WJ$ zg;85u_FD?U;E%s^vVXw^+mu&^I=>)e^JmEH!ORya=@4vZ*-^HEI>&!xbLh!LqyqL) zDPn}b)z4vFUip8EpPzt1Pl6$k0fvXB0ft}UlmHk44jA5! zq`&~N#ofyVET`4f{k|`oVh!`JddWdKrAXEhPts!@g05agM?MR>&X7X?h!iZ&q={R+ zI!mK3^ZqQ}E9$U(smxYZNoC$8+gF`zAA(J4zS&`lI>Komb6@6LMB1n3o48|_8i`xr z6ED*MMV8fkc}8-bO#>ok1uOQ0tn9DjRf?C$)-%vGV`X0qIdF&WE-8DHS6pivA2AZ& z3t;bZaH8Uc#v=^#ps1JSeG?-U@9_nOk^>+$IBxrvKg{xe_CW`s&A&3Mqv;9#c^cCG0=sMEF8bAC$534IULVCf9xkW*7egmK^n_qV7Ux%D_V z*;h!YUv7!}RT5dOwDkJ{e+f+YLxylq1FN+k>g{jBfDc~3QoQFAOW?mtfa?VOnr_F7 z)CHa6vE+a*>C&V?x81tGlREH|Z!=)#(yFNWk?|z6TK?FqfJ?gLzmOd770K~#&KPfv zx~yaWCCx>TmHp56$uuMosp!|1W8Npw*ld*S1F7T$Ev7yD1d=CcxqULsE%I6&o`t=O zEVHr<7h}mGhi5&f4$tb0;=5U~BQo^xtWvAEnoQDU*}e{qmRS&c194$b!;+~SGw{W} zMya3uIYa87Got>w#{anPN9K&In|jBfzBX$G*@B)5z8uD7#O!(X$M?$joA6Y4|JSmc z==Gnf)$jb*RpNjBKd<7NOyyf4a_&N{-rY zJ&wr49G{dfn$g>!1+lJ^k)v#p_VEBon&kC zlk=x0WB%+20GyZhQdDAgjYBUdPA3#wgnh0Nf|vgrQnZds6)Ol1Zg0vb6 zB~sp7+oZfJy;4e3RwJZ5_XggZD}{SE&$(0*9L{yg(b!I9G!BTCE6~~HS zt4PSI+Kiu}7w|2cq+XCAzYNn4%=7n%Z9ps`NzF?nkr8UbzmS7PO#r^IcyS`Z_91>n z50c$F7in3E{G3;Vju}nM{J#+;%}j{gEi(VhiyfI?IM|FGh|^0&)8kZO(PNc=rg z;t+{nHVd?}SwQv)1Xcmc1*g+9W%Y1Iq+ktB}jj#kM~jJs5EOJ^+!YQ!(TD@ZJ)O;y1Ug$}&CYo`FZCoV5hoHGivK3WV`obZzj-RT z&XcSxbt)!K0JMTnIT{NyF8|H*MPoUh`I+LSN^Q^It|EGvo92_vWOOx_D)RhZMZHdx z6jrJVyWdM1;vP_Ozf*B7E7ZG7nFnV6ohqV>h>-cmRp|mH?yD+p8F69r&x$xsLpYjN z4B^Q|lhl^|Q5ueU&)^)V)O^Ads#H!)CsAZd!L0>4JMm)eGB|aw^~T5{C83AjPOgd2 z!w`v%NhV6?Rq6XK3!8rlo$!LD=;L}eD&R3y`5=B1GGJJvY~T5Wa0l#IE4Ta|K+JmS zTej_{ogit)B-0iU>b;&J7}**5@Dg^&Si4smeQZX1Qaz>v|LDsNnJ@Hx`4IU}b@NY@ z{v1AkHXTxUi8v5m6N=xs0(=y>i_L4by33(KEMMcCD#3rFeRhT$Bo%7g3Mdnw*uqwm zzrKPgeqWy=R?Sex6}BS*aHm`=h2iq#5B_waEi=t%`kJ=irv!BBYedVDJ{<46mLPzB1J+dLbT$8#%~zx9!Y6yS@mC2o{5o? zz4*N5q>mL9UszHpDNd9zztrNpa=YMo71C8C^uq z8U-4)I>f25rQ)YqhD{{VW34p9T z7yT7i=hPqF`%6avtNV!vVDo?Q}zVCkSW{=Zl<;)9?-vifVPf^=Hm*7Rdq=VSeMe&8nf50madE%M z#Il>b0+Wjt*{b#x^C-u~CoU{BDzXeMg6ANvZ2c-$#g-oVlAl^k(u4B%f{>17*1jdyYRR8g$Es(;FB&@$J1tjz3FoI1TP=Q@*`3B$6Y#aP4gq*WNEl zIKtsfe7Q+JX?m)Pj}0o<-bpGF5@U8*3*Z?@fBPD0qa|+$ebHNmv+}3WPpS7^kY}6G z{#uT)^dmWuxbFO*yV3N|Dh6+7!`W!MMZ%Gie#6~pDni6I0;!PAw784am>I}aimh_<6A2EAVTxCflK1x>cIa@=P?$--Q`D=FG=w7);;sI0| zm4dk=caA@D!lmxL+`R|fyYAkr+x9~}S;DK`{i{mAc?hB5VXjBH9^-nNgJ&0P&#>F>RnHxAIGNdju~v@ki}JbPfit6& z&U`Jd@Yd0bF4I%eEvHqmABHJo=Hyj$*Q1H6kdYlMj&N!nGqPcN#MJYm743~aSX%;TVK@yfwLd#yE$a)6GLWIVYp&z;}M~F^w(Su_g#@$ zR>-kB2i;Q-OSaM+)dksE$GLyI;q=xN*=xk7_2dL4cK>EjE-BY+f(k!+AZ?+SuaT4- z(09JeaY-DUG#UejwrmZMHl9qj;QEyz2UPB%O~D}+`fec|Ql(h9lBd8S4IS>_?nKUa zIZp%LFP{}%Zhhx$Ae5L@oUt!yJKj53;4!n%bJ{lw*QDXIpd)39A9{de z(B#l$@H=YGMmd_AJjKyD)KP5>4Du0e8Q;2`WFH)`Ij5D;hvYU(14CczM?-+JU`Gb# zXCUI>kY$Qto&8zah|a$Hos7;N5|9~RRN4Qvy)Ln*61^s{z($x36aaCrer9E2tcCS!I6uDQB zz~27?64=!nNE%!GCI|6OmngIG2HO|JpHL}pI^HdZRo*zkP0rsjXHiKRO9DToJdy7H z?MY*UlIl~OH1=Ign{NB@7keb(HtP4%x=%PFuO!^bA@qI@p)XWN&leg~rWLEFV)>p{ zCRkuM(HqRlw*igS-ZM(iuI$mX5Y>_}u2EL0;V_<3+Q89G3g*Bym6@X40;-YgkhF^< z71!<1?vi6b6sl0!65DqNyCTfFH=*y$Rn)SAk2WJ#!#EUSjNY0?C+)6RloPyrvO}>P z{}ukWW2Yo_DVqqvI5s#&Ojao6BNyu2fJpYBwv07Vq~Tk$W+u*ljOzo|nOuFy-VsgQ zvKzkvMtTm7Q9}IH0$mZ}M}BGGL%uY?(ZEp3FXNu}WQjA%c$QS+L~yM-=)&3H;OvVb zsTyF+hr<&%TRJh%K5MP}ZRXf^n{G2+koFu_myquZ=nSo{hL*2J@pW7oTAq-CZLP(x zNG*^NZN|^n0+zF7QGYtg3}kshsQd_EjRWYfRLa?U3TVQOGdJHoZ(S;;5;5AJ z0A4>Ja6Uvr*>F`tLQdZkA#&vDdwj3S{HqbaGS5KqGvf7y$+LZG#G<{|h*!4Cc*QH* zxVpGNTnOF~5z`ZmS9asaQ(8#DO48@pRm@2`;>k3o6dLnw*jN>IHw(N-TPy<_P0U1{ z67PpnIh-G6?|$-U!G7n|qRhX*yQ&w82J`Mo>T4&;rWLmcGcUV+aKU$@1HQX7ke#U9F^hFNU zi!x9p%*D)|Gy%L; zF2DioqGw?Fv^L&`dJucgR;(>;ucfHchNm@^pVWP3xsWt*eT_X zjUo)&3wLwvwqlrn)e?qy?=~MPqZ|*f7V1~!nlc$t+^g056~wLNTEn$gOrBydm$cRL zQzJjMB9qOh8KW9=aj>n*S`T%H&xgIrpp?`X^f;$oB30Dm+a7gsm3#$ z8c|2{Un_EJyinDc|2$!kqeoCB2m3cz0Ra*CGt4-_q=x$OKdTn15kzmz|Dj}LaCK{G ztHea)(+=rzWmzu2i6sB!o?Lgc(+V1JY_cay5un5aFfljql;FC1ejx%|(jgGvK=b^! zPN2bvWb7|p0-cx+=FOB1ot@6fey(3&&cjr|g5T3>PKvPuFNFNxl2=^$ru_lxZf|~y zS~wu->vc~e?GNmZ_RwM1228MS*`4ad&K;Z>B}66 z*Z6fa_ukLYe6^}qE2JrrzZE-)bxtGO2Ou)!;XGhF%xA0;Cwjjl8{!6(au$po%xXMW zSqDbY9t^0ZkR@ljI-Y7|)Rm`RIGCb?R5KFa>dLOU$j6RS>W-?1?YtP*w`Z-=5Q5 z#GZ@eJwCxyfJbnc^IgQnAzaRVC3l_sYVOq}39pqD^TA>uCg%F^`Hs0>LbcYzPRKRb zAF-j9rdU6y!E$|j9;yP~0+^>EsW?ywFl$5nG^wK*yfK>-uP>LOO3ZCiM^I&2zh7QD z)cU=hFz4RgHN^V8o|mrmTTZJ@L~)V$8%Z*(-`^uiBz`;A?}`+OeIVm>n9m;1aQ%*% zyfSKT8v@BZdJ(ic06IweE^Ei~#jqQeiSf|n3qfEI2s`~62Vn*E9G2;9!ik#KqWQTS zaU7l*;4719|NBKnJz%U~iecNlgRu{?$}g;JugRK#QZUPp7hv4!C4<5E1)Tje&8vKLdAi82 z6)>K1AkZ{9D>vL6vvV4A#UMVH=K#%oq9uO56MuumUqZZEZ)T4HmfQRcsu``J@Lw`mc4M7$1n)WINS?TQ ziX6jP=^m|F;T*$xb|j9iol{qbu{L!~uXFrvimj;+=@cze4Qg{5^q1pkQ1I6bBiphL zGX$yc7qlshJKrx*A1n%$-~=7ygW`>!2`2O-_#7+Ntvl}&5zgt@*mTEY9r(s|=7@is z*76x4cglZc>83oidqD^t7bC;QT76~zR5Rxne4d;2yIUJLj~_O2CXLkARjFf$PZb!z zg7xF}wJ&K1oVs;|eq7^}NSu!jm{mi7AV<#yj>E{mq+txaI<$g7D+tO^vyPz`TgNU` zkmz?@R<=z766dl@J?;Fro>z)tIF97Ex_(Y+M{?!-_ovm!+AXhXE$0a_XyFs>;J0%C zX0_u3Ir0F^7n&W@e2bt*ay=q>2IU47=^TV2>k+?FVdp#)S&#UE3cE`K2`4d~*lxXe zr83X&8i0i8Cdfu_kf6igzm>uOgi_ zi5e=FvrJ|FqXPN4rBxu;4iCSBsd?pFwM*abCIwzAyYX7-60a3G=TaB{m8wV!5l^NRB->aiwD3WyFV{zeMA zk$amQW?>?xRKh-&6(QHt)k^s3vZ9sFbGbaP;MtsBMV(XN64MjZu8`|RUd@vY>IGM!Cumpf;(C@4j@SI+_d|FyPcT~H=H_%E zodG@u#|@f^+{6RZZF-(VQxnTnUnJnCf79<3WOxo|lJBoXK-s)phVdR}@L)O5S34|6 zPym)wJZuRk{i!~lu3XK}wJYI7BbIZ}*E8^`pTa&RK1If+@Trzuf={KH_|(G!@V~*Q zHo*G-G(J6vaZ~U~>Jn@bKErn;c`NuNp(H+?AjV7H^{W&KQV2esLN`Dl#IR*WYo!bq zu>>!Zh_%{zR){4C$Jfa|C1MFmf+3G6d>oEjKPzf>qOU4y8iHPea)??f^r``4s?F&W zxmAHy+(4@F+%k|&;TRtHx&0uMIURQw-&s772umV6ewL|z3%>EnZ(Y{nV4E6Ru9b6#u-gR&kJVyxk`dpj)G=M(hsGq8=X`TWOyRxt0K0$MuGYA zR4YGCNYu?-D?sw=wIyMWjArA^YTm5jT8l1GbN0BtY$Sg`=84Z;8#4R!r+M^97_}q& zGC<4*Ax(nux!oP%HkJ%N9<43UCB;;8*)IbqN!V!Yo?vU9%Hum%;b^qY`rb)Ixh`3~ zD3qRH>v$(gQ8G!ro5aie-v3oed`^;Ml1ZvaLLri(GyKR|k}$3ke%48tmrQtaI$__n z!PcXl=&WRPo?8|3-_tiE)s+2WdFOav%Q{ijBhhX>eMP}mw>5pqR2zq+nn&V@`Qb`> zS?Geu+YyFf;EJPs3DpU~>}M)MAjXm{8YT{NRu;#pA@ekzh4PCJ(8M$5}L7*m&CjgOo zf2h5w69r95i_b0X2>UbI*F*dK)<>TgT>*z9+UG#xgivCdK;k`Y_R@9btC;ZpNlf^X zV}?|Rs1Dk$G6p(DYSvmFlZigi;RIwkO&SysOkp;#I}w9MlpIhTQS$SNEYBgb_kMmP zppcvgU9wmcCJ%iZ2=G=))aAjMUzkbUVlC^qVaa9AsO0i6BKi6e$!W^)R8DpxtZ$M$ zIau2ARNrJ`{)c^&ELO#c+VQG&YU5#eGpr*=Cif?k`$s1K+oa*3@T?w@%B^j+$NDYF z4RjwtL~Z%gnS;1XK#rqgKz<}gl?+%4=tGB5*jREHxnC+tt&20;GNdFSPm;_bQ^{dG z{;=s4#UZ`iQ!TLePE1!iw59&cbW)MLu-ZwV9>|gXQ&^Kks!#5pB(-inZ2g{5PXFYd zZg24L>4BFt)5ATTzToib0bDAb%E$}r)z9&k9S*#ZkF~4O)AdB&voi&>!D8qEY_ z)^{qGh962=p@cKl@5u@oT2RT@S|6AIB?!p+hRS!5pQH;TMNQZAT@Y-YAQ=LMKJ1EG(!3@{-zfEp9m^giuACme-KLas?7dP!NL9eG2$|*o zJhOasm~{DwoNoDx-129t@^QjFm@NNeIn~(2(>bI3o+0IbC9{0;r^}y`D!k-C>gddxY{85qr1~VFAoY?w;p5k$YLYEeBWpUw zscCm+O>IN^v_+MIqh&?A7Q;%HuIWdqn(lFH`kbn13!>C0Ep{(2%l7ofI7iy{{Ss-G zvrT{E+F_%_iOAcW$0u|9!mcG5TsTH@@ATnl7CBOIw7JX z2FQ;tKUJ(2AjOEg9WpQ?FGZH#@Z1SP`~qF%=3WKO#4i1=E;FQ=dZ`%uv)nOAcd+ps zZz|kaIfp=KRuNc`)p!+@3gf5mFD){$ql4DZp@kH7wiMR)<)GS`Mf@+4@hxt=aFjj_ zR1PQdjmA)`@?Yo1OZ{vQ&F}~B40pSY>FNl#W{_6QozYh7kz+ntzPySWGUUs!1k>{6 zNlyK(*6YVgy#q?VTzqr%3CY!xAeofs*7tBJteeiD|^}MA8glZB{9X(*jAdPievO)AmWp zmH$A?%IuSD)>1XOQst07ts9Zp?k$ClpL{3esJ=RquPD_*{2Q9w1nR&c| z10G~}(qKOPsB!S5B(+9mHX~h(%v_Q<6F4^hqqYbk)|HLnrnRNkuk! zVckyRVS|<~U*Sdybw06AlGHl=&^0UhF??TAd~^{p>oAFd4vkC7j&9=T3l0eYkRhYz zl8qPE!z2!wujuA0e11w!&Z&{}Agbjn6YU3bWlz9G>`+vW3^>Px%&Gmdg5qUkl zU8eyG5&FvUzXXw1DtaQ5gg@lhrVOI8Tc-K{#oD{TM^#;m{|N~Y1)0GbjfxU$TBF!D z3f07T>kQ10GcbekfyM_)>!Y;3Fk}!!FgS^1?s1e`ZSBkI@AjdsZPBX*A5{~1BtV6r zRYBT{R_lq;3bh4%&i}jCK4+dm?f+9gOlHnG`|Q2;+H0@9_Im9F&QxX&T*b(*cTh~% zg1P!Ps(;zrvo4HdT_E*S<(J!q(aD>)phkG~A2c|41EO4X@cQ5*be@Nm|3D>d%FK~6 zmvDEmCHM$Pm#4qaJe?*_1?0g~GEZ;LJUw2XqUca2@7CO$c{)a(%0#EP99fct!-Z0# zlRdAL>lxL1Ug3yp5^-4~&Mv@la=DRa7i6T_=P~p+eI(UWX|@FxTkX7kdTGhJ;h+Df zU=iA@)0>mlrqk<&AHYqr^-JieE8F5Lqq6O!H&wQUE98o{3-ZLM*Q7>IwC$G*Phk7K zT&Tcyzn1B9{x})z8PV2v)5%7v-I*()ZOyCt-;rt;YSk*$J_YKKnwJ@=_JQ2`zn@qC zza!NiO8u@>tMz9k58q9TjjY+9+tMR>E&V%E?TVnZq*85BOFxyLW(?tOJ z=OPM>1P~O}wS>la`f1LU+)tS3jaUWznIN^(OehiYQ%_4M(J=rap+ucR_AV`Wp+uwf zSvHjDgG;2WnV@2>9Mo&(s8sY9y30ZajrLpav9NjlE*NiL$M<9?15%1Y>Bea<@str#lz<^3|qrpYufHryyB&UFJaWLYfo2F3E&8a*SsGe8KDE@RfAs zZ)O?1KChf%2)sTl1)O0tUyRqcPVw+cLL|Yf$YQye{Sm<$2D3#-&BCiE0^9FH36aKr zU*a&hL6kFN8=v(p8>7F^^G8T<_NeC`A0hN)&pVtFuBoVKSmRLk#z-5~`4hSku{Tmg zB1{XTvn-rX<}HcI(R@8}=I|pX{Oj@&`Ey33&Ix2|*jqYJ5cM4xWV*8BLPgZH(%6KT zc%9Js$ezLc_auOIi$mn8RrWN;Kt_F2`l5~NBfcrb?BcSr(XrT~GG*mtM_-4}<$9^n zoHAbC=NmiB=a2T`5;e^)zHxokSX(DoTZt(ZSRgf9ix6a#(~IQ*v(6RT(+xRjnb4=0 zDTqjuKKl|&DIK*Br$B51krvzH*)0N2!GF^Yp>v|!Z`LwRb4v?<1RNx#}#c}Dr zvAE93LFtzCwn;i0DC(@4c|*^?lcTcU>5<+V!Z$%iG~fLntUEJDn|*T;ug=n+P!e3o zCXTxCeP4nI22izMv~ja<+5h7=URTiHdsgF?I2jA@T<4@uH}TZ@kqKKeA*McQynuAh zg`uV^BWQ_<_F6C5Gzgu7bfg}aQhK6|JwO(RRS?Mk^u|ue z1iY0BNx-)6=RU`ftkjxd8+2vlZ$ z<^)MKpC7?A_Z!iy+&O(;qmmu*?D%{#KjTbd!_Nj`mH6Y}o`R%v>UTjK>uB*h@tPEH zPCRKSel||->N>FxeztM)C-O7DE^SIokW>VE+Tu(6)*V15?Xv%s-z1vCjj=BG+$fy) zCmoW1RHxNA-Ot(HaE*VI2@xvSz_Nn8aK)wtkV2{97^TWFU>K$NP)@Ffhm_XALvX4M zl?J#IkUP!XX_h;)xr3Jwx5Sxa+FaL}b4tYmA*gzhHZESBZTdQG+RR!i%AK_`X=Tnb zi)qiIy(XFqQ0jDAvuLe}#sbpV>C%=pS4bqcne150ufp~9l&H<>oH-{jIstw=AR)(` zwwsOcktu%WsC`WFyao#D_jRZSU(m{2Y~0Z1(_|9IS&42Y)6?-?`xGa~;0G);l&;*S zv{o!tKC1V#EBr6@I=jOEK(E~u{uWcX&)Gil6Ib{G_}g6mcfZM$a9q^mf5Y;T^SgZ^ zCp*6i_}@xs)Z>5X@`vVs!*zAc<9{O=nc;=}58(m+H#3*L4F6RASDMTJ9(Jcs;*A|H z|ATr*`sJihA*kw4WP}FHJVrP&!w84xGQ!`myyKsB_P+<|x$rS>RU4B(KZ<}$yj3sC z?`pApv@fhEA(14B#w#DD$>l7H`$-n3i_ennY+_CQYIi(=^AxBgVnEuBxo*!^OestTSXyBrjoe&cBvWw#>&Znuh*-&9b} z5KzR$xN)!LliV7A@pX!z7D<)YTMa?K7Clvqeo>3EfwmG0tNg8Hkj}btS*$pcr}5+K zj7bKn*ICLI(n``70bk%N6ZlrI zE3c=ExtlTq5CTF>!HZCV6>g)RF{SN1qRYDcOfiTAYXQmxu*bXuqA zEy1N###+<(%={C)LZ*Fdf`gM{)Ltu@6aZP5NIAk*K)7gtAWJwUDR_{p_Vjb?<#n=M zSOh~)SI%q}oIwes;5oHU2S5;=!R$;e^%X7Eu&8R@I88^s(fL=4LG1a3=zJ%OAJADQ z=sd|q=a&_oy9son8=TB!+0Q#)ku9%aQ(h~?C{XcgL3auen#HFEUF6d9u80j@a_RZb zbEVHq(H|2h3x!NrYc=kKrg!F}X0{ePMT+I4Ms4UG(Ve2D4$@LWbVm!p{TWrYkj!PD zL~ziWnI9g(o$O-hD=voof+2|R%V5bFEqmI}j3Nng$fZ1{cdeD#Kt>z8MLqVEjOX-J zc{@p0$})Pkr!`H{Pj#%t($V;|=$dZVYnWW(ea)eh(Cftes%M*T3in%NTR`78@Z3ga_A?0`w{Yj7L zPyV0xp-kC3SmBh*EqkkmoNOf1VFhx$q}M{d**OEPsy!x>*@#Jo zQ$}63I}*drSPnACfw5zS;3_ti> zX%Z8d8z>Tq`xI%FG0~}bF|=#OgSA!uk-tgD<10c$5hS$8(!JW_Q+J!eI9RDGI~XJM zIy)G<=@wO*!5GjBJ2Mz3948MAzsX_s?-M%Z4ZztMeG5?=IsjMV9)eTEX7FD=6#k>; zd7pFVu|@drkX!cFh)p8tjWL6N-9w-kW7xPKu}OzL;UA*vA+gw0zqir)o8XcZ&zi=# zjL7SEBgrLx&2bz_N(OBPG2)>!kFOehdLo01&zFTrB3E=TD4^|LLp&MUUZvMrbl$7i zE;@f`3TM#yDqca1pTTFAp5ZU=dFVXDdc=bS#v=55#B@O?JE5U)S)vc2<>^#iK+E-V zOK2JK3tDaxT9#!4T7Hz*qP8!4M7IQ(W(hTQJco*ZgsAn55VKV_;2!3FdJZuc?oT8b z`Tt}Lu2{#`N657dnr`x2SFGilR?RgeJ>HNsV<6P*(X%JpxcI+Vh=lnN7=ph{F$9zu zgNb^b9fPCw+8u*~P2tQKTy%_#LGHhz@hIRg9%L?gb9N5nks6JDc4+=`r50dJR(-K( zOnxS}hUPC{5N{;mFIge6h`$W`6#kO4C6e)Q`OA$t{AK>X$zMjgZrl1WD&dCeO9=Q5yVI1}I(mGaaX5r9p=o zzl`mrO|P>`gG=?=RT`XY3imm`I9hQs1E{EFxR zQs0w(DLpEAbguU6CFek?7gIm#oHA9_PG`q$`H6ORl|Iv5rOy;s=`-C``b?QEe9RKf zkChxLo)t8x>$AN^TAOS22JVwBu!)eLgM3N=jA&qYS*b65g_YJy&gsY0u$6sBSYxRx zWM>=>;hR9v5W7)vqXo3O$5Eh{NQyK3wbGTx=M=7@aF+@QJQ9bH0I$wQLVah7)~I{! zclnKb*6_#CPDp*0WskIaC0s)<=SY|_{u7mFk?xY{%yi|m)SB2^?h9`vn@okhFZvJA)nx(1f^qAaRY1CJEWc z%0^yiacYUbxmx*)u@|}eA4G>EBu~y5G2(3GAjH|upT0zsiPf_uL0tT=0$-hoz}im0 z(2l3k5ONBr<}*?)O^!0L_+zw-sp03a-iX1}7AqSm<)xR8Nk2Tz$7N_%!6CP8xvd>N z*4&edT_Va^paxKELv>SN~qX6KjP;`+j-WQ=xJER9E2>pMe$EIP)cCU#5C#RGi)(jc=6^CyE65sjk^ z44D&OLE3N!(KYMu|5!7S53CzMB!4WLzf#z;Uvkc*{AZZIC}*bx#Pkxdr$dBH+@xjY z&sz`W`15}H{Jn;iEIWVKgD8)TCByqC&fhC@3b#;r$ocyf9(WWpk>4_Z2hmP^()_(W z=FZ;(X(509?mTVi`CD(A(fRiWuT0+j6+Am994QbCF@JaDynhStD_uXtO(p9yx%27s zcWD0Xjc@A_rqh9_-90pmhH&c`%{sp8`L4I81cJ#x0Bxli=twfakKQOuS|`_M8sFPk zce3L*+&6!IXu3PTC(=Ux`1+}0 zDE{1Hn$hun3)PdBDIVW5#_eN#OLF@1Yf)VZ-+GQpb$na7^XcQePk!a`PbINUhZ`gu zj9^p9Kd%=AdD!}bG^%h+m)i=+06HLk4Z)0<)B`xC=km{i$MdfhzJO?~-B{HNWre~_ zTF6g-B&~R0{6sFY{Pf=3XEzyegr7bo(0cED+uU!0ToFT}_UZK@y9q))7y8`nl-~!F zRVK)lbfy`44O~3%@L`BkIsf*F&q8{+>R`F}3FkI8)(y6>G= z0G(N?1`aNOj{I?Su1b>_pCvgIum*^VlJkn$2fl;|<{bVd5sI#?sa2KZYPA(GB+wLT zDpvpakH^saPDD$DMC)2j9|S;tb(ZO94)T2S1nBiEY6u(PjLwaf%=dPaje@9DJ_ID z3}LVOFfeIzh#}y+Lbt`* z5eXIpzH>hrKEHI^Gd-Yv2clh)Oe9zZ4g!4p+izI@VM&}0V?7}^oS31-gpfM7)cFxg zSg~cTD<8Q(>ZB@k)@>)ssXKfhclq*&jnWMjxRxH~T&1O_SNK*9UxKaZ@??2~mH2yU ze5+9@PEk(upa&^J8GV`ofo}F3n?2Z4{ZusTdNd?a~ znr9y(9t|E=@N9*7)-R$Hg0GAoj?2-;fBA0e6S6hi5kVtE6p-oKBGXlq!dz$s)PP|N zLS@VVU&mE~eke`!!fcxixg~B`{!bEKOj;iQLU8v~qSc|H{n%5TOvBC(KOhfddg@-$ zy~w^4ZV*f?*{g%dT5u&K&mabTH~$EPxfJV>-`~j>AdoYLFPn=sD|mcyc4e(mW`cL&FpMnt71YE*L` zsG3*VnoY1$lj&@C8;4#6s$@elO8wI~!n1#H1m`E53{~_t`N0I1!|j~4##pfN>r1!a zP{tz;!j1pSO3eVpb4q2ehwX~JCG~uc(G5N-+ww3*W&3&R8}pc4qBCXxjCYM^OY5`j zosNB!ke}3@yw2}3gN#PsN}gAA&*Kg9E&WApdSwkvPde=~3c?cT@h)c;83gh$TvChGnCx~a9LP92n90L+)X(xzCbYe{X;KGOMx);orD zv~Ov^z$WL8;+A&Z>~*AaenCs?G!b(iT&um(xh-8(?lfkaTIx1+tk)Dg9$qPn!p+{x zsmj9(n!2F4sTQwvUQ-oLzxH2xb+Fsilf@W0+g_TW1!xwZeNlRe5PaTOGx1E%i$0wN>T3rx+L`ZT*Fy3z!#9SvP;A zjBt~)y%<0BUPYQfRqpt#xnz0%ocotuj*{VR55kF1rZbsLsP(GXpvxSDo{oqQHzlXR-;T z^Jx=0sDKVWcIn``Vno$>6{adH3z{EZ-25=FbhdeCNpYpyZ~9a^*sC=a(Lpp-?(sHM z1QpIv#g#9~R9<Y{#r-|ddplJ* zh|wsQ5Z+KU=e70P(xC_;;9Qt(E9kZrDQ@c*-rK3l1a0N^Yf*7q3%t^K{j!{$iURG| z^Y0o$xXQFCt6H;>pOA89BjwbI5OwiDjOV$_neq3k%h{ocNAg_OY}FX!MT-#65?jF% zo9B`%_A{(~U^lIlftS8Iy5=B5j@57?Jk2hxN8-U^M8$BR&Jrz+qfX+Bc ztmAoAy}=4-7^HB>2Ld~rF>2qaU;L?-!+2_Si{|)XFuGPsE!8;a)nFb7h{>QD))e5F z<2jan;{sV#c(YcQ7TY81vb1?sP{mTaSN2{PuMWsIS!Ms(YzWVJoWYr>8$wRh`dQ6A z3fD^_Y&MHU$ll;wq2tBD&ulmmCpb0E>AO*&$Hbdj#ESo#oP{BWTZFnUl|_HVqKJK? z+55cQE+RPz@lvC29CFd$D2x7n#VuU!wt!4~fv9Ngjnw=3byHaM8hf3U=KhNd3CjFsC|Klw#0IrBQ(^=G`&xlJwc zJK=0w$GL5tSlrfa-rK3lg9_TZw79LWd!@5&1#wb{*h^;P?Cd+G)D?R55PG zd6lFpAH1XkPdJzSu=ZZj(zYoXbsHVDi`Rn5NDyZt6kr;%MLbn)1rjuA^jz?%0WC(3Oz5E11gUVv|{AO}|`8r0F)5X-6j>j!n z4)bdfvRLM|n;q!aZZMeeh}&KLbBKG`C$~wxLfpp}SDkdLhT?vr#-}WCU!((>HzXH$ zrL%a7Trw4b`O38=&Zc6#PH@Y66Nc?q+t^V3WhJs5d0u+^o(^GNBfA$Bjl0uYT=!c3<#DO9slvGrC;G+RZ8Z^lF_unVJm_TS9}5L5F_`t) zY#l|Fx6TQCPDVM8R5nsfn7=e3oHp8UNmbjmOg`D#1!NM2>M!m3Fn&?Nd$(ubW0Ac= z)BU}rEi@s3W=*R^q?O&LEaI4SRPbgb!%t30@&h+eXeN2 zoQuE_GoLbKHzs`2EQL`lO!@1E$Y}|5KCgs?BK;Y`g2vy zhmcOH5NrGgTRndx3G2DYL`?`Er_YVq8}xi(k>*mWO^sJ5i73cEOLAmK77~ow zhTIA$l&Lz`D%%b*ZKeLI2TBK@^tJ;z_8-x&Rm;u*NjpOCgH>`q85;nZGH9)~iuaB< zLvo@U*cF+4b`TaGOe_TV=SS=_6!-CF!F$^@jtNZcO}9->jjn3kJg;$Q$oF7tRoh7J z#orn{-d8IX>&X$Y_kd>ig0h}*t{h1M<#j&JXe6<9brg#Yrw&Fc6b9Ok|Bo=p*|Oi7 zBOdT=VpqG8e13^F&FZh>>I=R2jBIXR`z`63WuH%_Oz9J2=u7=5NK-TWgZAF&gn53k z@Yt7BaY!uZ9QQlV?kDR{7}}Ri1>t7CLB*n~AYQQ8u)lr4}uURuDjHN3;E-8pVwHNOICF^y4qDwy_#dy-T6 z{aRaDdTK@V*d=~(HEPDys17$HjFyX7JN0A!9$|L@$=WK{y;0GhMpKt8w8Xfh^BaC>gG2HCyc#LOJ`_vj)^Y(Bw^uJ<9|5Nnv!((>I zV0p}L0>$K?OmB#kHCI_jKObp)A%1<(4lvoC3pTD^GTeqVS10xkhbeyFz9rE;k_D}7 zWBQfUv<9u!t5a|Yz=}};H&wnrZn3_^CCYFvR|gq z;ol>qj`oGZz&{WSQzTfnPDgoAP@g7{OWwjwJ6I**R->a6y5s*lNK6sUSc#9&%>7D- znQ(6j!@r9Qj!jBO_H4<|N9STLf~?BmgWo=l<~q73a`bwuvCr3e32&0Zia}W&Zrr+L z1jpI3gvh9zprf+Y8UV_6KhC{fR7_be~_hT0!DJ_Idb%KR^yBDIfkCM1RHw{J@;hk zd8dcD3_YI^6~+!|sp(MfCa5>E=Ut`V(KUmgcgc5q4*715HLmq_UI()D&4@jkd2WAMOkUk7_W{1KXNpA9=6731v$DTX!nE~H_(-*gZ35~*ku=pB~_+p1;Sn4^z?O{#=y%obt zOK#mHuB@!i4aJACAx@bjvfVtqFYB>jbX zZkxE?cPutKS>zXR;$is2-MF#HLD@;GI*n~jxH6~GLo6iy4VJQC?$8BE>uWg`W_{2X>ht+eLQTj*$`MVzW81hRN**caOAtBv<}tZ$poTFF{Ea|fFNSub}u6PTVp|G_^01^{?T1?4T-0aj3C zrj!P&(xh6j*&|IB2GmB{;9$jIIjedzr_1qVnA{z1B}?@t3>0FKk>9lNDb zkv)(%Hnz>7?f4F71rrkoCSH~nn*RT^aG~2mwolZ1vJ&dplLtTN^N$eRI$z)pzi}tU z*&6i8*{bd`XKC?o6;E5=;QqyoVbJSfX4Cu(XfUfp) zaVZ#EzEwNIr1E@_ym(|8@of?gd84DLgc^xf2MM67laD)cH~IKZ`4YktU|lcwjB8N7kK$}0L0V`-^kkt;3Kvl6oPUDR{&(1ur>bFceMQE5Bv`TMHP~UXES+PU#-IZCotw^pk$n z?gJDK%HG{JGGdq4Ank^dtE7Hh>@pzy5G;5gOjQtwL%vYag-xXF8%U7ds3YospeR6sgqNvC9?lYo&x#-k%^69abW z>Pk=m>~q+tnJ8~D0J-3&?U%D1XwEjedu+A?@AG@PZ{>_}a%?IGnJ=j$S!v9^*^Cj+ zma&TTcdKQT_DThZDLovJ1NEa)f!#Z(G}$7MGbR8ekgIi1Jf!}!@Sl_e|0$mZ{@Mcg zV>u)E9x8!Ph2IUh%8&2J(2dM3dXC}kQJF?8)4mSzYr`4O(^sXjQ7BF=dmKJqwxBm^ zm)W<=tmkk%=eQ}*MKt-4Jh1Gan6Gg#^Dhh9zhVl5@5f2ygL#xK-Qb%}Q1(b$c3Z+5O_N8I(dsTuW! zG|5C04>&@ntn?Qi^dUk8*xZT>T>jK@$y3Xf#w=4Mg`BX(yt<{m_mo2*# zbr80lV7>k4a0eM~;Xd@ZQVA_e<@8E@)0_Dnve#M-zZFdQTx>pri!^$mbJ_X!eX*ig~RO{FUMUo?eX+2l5@m7JUwJSTv-ZYRa%lzIL z9W$A)Iz1G17w5REu>oBZMP{^;&2pw(_GC)$q@?8{LiL69ECeE-LW2A?-kme zErKO}YlKs6KOl?nm-!vFQ^JW+#5UYz(0&e*IQzGaU~UIsHQq*$7;iNxGreFh`!gkkxZcMxyPdt zjWlYjE)=a@ZjT4HfRFYkq_b>&|G?jf9Vb)B%>v@Y?e#~RHEdRGc07iej_G)e;P>*N zeY5lo>iQyMnAD4G@d*r>x7@rqn&N|(QN3VWAg=X71l=_)Vt-H2VK}@lX**fcMg@Dd zR$x`vL9>=_$pK`s)GXGGbts)eHX|fFpQH}6OXEKEoQYQi*LB`U1@th6c-m7%T$*TZ z0Pdj20^AR!ru7h?2AwnO@i465UTRu>x^gaEkYb;=dP@X^iR;;xFff)slw44Cd2e7` zNh#+|hPq~tx9q#5h?Oies!aRmf|?lH#E|`j3|m-)$eBB3bb1Dbjidw>A$x4F=gqQU z`>S|Y9)wvcRKrvp6JdL7fJWUQjxtvU_=WfWA{Oe_eIg6&Ls;Ov-fGsh6(BAbMr{W-5r01 z=)kDyx8%!4+~?$!#trk2R*|Hv8zo08LGs%;gcfp1>@92EJ~+~SLDu9s(yXXz=fHt6 z&N=HDh+zBTNhR^x&hCX@R2jgCxQ}ibk?d)FgCxn#FgPC3t_xuU) zIfp6M6S99Uq#o{g&DXI(dN(Ao??62hj zvRrBwV&8@>-|dnf*VY^n57T6rLwcrr!34O5Zi(~{k<_|#z{J_MYhEWnM zTW#6DlgU8XecoIEX#BODoVl=HuxEg|@CqcyOh;R6mj%}aWF|oJ%@x7je>$wYraj2X zt>LZ(*~t)KGR&=w^ako^r6k;C)w3d3N9>uvKFXQ+97)M{5Bx%=hdriepxm(Zv+>R5 z*2Hd@5&Ys09LBcW%!vi46x#X7iFm?^m-G9ci2b2(@bAi`^4%Qf%?zo#^TqIU)K-R{ z3s3Lk)9==@mER6?i2LyKE&?!e**Vi*xuTKY^BKDM0si){!qBlB+f-uAJN=@(A1-^e zny9^-LGBhy2ATX~{l2lWE^h2#)W2_u)Cs4{E(Itq&-UFEMV6t4NbefiG5I>KX4crh zlSvZjz+0q`9~jAVR)q2Hits>olI%p^L*2R}{FC3GGD!|aZ{-zVE5(Oi5jeRTPLf#= zpV0xA#^DXhGt`C1R zIRYNhnmKToaMLPs1XQbE|9klD=ZMb}w|n9MrpKldVa{ugrtw4$G4cG;EXDpXfHNPISZ<9X8?2$qHPU(te`<#DH zg3rH}P#PC|U}q zcA`i*zAjP2u;UdK(_Ga`6-c$f6-vcQAKOd0P%2yjkbaeFAC)`l%I||7SrqUvW}@{u z&2P~`(cEyvoaCLTW9_w)<^(VMnPjI9`vk#){iF<%l`@-OQRkMQE3mj*NrmTlh6Fv5 zEKdjUL|;hvMUo@vx#*TILVFB8*rd&LC#b62+?|=6FOvLqO}jKB2Hf|UpT3o79E0_u zTEh`5?DcSWC;z%mjNh{9Z`qF9p1)cIPbrx#X+S)~KYnK5{2 z6Ks96bEa0UXgV3;iAJl_^R&_W;zqw)P>YPsx7D7i3 zHWVZKn1a`(wPkK=9{_6s;p}ZL>Y}n&Y4FNygJ-x64)7gJ+*pSI{E|-3&Cb+f7=Bt% zslYJOh2gql7}^So3+dMh0E1x1kp2lzW6r$7d8rs(Cl|adZQWmGhICYMTNMSxGn9X$ zR;}2*l*Y9G#jUMBJs(}t+JC7@XtQ&d;wX=ReppakDZjF~+SZ(M7sy7qRwN7MOu5u= z3-F&z{Z9$6Et!9`u_xJ?I^^%Y#aaIS2H#liN(eKlNRoGULb zppie$$aZ@FC{sZA`YD+9<@>~(^9Svd33{cnnRrZWa_-XCW{CglZ!6XR^VoS7o#D1fmpr;-eeC%G_w+2qsWm0b8fib=N=zA**xo#w@R z2pK%D<<#pJ=g$myAJgLUth{j=z9ZfOH29u8!UnOxvfXvev#89<(OAmjjWFuP*IJ23 zACSQdivJSY-fg~4If;ptym)PYYz|LQTKxc})jc^HtJ&$wCjmsy%?*0FbZro+C>$f(16Js!`ia|B6}wfCiYnGq;xzBg4$Im6ULLYXXwu8u1S)6Y{-Q;3L^ecq ztM^1%9TMe1=^^3LCV%AEgG0xT@FY}Q=Z-qg?-5MkZDnL#G)#_e@_e1o^S}h`b&8%T zM@aYG{D0h##zqCxO$S8)rBwD+DwoGnv(u5rw{qohCpuwmz8wB7C@W(B(G#Bk8`6x~ zw=S1jZM=oP=ZUmO9Va5Kiex+khLa;H)vl*2rvkX__5LCvi-cSkh_tXk5or&k4Ec9y zZ*ZjPJ%QP8`#Sg2?U{$(NDg}cwHx#Xu`NIWb3{i!S7=0xiE6?U-#*vSYPM&e$)2?n z7d|x#iGhQGMg+tlhdZw)12$*J`ceS!rr;a_#(u}ObIkt|6PX#Q2npF!>rj$g_UJz6 zThFt?bX#S6(`yL@a19VI3STRWB)^ue`s*OgtNH+`S~}UX4?CUPJ(2gb6pq{XHr&4N zk8YqlFJa)wF_>YHcx-m)Z~Sog<1q*SM3jJDs&kpfd)Z zEB>X>RfuKz8h-bN@1a^|A_s2>Xivj122xlP{HOS4nqSm-sA8jmXiknun zA*Y@6u+rof+@`LuW?p5DAI+xL;_;rSPR|?tzpFR5gK5^drz&kJAU-@>+NO z7W?niW-7()KN@)L<>b-0dFWQTscF8p}Vi(Kg0cgX6D0=|4}BsF@g z)i}4h^}krjIU~V)4imtIZEiJACGmrSz}GR1y{<)ri2?jxH>J_^lgtP65HHW;EFf`# zA}u!_M)U-!L3Igu?{N+*HyAj-Z{@UcIHuxb-%%rxXZTCBl}~&dH#qy-bv!%*Pa*sB z$y;iwd5A@$*-G6}Bg>nhC4e41XrJR(OK8+S9?R6!3K~I#xm4_+ILF&otwfqshwMNd ziu;b5`*_!x_Xky-lbq!Drn1HlT93JIc(n1O)@nHzwDAMqvSFh9Htj}Xf&$#YBGx*> zPOH=1jU6?0-0>}YkG~e>G1MQC z<_M7(GE_)jxU367pK3LH%b!(Q$~{z;Ulf()+-Rz!rd`H{G3w?U6{finvDZ7k7sTQ8 zGT>n3V>n>gNSCqNXyYqwr}*ZCe@WF6rr zMI!Y1vX=~y(2v{A%%f^%`3tOuA~@e0+F&An)~8t9N`RT`cx&L7e< zh`m|jtE{Mf-}k9Jij*7(+5eJE9rc_(@9X$Ech&wnW{90bJrCJAtY_!YpR;qQ<2<@| z1Q~k;rI6r=Ks_O%DT!_P%DScsR?E;cR+7-7-C+j23x> z3+4JKuGO5_!zlq};x&9s#@;=RumU+S+8*5#Z5TUmM3hM~)9KdD~lh9m^ ztp}qhleoFsh;v!Ql{?Q(1tQM&h2Pz8P_8D~9DKtJ3Xpv|$PPiyGNkrL*^q!g!_E#3 z+v`;Ky~(j86H0nDRFn8%SnFS-%sou6vNe=lfb{Bw41T3)q;69FEvze%rSf1CI{>?L z?s-w_K{@eu>sEW&`nK(VB`}nOZXYp)7H;N&Z-2C99ikomURq-CtAM zRca60;_w|r`8yfX#Zs2IzPdy$V+^T)6HZ}aYJp}UzLye{m=>2Uvt+^#-emPgIryyo! zeA4C|DC>h-c@uN_9ZEj5fG;X;=PQ^xhh&UD)Ou~gl;`;)z7Ufo(a%HP-fF&fm@kJf zF(OQ1mQ87%_-ycz3YIUbn$2dxS@1{k?Fc?nFROLRmgUL0lzaT1Gr~M7p}5nm_ZH!W z)|YV5%7SpRtCWAy&u$ArJQiN{hw%8bccP~=g%@|-uNpTaGD9k+AG z>g}kJ?KjPWhD5s#$Sr!yO4P$XoBCLLgDgdL9L*3)qTNqk;}@n5KkaR*hA%)wO|{9> z>jGWZOkzo@N0FAUy!j1jXI-HF3UQWRYbBRSD^_xa%uK$7JMeW6UxBU_BJXhZ0AGU$ zR`y5JlE42zMqRpxzXWVr2KTXQ3a|>i0+l+ZbDq;vRl0H!WTe2ViwB#)x%oz!)h^=pYNKo&)>1H;!k|z}5Ut!GXzD$@&W(E8-`G3HE zP>Bgse#c7bSL#viN$S)+>c1|+XX?himVLJ%T~_NemG5$1>?dyr?sq=ZcA!Ixk(-g4 z>xAkX(H4A$ZA4Vx69LoGD>5u$C*|{WyoU zoU+C|6t*PHHXy9h%KAIwReAyMXRw{?kKeu!mM4mp;oXF|sXP8soxb+)&2l5TR3de)>rewXjN}8*H zxteFLEOQkzSIy?C$y~K?C4&<|H&nS1I!IT3q!W_>oK)qj+<=G9rck=_b-ABR%#ruS zxj3ze8!7XF;LwEWf0ND+xi%PR5~iik4Gx;BylaDnrfTon;Gs#Gr@U`4(IgVRzHe~R zRPS9IY%~SDYlDv_%eyuhX=?Va4NjU`ylaD%rrF-L!AsK|?^-Z}+s0h)T9Cl?0`FS! z0Nvmh<3>P;>a+@#)w%3F+Z8z-HGO>J6^|U-uXnJwZ}a(C7WI(bK=tK8-;yo9vAFeO z9A!@sVvTP0jYZxL?(QjX``qr1_b!Dg42$mQFQ zhw#R*_*jc&VGg01!NmGfEzj-JwyWh~+y3gqF@RSvUFoRV0Z2ictwM*9vh4$96=n`} z)N~89Tm)qQF&8gh{K|&m_N79y`j_#rt}Du7XK$2;oA-P8z_aOUmjWB(78tr^Z9ts%n znwg@2bR+KELG2Ac;5xz}#?P$|OEk+4qh9X~ce6aIWTB30ThaDyMJ?Fh8{To~V?S=R zlDM9^e#-U1QgW=?VWl?$+`@YYG4+R)+(v65g2(ypQ}0Cfw&6MqL*33dL8~}0d{f3S zIjg>YR&!er9a+8fv=$1?hPcdRUgx@gEj%=8$LeBsZ%~Yao1IgkXnWnj5mJ+wIAHp$ zaEN#y{!`p~fAFe|Tc4DyvzN(GM|w-C2PebxD){(=#3WrFRBx(a*Lg?Q&k}E{pn5k( zk@M~Nf@pF~)CO#OW}W!z4vQoYjS<@v?)o|#c24ixdf58*SK5PJkt4&&YpZeNt&QQQ z%PX_hAUDRj>sRUYbPgiZFs;;WDod92)r*rI;Zd_5k!8o|*zA|HI=0*Ry`22l)9hRJ z$inMS%{i8Vw61}_TEB5$}p8-42Nw$}n3Z?_!~g!1Ft zG>J?6Syk16@3XjD8ec7_hy0i7G;=3x3?00UlMTutrDb~y4;}#sZs_zQ{mDIC!m%=K z4vgp*xyeK<%Wg^Dw6FI^?bY^+h-7jpqLuTPFDc=5V(NqN9^9{7|{lgvU zI48+=50(WVD0Le~1tF2m-IT!|;${_)%mMdsM|R=?s1_P9@Gt-DQ9{Ij-tlD675b*L6!I$nw6Rx@(c-7ejrgMtK6Vsy!O z4t7_Aju$-&k*=BZT_o{|$uS=vqbWElT>r^>QSgaU&zE?wG=7}S(sg4IMd4f1SSi=X z4`kC-YjOpHL!e8&ydijo|X4OAYO#`|53LMJF_#m%2YxSKe zj(Mj{zfu=8n|x1^)Lr%amy|r=-}n2SpY;C3?G1N}AU(fUw^y0{6~xV2{IZTg^Xs&^ zx>3uu{>@4)6A6?c5ZC2bXgb4UKE(Bdu~DDjlI^gVsJ%)yrW4;5!x^b5-1q?us#}G# zN;CK$y!{&ZkS+e-TVKysHWFQ>SJ^{Uyc}=0D{HD7R_I>h<}b62^6CrOKYK^Ea?#YO zzX25`(Xx*saz0gRnefg?S#J@%#yl1^@Tz&Nncrj$c}G}h%s$`G-aICn?H>qNNG)MB zFRDL1HTo;D)I#)!-{PoM-PV`4i{3C+wiYWbyZo!G1(T#jsF-Gt-U-b#^STlbIs5rx zlL-PEnJs&C31&#%QPVE>xi{C_C+Zl(*uKw$^oaD-a;x+lzX%p{BaJgjo-hTa70Xc= zusY1!4ToT9LM=i)Bju)T2^wZ6-<`~)4}pcDLFPxjfYU7yc#jD zDl`o<+16<@tkc?z)EG%c>a9eB)aqKt2sN^ZF`-trc>!vb%p3|Q;VQlU2f9t)b zA=P+X8cUJn#PEgxbbHc(wkHOQdl*_V%YE~ALZ3f2wxgwMgy|KS+p%he{tgT!)D6>BfwsT$7du-+Ro%uzq{0jpm+r(wbbP> zm^FJ*YU_zO>bJ8Zu$Xx3#UM~;!P>;`C&c3DhK>>ole zc-67AKMFPcR%y5~-1%DkBzyDjj!mXtFfJn=iH~u0M1<&DHYxXSrB5My2U@fz<<-`; z5z$DqN4|}gq?!r%P6#hAu3_|EXOekaHO%Q&f>P3L=PUj-URPe5VNXJB{aamOQ3;<+ zidyk+|KJr?1b6w)dR2=(#&}gzCSCb4;~z`vyVM;2m94#_mTyN>D+T&kYSQ0TAC!J3 zmYTmexclZ0VFSb3rpD~AMeJ3k(cRMMX|g=uypKk~hPrZ7hI0b7QB|XLz8(KXmI1Xq zpJhPh=udHYS%bjTk~4=gX^=7MRN%QWZCnsx$k+LrkecR+V-12^kE^kNAkuHho>nz+ zeK4^h9p*r{FNYd8ww*!hmz}-`&#xkK@o`yE21j5l3)$Pko!jlzt`l-YKmRxm<(H8^BjSojQT zDD2n5<^N`5G)lkmXUo(xUh466rgasW21Er&sGoMEuj3VcD!y)AJ7pZohA3!QAp7U6 z|BfVd>&=`qi2`(vZl7cJ4l>pG{D=9*wmjhTpNY|+xf^fs$#TX+px@+)Ax+owqGX3* zMSUSh@?O>QWV>dN*$#2VkDYAavdgHc@zVM6Ge%8~C!bMM52~8_E>TlQQ;VxK?nI^; zc4ecm`drA~q$=xBYAX33e1*#DUp_}qU1jxmsdaa#=J;Xpqr82PmFo2L)$M|lhB~9J zJ^|aG(NtdydC5*E*~D=yV*Ir)clmTvnTIMTfY9G}Q&K@Hmj!Zm*WtdR0J*6?+lxB>>i zZu?^3(e=n|nX?mrB>iHgSD|+lzn5DnRp{c4=lrx*t}{Z)q%GN^J2vx3w%{h=T|HqT zf!sZKu2bA6$N!^w8?f9}Q*u&d*3IT@%io|i56H`&pT;&%k}Fby zV*Fg+c8p-yldr85CoYLDZqn0|`g4IAKl`l4_mJ09rYw?LKm<|myhCv###q}uBQJDlCCJhgF^Vqd#hN@^Rpg-c;Z6EzcN^Y%o19Hg z$l_pJc5)YoJU;!3R-SRiOGr9}jEC)K$(Mok;7xXNY+p%R)*~};MrzXfiM#RTY;jG< zARS4w#Qw{WD1YafEXv>IHz*JLBMnnqT8}3qX;u4;{u1W=x|n!gu&#+dOEk$!yST>1 z{E=g4vJixG6<67%AWHHlSKb@w;tSX9JvXBQ^{R;v)(t2&s`BBWCxJAQ>fG zx?j4;RKHlDVh_UDNC!nV4ny0c3@tTjvn9xcc2dg(k?x+7>0-8LzajMi)vB=fAiu5B zt6Q=p{YDOcZ|5oa&9`-^l~@NpPniV^7kx#QNvL7JL|zx)$}@33+LA!rJE(mTrpGa< z$=E()&< zvJ?hhP%#o}Z8U}WIT4p4!ufANr7S*1C#(|w*e%XPA_3DWS)>=$i6@LD2k0fo_*PCo z((ds(F#O}5uIcmwxJf#g)7i{!AU|F!pKRFSs=F`2TDt_xwAVIP5QqSu zrw;0i*c&1f$W&#R+u<}Z7KRpSIJH|CTC>N{wzR$;NtO4L^QvvKGPKRDrwE(-YGgvU zVQAOF(ALd66km_-=!8BP+B4Z$$k#UtLwiBO#P+Wig3K`fMxJ-^$a+L2zJW zlu#X_)HNzfH6eZ6w;Vi(s5DCCG5n*N>IN&@Z#=SuTobkwXx~WztISxC#?Z?M!3sf> z{D_R_csNyT(Q-RM2om&Mrc_)S=C$G>D?(MY^ZN;z`#cKnTPEJ^WvovVO# zUHDG6SDT&w|B-o_7f14?alFO`Uqbu5;I92%mwTI?(&a_PKR!IaxRzU3TKy<5l|{`Eh>IsL2)vVt_&Z6@6S=l52VdS(Y4uGT-?k{hi1E=&4@rbHrRez zN!uuOfZFWbTRb*DD|kzx8ed%d4F%<(q{`*5m`*eoR~suR&KSgDew&>&+F{mSpYuyw zbTW(m!0GGYWLItaXT=C9EvQSH{({@|^S2b?@7bz+JV>>{-C+u1plFD3_Ql^k>`OlOfGij?;6{1NxA{s2mDXydkX)iKYtSZcjPnx;JZDK7gW&)z8k;+W zY@%LH?TOgSL|qhZP_J2E$1zL~$p+N9f)|1h>$0+t`9u7_Ws{;^BgQv6FH{P8S@bJ2^t4$AGMfX!C<_%q$s} zMQa6aHKfg**~|4CMB1HEeph2}ctsUnHof7O&*Cb5yLf0N7Oy55bNHPg?k9=GPnDor zjHS8gK}3ojLUyN-gMOMBnZ8~8#lk0chfgE~Pj;WIacn5-nMo2LfM3E5nAOCfK#xis z|6hbI4T8@8kwH+EkoeLy(q&O+ON!@D*gyQbrhjKT{Yagk+MfO;Z%)_&k#>6&FmLBV*Sgsp{<>+t+7>yZPO9hHnaU#^btnLT2c0SqLeHV ztDf8{l?#A3Bm}^0=^JuN3t05yyZvLl(pSSdg7$9lDQgZl{7#H{zMJpfkBO35kG0ly z9a3unAV=j??E&&%89?5>32g-O;$~-Cdp<}5(tJ(_T*$wnZ4{vP2`y_-D}>b|ghgYs zsUZ)&e-vPJE!#JG1wS7uehPYWZUIDs(kou@kaR3P$wle01*I{re2fH8)*Z2byjGR! zinH%%OV5cmq}quK4)pe-p#XT{fP9n)0Pkl2xNm7b0QVJ?b`f}#)?9$V8?>w<5dI#j zk#H_S;9O`a4}rH!uTtd*10?1FlQ(HPnmN#CDN6XcL^VWGP3O|f;O@}cl2VDZm5%*d zkyC)rkp%!scP3`KQ%DcnM(1~)p#=mv`hsdD&Vyi_-E9|4PnFMazu~42OXAJBqw%Bc z+xttoiILQNqN;Yc9jEp>wTNx}~5rczD5- z>}6zGlkL9+8-8b$Ayb*D3`_I!u>IkbhK`R!3+n5XWKQOd%AvrSThM@jV`4T0oXLQr zx`55jF^lsdJGtORw~+^QViyqHC@q_vu?uv@hT97#?9qbfq6xde?8@7f9ST%YZKD9M z_u|Zu%R!_g8yQj4_k~OHp_lv%xuxA=|4{T7Kp)ex-gF6f>1+uvFPtowOD79}50k)& z)J@^#z&^!(_#?502X?P3ZM)Ebb(6MN(33w4A?5Hw>B)YXo~$SshZO~-OWa99GsD`> zH_aSAR5SAnUX*4Yd)7mkRnUx8P+GcC?$!UeVz;0h2N%2{^?y54zjH(WfH)QTotFA5 zy!tN~s{WM)Z%F-zW$N!LsK2YAwAAnS>ir!Cb#fvElpR}^QJl+Y!{0pDyo_H5K-m~-i3ry zJ|4UKY3x3}FI5s~XaK;{C55CWe#Y)eSC*OPQ?hlCo!tGmN~J%p(Y?jeHL~@{@w>r( zhkvqIqVUcjsgmq?s@coQsY>=PiFVq?{&3jQd&8U=6t-JNhwVUlFyRo#Ab))wY}rZN z!!`aWdrRZEuxuC>Ol%xISmFLEWwR3OElu+ey;9-$<_oP!-!36a65A1l4oHTI z%cV^iR?fz^eoCqCj90N`m7VY1eD3A*A|Kf)lbj`dCp*Na*l)2z4sT{l+^jp}JI&6x z*X)cXmMPS5+5PN{UzD9Ozwecu@f+UGIL*%Z9o-qflfGRdtIW>WvT>Z9$v#{Ils6)C z*P|xHpHTOdmog({zv;ZYNoMXL>R__q@+fi2o?CmB*y@gt;SoiO0_k1)_G)UEY)RAb z;M;)TIj810-}#a2p0eN&(ScyRJD9K)QEEBc6#aNO94wc=?1yP2zbK8>U&2OSvzfTq zlmIWtESfxq*)US+-v+PQ4Z4I=UXZg#ACmq}_>UYaVIT1U1EXWvCg-~CWiS~wLds-3e+iE4@FeZRA$zvEDuwFs;drg}40qQ2UD0Jbq+kDLtPooh z@~~Aq1h#6xmcC4U4_SjrBrnz##m}Q#6V?Bwmi=**6!?+Gx`=ObJ=SAi$M@w0*biP3Y01;AVW{I_Zx!M;L4WqkGEMu% zbU*o42P+NF6n~`Qzm-|z;9C54)SkkLOgp2#$?F6guwzbcVoCWmO-Q|{qm^}f$Bu9Z zpQe6}9TDFqvOul>jev3mF9|rDBx)tmXeS8tBXx|Y)S>V7suB7ao}fPK)B;B?N@4^3 zZs6s50w?B5#r%{?{?quoHy3|F%PvEVzmaq@oR-!t=NUG0e%`ZfRR)Qcy#>>KI29}l zrk@qG9rzf0kTB0Df1N=ic=B6~J4qL{PO4y~Ig9+dB(ex_UK8d3wqjvFMYq&cnaX2hk6l9fbfy8wK(NI4)+}T0rSPdCGG)Jt2bmmHNIuGEGiWjU5j3*y?=M}5_ zS2YLB5d=e0lrym9!vuFRK|Z3g5=prtG=e+k{-_*7x@oCUcbd`o2S!tt9yQk;<4lj& zr=`XrMGMLZ$;Sb{=Yulm?GNE99H&QM){F`{Qm^L?EHS<4SGSRCjHKc}zE55_haQtc z5+1`wO}KTyN{)U>`T;uDR3o=|{K}lC2i?~S(gan2N5B9F-AMwwGq8*HS*bI3Ca$kY z#~XPV|GTpl5jbkIj#l6$EddF*E;mfE;wQ?15_WeK>=8(2iQvj_c)T2bhhwqJCP55-uu;RTQgusa=e1BU%io?DIWm zW-p1T{r7#|_x(Jd&lA|+nai0oXJ*dKoVg2#LPidGQV5w)y5EdUd{I?{OUYR+*}2k4 zYyMTZfr^qBYpHZJ-?E&nFY0ZH^0xZWNU}ZB)E0=AY^mKPY0oE1wmPjp|C#=s9x2@7 zyuVkktZ~_MnK<9c-Hu|LUlz+P<5>(7Lgh>LyAk!3{N&cK`l+G@7dftEKU=^qV2Sb0 zZrM}DHD2y8Wb2vw6Q$8dV2m1$#%K4z({!w$O@>MXDy2|~Uo6LIbaIl6`r_W>DB@J< zu-~rfZAylOLvyseDlhL3=1SsMifph{&9ZfAchfS8=Qxlko(Z(sTnpw7pmZce)5d}g za@R>+izs8C1nqKX;jxtQbIhSmF+$r3E`r$O~ z>E@aB+kK+bRcmkhmo9RYmHa0YGt{ny&;hzKB;0&eUZ{t!e#>cZ!Q7hoxeiKut1nF( zOR2fw-|?))`j?LCvPkt?PVd|ShmbnWAK*cq4%rh}6fN8^?io3iUqoFd6Sn#TxjgEb z_`EE4H2o9)ln7#HdFpq2`(CCWQbnrrq`AtXSUO+H5At=N?2}99YYW*qpS8H8|0_S# zoMnrJ&Dq~+nlxuSBxje|OnfSpv)ml?YEPe>?MdZq2RYl5&eIs2UC?A2(Iyt-Tdj^6V> z(GPkgXR91ews7dD3er)zTy&LC+}pwS|ZGjYwx{G7^LJi@c%SFf1)Y*|2sc_ zGw}cA{FFV?pOl{uF%g`~Px{9`<9!5`rEYZCcT_{N?W|~yoXck=Pt=~Y3K?++7?r7O zpuUcf4Bm;B!5mmf+q$G#e{>VNc*-JQqTqc|m1#xg{nzqQ+zrL0hfR0n=hTPX=i`?! zIhl8vdw``FcSkJx1eB!LW)FRjTp)%S87!5ybf$Iyxtlx6pJWIqG%pS*UAk?@X=c96%PgF>Y*CW+fE{psYT?x$}?U&XvcbC)tIxX?b($f8n%!|f3 zW?k}b-8kJ2kXcW3r*u2yEeO!MgPZ7Kzd;WJvc^&5@)x9d|0&{qt zg98Cuc$uSQ$IVA_&XF$c>Do6VybZ{#4~GISjKnYPHu{5TuyQiirIWlezoKO4%_q?x z-aJ~4B*+uA1m=OT`a$xrPGA5N_WmJkLZEdj>8RZhE!nhmY9wCSojsq&ZUdnQSwrtQ zp?5M9kA!(q2=(&^2KtB4>J5mNriB z>LZX4;z(zeG@+`LQSSbgo5?`YT|?4Cps}ia?~@r7f*gp(Z({Q8V}(WE{YHb@@l>(`mI*D{$1xV@`rP{ zFS#^->8MP4*w;z6x=weLDj@|9xyXb|^KX7b@>wtGD|xx$%aRO7tD?+lBz zo;sypr1WJ;Q;3}*=vQdUsY=%^9ihBYB9)H5k$NbVE)qY1%-osSrPYmLC)Mo~6u%xS z7<~(z5y(UJs+H=+MZWF}QL5Kpq9xDC4d8UGftCo*+Q@O6Ewxg+YN=h1L+82LzeTtc z(cRr5Thz;Xtxr%9NdAzBgxXPif&?W?w@5&#SelCGEAxh-yC)@dok5dg{bLC&nExR3%)R-l7J-Tf3Fu4HF<#e>+Ew z>j%-GI_^i!Wb9ZhWH{g)LDH8-~+o_C`36O~el z5T^c|5Le|09U(VyGXkJlC|H*I?F_d>oa8s6*ty0j)z! zpHthYW4!QZIFrpjp|U?Ie}xBn?eB8jnP}2A1ZD~zwRT{%wOLm`Ddrr@)J=eD|D~9@Yb&tG`>3_5|pEEcE%NlRV=MF)x zlIpL<24sT^ZmC}!h@EgzpmrRnWfpZAsHYfIjDpG-5KY0N(x2@XnZ7^DEPFMvRJi^6 zM0Cj~NA&qV5smufh-^*%bM5!2$((jf-S=7YgukhY61Mgz7mT`eCy!3eU%5Gz)N595 z&PpoPiL-GLlhnzeMl9-WB(=n#Vw9wk8hYzXIh$4)I9#i6QKnURqIPAZWWWEeyF}>Y zU+l5`RLI0oNh*>7jdv2q<-~D>m>S4(e|8iTIsB5v_j$efjxrOUVn&;o2$f%&n5Z-z z1KIn*nJ8%z_Sde07inUi_@6g1dY=TO2yB`dv>t6@Wcoce_^?gPQf*C04c^+sII*Mm z&Q?cy4UV>ouvgmFrL-cJYF*ecm!7E*?ezbn_T_m77ypy?<*(Adh!RmC>VhNfi?kp* z@pB6`AQ>$==wJEkR6C;`xoPlP*q<8}{on{Ikpmi~LLH|PAoYpHklqJ*H1$cFnlI={ zDQ35SEi;R;6T8Z|gWAdy7i!SF@6B%%Va-=!!@G_UD90I566pnvyw=fS^s8`pbZv{o zj$^D#fqvv~UPNbLJ8;!HNRDZZXnthxqO?E5StWj3`)c-oqX&zEyoE!6wB7~&uWZwW zsw16b8)_5ne4y#I{*o=I_teWszBbr$>W*HX*^~A+y%)K|j-g%2Upf^1uHR&TV9+s> z83P=HeldWXkB`CsXfl9FCUDNiKM()FprZvzK7)>-S7*EJF?xCRgtGjlK10aTpgpQU zcP`}eYx2_4K}!sv+60ay7VgO&B8oROI6f+~S;Az^}} zJkT*m3tuK4k!D(lT+?P6WQi}W%0i4D8D^;bHoDV=SX!-xn_i(MDp7Yu85<~hLa(0= zI#ja+3{8v85E^Xy6HEMac#=q?M@vfjk{E1B3?aqjr}X%U%dAl$dK_uuP(UmxQR@4d zp#+3*@0T4B(;+nPDSe$aceJfc*k4LmCCflWLVv736G35pxguVJ6tWd~3)*{>Lzv8s zd`s5VL>1g6vmDIxmUP}er6n34R3XcaGTzDE%w4)$ySZzc83ftC7op;DpMn2^%@w)O zXqVl5NY92T6zEP7*perg|F`tG$L`;d)4(M&yTJpae0R5^RB$3OJ~@dTGwwtGEZKMa zx!Un}R>({2m*!vM=53|z&tDpnTdB7a+LGQb1Jq2g<(rv8l7t78kzR|)8X`R-OCv(r zw+VBF4EMIkB9zf^s9NKtYEiGKg8`{r6GutiqFSkHWJaX%iBFInh!)Z;)v8}z|BGVO zkl0XLm1s+l$N_c7T@=eZ>Y~^%M2Qm0$;Pa3kBtZev`gX@nUGt@oin^3avYW9V)q0p z2}6K^b;BXyuZSJ{F|-PI7tbC<3wJ|9^h)c);&sgd>w?WYdjXvH869&DtM zq(*D!C#e-KJazry;^LS{e@;{t`N)KPj>yxF#ia5gx4qCDm%9t(rVfW&EKL$69ppty zq0yXpN#?C%X!xmo*lLh8sOfgn-#+@6l~-gn5-(-ENi&F`9gHBm$ZNvLjWl%#tIiCh zlBlTD_(5;Br<1a)pvgqzv3BimW{iQ)7d6cD-(rY@d=W zKqlF$pfV}bzoO71kSyq=1hP~FlFnRPdl>*Cmf8ubY|_@h$)#deHsfRc(Rd!K6Ms(p z0+HsSPu9!McM7?i`uW7S)#YUw@)f?~6h5WP|CAOuYo_BVe-1I_SMEXqs?f-fk8}s8%GU6}q%qr|i$>!w`@sYyrFoper zE$l0L6*k;RGc+w#-qHe5nkbNtu+dk2U=Tv~QhluYa6GS3Bt`@4@Y(aa(^0BNr0a5Ch?w3!od`ipWm+B2sGsRZLRrUFVC!j8PiCbkd z_}#B6<)by4Hm33s$Yqj?a$ekt|3a^zVsFqMkyXN`uiFpEDy8f79hW-dVRjEuV-GkT z&8uLIeOn$LdyYNQh@o^J4Lfvd zoOpa4K@#7Zr5n(;%W^_#;`fzmYwpC0N*$=3COwR+sLmu$fPGLC_qUI%@1G2=ic^=E z0G7prTI|czUfAh09muc4zCi6w2Togx(D;2_UjYtzI%i#8|z=y!lt;-n^;#7hmExnv)Y`jBqv*-mbm0<$-Aay!vnR*7>Ut5txTNTa){?v z+L#uX3w}zSxejd~h6X=}_simy+(7?}7(1M}`i{#glfi56Hf69DqS<92k@U%pJd?n5 z8N9~~;5H{y25+d{=En1CAvaFF4niHxLf_`b8jPEoB?8zEFUT%{%05wk%}_~4xuQ>$ zH)TgTFBRo)=R-L@gS0(#WBhBCjDlj*2Te58i(^nHo|(zuFTX5NR&*zWTgAlv^THc8ATHa}IIJfCQzlO7su*-fVAsNl(AQB|P-z*9o z9#^J35|U2wV8@JjEJ%4go$~mG@%Vnq<5A<$aK~f+A|>qKX(piC$#Ww0M>TzrU*BIB zX1WHgA4I?|@A|=jWN?jQO=l7lqw_T!&5Qd7;NkR+4}o zL5TZqBxo|&g+(?iGUHC$$4&{;3)8l>kIC`ymQ2HSu5S((2a}*b;ZovzjIUIad{zPT z9rl6Y^Hbd1eAj!QV6^)ux4XWR2~*;0!GWfhxwvw=B9I9b5X>bhOnjO61SQjXW%~cj z{PR4HCz`{Fy&GlM8yl8tgLg^xx|0Hyc&S_fIn?g-04K(_XNC>I>hpL~abE3o&3$Nw3j$bQ3 z;(16$Hyzpf%_oudn{zRHFNJ%Bq3Is5F=9X(a{DnQ$hOOf zdr5(;GM{&o&{-BFchJZf@;sfm|JSm1bWNfwY8@{BaWct;Ocm~jngk(-0;iiK@0Ts5 zT_@sucrRs53mr}2i$~>n9yfqLvnu;V7x$QufmWsN=GKFbvDm z`ZyE8iXgrnVZBya(pFi_PEHCRENz8=pVIbldf2u-=`HDN{|(9L+xDOj{O@RcR4yhg z&xi%)Pn8O7dst06j7(_T^H~xhOHmnlICV{??a}VMchkc)QmLk=QgYaCXxFBvk^-gu zP+|BJv#qJ7M^}lrCQkUd%)d4yx-Pa3yMMs8cXp6V;n935y;R$ya?v%iw3uz8hxJq5 zw5S`Vr#BQD((%L(iDY4W#<|hpdmdobX|Zn0MjXX*}Gj zmTOmiMC>cxu0kF3kEz+^AG55p+&`wYm5;NQPvvSpGgt9hBiH=`*De4$t~Yl843C3o z^L8R0_e@7~04NGOqfk*Q^{s;!swHDbEN0pu)iX5HS$O!*WI;#sSPYGrxw56BxyV{N znm=z*qI4D=&BfM+1}1TIG*7e+T<_G;T!3XXB+6V!lv|$;9Y^Qm%LyA4GJhN0!Njk;pMYHK1K;e>6_J0D1c)W-i8+5!d}g94L`akuCRzTv~qb` zWAYoRqcohf_qfWE%;L;cx!^=iJlsgliBS(ElcnK%>887tkW?wv_r-u<$gq%{mMUh(_)-YCHyW_YfEEL?1_#OD#kWg7+$V@- z?$h(j{Y!jZ-^MBTIWdp^99tRuSNvVf!Eu=9`r+DUSX&aVT`R&58`>*r;ulz*}8!Wz*e!MI`IS(3bTudI3_puH^VaZca>%g*PC*(TInjX)`A|!WnNAxRC5O}-zK8^$E zUC&yoVg_L`h!DiG(-MT3e@UmMvGl2T%lvmi<%y)Ztv6#7MTUIA9$2vhFSQxQ1e~^>`UZt;a*=NP&#_F*5~p)%PEpLqT^m zuf&`GlMUQGPQOX~7!YeW6(Xnw$2m_WoNm6+PP#yTQ7Z5U;G z2q1$KTpBS(W0Q`C0l-(0yR@?S8P%77yjz*7nvY_0n?>2MRWrB&T2eW^=V9nuKcC`? zRTLu8vyaF76Z^$GG^{9AT0^B@yaA`IesR>ibicUc7>y|LwbjxuPI??`xoS+ob1fPA z#rMf5Q!zmqsqVfa0h~*d@jj_GJtOTEmIac`bPHCK4F2I3?5^)4h)dNBckASFxM#hm zR1m|VvR0W(#gveNY$XP!4CFO0CXOLkDg>p@7nk$lG9HeXiYdf1VQ1jQgoCuQ9pMk9 zgizx>h!#)bgr%hxJG_c_$bfE%wj=zWhd{~T5@pWoz?`t<`Yy+mX6WO5$PD-4c;SB1 z9ypK*AdKBYpEM>J%nz-yV}kM#Q$qfF1#v4xd|a zn?J;MM2wO7v>dg|3 z?-}N?lEDu(PT$|LFag1g40>0d#KQ!T5g|Ii@EU$}e&Kay?IGNWnYm`~R|wV-ZWZ=% zX-fu28`|Gf+K$rJF~f-ILn#NN^4ylKBr&)&j~Z&PwfH{i#(c+sO!Kpc2{2)($jljs zW$ac&Jj-NmTSlCWHu=ruSLL|gy)s`G3#W#d9pUMwr3stL(h-&x3wD>8+@7P{#kXUV z3Di?6rHy!Cl&DDA>Q$M)pst(GZF|E0f+^eiOl{*cZ5^Mh*YKI!!smv@j0$EZ8ts?$ z9uxxKHNsmw_*{z=K)_^D#JIhNn-lY1jV&4c*HSF5?@Z(MilV=Y+qlh$VyNf6Z?jj^ z+lVjOvxCVTIk8YcK?L6-<1m-%O$AVR8&Az9rm+ZGWM)|ROOU3GGc!|?QTHX89j=OW zOxZ=hkc`BxW?#+??B|r>lh=!As`f|w!&LdZ{yV7=;BF@;rFMkw0J71vH4>lNmiTIs zar_dFh7o273KGk#^NQZivvHQSK8B)yk>$Q3r{o{?#b*0@dVb0+(UX*-)cA(e)+B>h zksa@skBWNwc|6>t-LxUkVY}+{+_}yBA?QjZI9m7wyEWPA@#n+|OgzYdmpL~n5(hwE7^-8HcQAGyREN-@x1o5v!l&z-}u?R$X{1*PQImI3E zWib`XX(m};&uIvjPbgszAmv;1nGwr{m} zz2&(2gz)iB?7Py8II$7A8l$&dBk_d=@aDi^UQgCw6kdD7K5qF0mih19#I2NxnR>}a z50d^UCGL*TCw0k9t~U?JNe^#HxBtTKj^;{AB=O|8OZ2(5Dn=tS#o-c$NtDZngrtd= zsvmo#M!w&z=j%4#3tO>fX#WMlAe#on zfp`{~c-CR1-TDaH6$#hdPYmI=N$5#NXA*3qEFrbj4SHeb$#L!KPe&qJ6+?>dC@VWs zY2)`*;zH33_DO*zZW2o{N1~z$wn}T^r5eI{a9Q+P{(zaGWA=`RB;9ZH`=3NPn7;5yxc0qlK z20BRtd2bbLjurlea$Aa-DnnYO>6X^Ixf2Uuy8*il*keG#fNleN43MP@0^|dz7e$G$ zBh6**E9K0z75VSF8@kd|$V#5~ckto3gSI=dxDf7W7P^wy97sIpyVyD1S#3`t%-P;&iLlFZ9`tS$M!bSsTQ;t|>kcUR2& z1H4H%Oxm+T6)<)&}ULccV@t3f>z8z}bgSn&m4U2*DkT$(-O|kFp z!UV_N)&c1WNJS$#xP*ulm+cs}izMT1UmcV8xI}f>Xi_&~1LJyB&9Oub2m>4@p z(U##c9ZWzS?~~M+NuZQgy`E^?=J#-af0w;5!ijrx1Rb3ugd)*~WbM#W*_ycJ7ip#X zWt@5|)wKw#%wL251I6#9RDT60lxilfW&S$x^h{N%*?$_m*Akz|o!ao8(29z2wQ$ce9fJqF# zl9jIamq&_-eO^47g8(2YVyv_n9nBI@aLLoL#B2mgVn?6~37ri7d5Ktpk0L%&#g8#n z+^>EU@FR)e#!qXKX;`%(W1L0dmqMDx1(PXTO{bJZG0NPDc&aR{ru^6NTbG`E1p)io zj|6u*+drB(POHsy0h3ryaXFH9CKRwi%i&E)m!Hx@sboRMO3jR<9|d949$26P&* z2jF_&LI7jU4`F8H$5(jhX#RyYG0OF}P|4@}?`&SDI6~tsntRtfkr-)E)OA}H{u+zF z9XD!$L_=-jOHOC94j;)X2CRIws5Ga@4CSDF)I>oi8|0VIFTig&zXE=eVWo^c@;c0b zCzEL*SqT>JBD%}UQ*Df>O_@;|o{jugfiO=ajyJD?2SJcAU5zOPiI??IB59-?^2jAE z2UVke9~~hDam~8ZK931z% zf+KysHVoJ7frjJ?b3Fn~$y14a_h^I{XCiDQoFGf^c<@FOp*SXfuv?UOvvlkJQJkDY zYMnGZw?m(yrA!SLXJ{8G?R|@+FxG)Sl%Gm?tAvd;w;8Yt(9yh+uV#>I#ng=H;=Y5; zf~*@M5cd>~@M>aS!><)_mpqVBuN2Q(Csx5irbg%=(_BSCuw>i@hJQ@FMr}F;j9;VL z!YvqE)L5|woHfRE9j@yzD#JBWmphv40U;-DwEI_0~!Um-uEeB zw^Jn5@t@xQF6;1u=K5a8-G9Yqaovn-{2p9s!1&4^HB(KE=aW}e7$LtvFv6ar@m66@ z)yq&vn86Sz@n(&anri1Da4})S1yUi8u_3c7#+tKzqSLFPP(wfizE5(bdq7wj`HS7}geU}I0lfiST zV$w{ZcNXKgN*xcT9PeupUR>Wz>ijy+a!W$}Oxiwe6>0R$zQsxGrPYm#PtVhhix;wS zu|qa4{z>hH9KlQK#>HQ&y(y_17augpJ+cwxALlFa-SwA4T-$hB?9`^n4NAMtpihDJ z2BAII(B=U92TFUIq5Xlu~HUB56jf8I;qzZ3aN1}`PW zgm_RwSbhI{N+fuv5((A_Xg`>nc;yt)yj}r$g!*^~s}G|~c9X4q4Dmp@@Ro_AJ7^b; zxYk(L+dmf#x(I85#&#t>Qj_ph#u&1A(+fI~{R!O8zXg}(DZ^9d7#%gLJw+yqyPc_)hEV z`x-M5$>0gXxf_?s2$}Dwm?X##jFL%=>{+e!Y*pzDW`9Dkx|32E_jlPYSR|*Wc=VmC zX3#`M{-F#b$zXptcVcI()dCCd5MWWNQz&mp{IN40QIrvP!7=%Kvw)0naeZfceup}? zt4Hsx@tu^S(XDCl@-(#y7&m$Z|0Ve!^PZzlhO2#yv6ZVI%+)BL5fnFgwE(y(MYfk{ zC?;LlJhv0{(&8n|Jzh9m_D@CqN}E&JD%WA=!1i?Et8-wD{Uws5bf*)!R0wr%Vlud6Hhuo|QEX<7w?QDW z@OZ8F-xZ6D<|i&4EE{qS@E zauy!cjwJU3E~ctKRZ02&atjg7i|FeSgPj!{6r<#ohF7`48E_n`>v~X}xNpO4V7__r zq2wlSC9zt)nx$dsA8P0@9tS~Mk|LSWV4$Uc2!!x_LNM;l7VnP}DeS z|67ut&k^fJsY3CJP~x|S#65>0u^KP-;IK93_8Qf8M6W8f>6oI*KuVk=S@8=ciE458} zDR{3U`N`lZ*Qykjita-A#Je9JO49p0A{N=BM&Szb!-~8gqK?ZA03#kcASozoEtdWy z9xu%!=-cIDzE8|_q{(1qm1rD{Z)~K+&{;&dWbg|nQufBhOCgXr6jagwqgZ5`f=C7V zd_^7tG80^`Z+pU2jN+wrElmfK^+z^MT8C4T2hBN1F|4#{`SW?6rI&EQmv4XG^Ev+T z)|l@B2?GyBGR7)mcgzPLkX{-#-)lX>$%mxK^(_U9_03_fw+d4e=sCRc8S`EyDD-L zZ`CqkHdAveS`w2e0L`Y<9I0G==KIt8l0`qA1utNWZt~y4*LI4nM`Yz@hF6@u_@h|# z(K4_!;9VN9hJfRCv-`Mc>8;WD89787jgLMKFu#QoTiAoj>51g>n&+QIse2Of;jBO~ zD#z7C1o*((gABU5#_;s8;QZ9!94C9WvT*JIXOlO%pM;y~$J*~})s!ljV)oV(wGHlQ zUPBtl7*lX-F_7~z%m%a>upNM^EH;$tPH}X3uf@H+C4v&GbzM-RCAY8k-`h5Bv%cxV zE@|D9G;Xu&J@rj?V0(x0WxI8$)Y^v3n1J)X_PJrvpAv}>SGq&CTQd=XoV&8LE>iah-&3@Y6 z1jrGPhZ_(wpum741GK7Bxkl<;Q!F?fy}kwkFnh-2W+`)-UO>+l=r2GKjh&p3fmioiM`uF##rK5p+ZxtWkP0gUX7nC!6kLn}=CAOA z{0WALcdmHEX9#PnmBM90p}7d}d`rX(w9rp+V~-VIseVp;#PfuWFY71s@lkd>!EF3K z2?(vI()#Z!vFLp1~L=KDv3qE);{F20tM z%Ngt7``&jY7cZb#6F=R>1b;c#M@u0vGY1Qe{A3z$netISO4gB}D`F2=84#eO<`5>t zZ5JK@Vb6@01yNc!>Dd!li2v?X{P%4aY}YqLS@q-OOCS9=pm`gXqgRlFtP&Svt=mZ(Es;IUVQxuv3Os{m(*7> zsK)nX2nl!QI9p8j6oK>wMbgzbv(E@rBGlpiGRzcxlc2=!6Cy-0TBCU7ax3W+mBR=| z;@gc}5ML?AQWJGCiYR`kq@t!`Ng@M6sn=8$K}HxxJmSaV4uJn;f=n72lLQkJ+@U)+ z563KnppbSD8j;+6ivItOWWtZggv72_lgSpj1Dw@nwytxp15G8HHWQbeAlmNK; z`}*dDRD{A!I(mFeCL#D6$!ez@Qhao}Obn#%>P^oYY|@eax75?+P9Uyy0y(DkLQYv6 z(vkfwYHvEEBl|@Lxm(_CX#E^9SFjp>R7K&xBJGEA+4M)>lUR4Wy1!8$lvVPYio%VC z@n;j?6dxFGgV)4QVI0j{ba@#<4uF~rA!Q7}5VDAUr|dVqt<(-7%XsZRYYbt}=W@PD zw-u0a7g1Q;3p>N(`AtqD{*DfvPeimiCSb=+1z;d>$>4RAzY`np!I#m?EdnfxH-(aC zP_8j3|BcwBP+t3|rSdhRg35#9$GFy_oWmIKX;2t#mEu%dd~^0ta8s|L;NQ5#6AG&h zh1ppY;)cRghovB=BxLwO|NSFF;Z#H6`78=k4TXn4i^5z(;Q>P-Y$?b{^uRxCLc6DF zLdS?-e8e0DlcoZCjjyLM&D6yeiOi*Plgx6=I+f zm<|9v4tiOuG#AO>N#v{4b#Jlw#k>;)Oc;MRpkCJR-l2>Ld8-LIO+!k}N(TR?agDE1 zpWtTjQiXO#ik+7X$}h#w9dRnovgCXb*V)l)ajVDF0=>XaFO*B%&jwh%IBvebZdd+% z$A3qgbiN$7L-(Zw%$4bA=HIM@PHg=1e;2C0HUV_(emV1cyovA^>g7E{Sc3$8e()B{ zpJdmMFpFr?zl@))yagY$-cl5IS?^%QWg3y#abBV*HljJk>uUAVohS7)`>7o#sr!$8 z)Jle_A(4B|?@TP^EU4@AYEpLBkLAkb;W9~M?)haPeWXYr zKvebn=q4xon5`rzj8(%YHNXaz!C{Qqf)jAX;;=pER;D> zF5p+gPkQM_wpUGFqu#hi1N;M*2yh$52UdQ0$Q zOy$4g8Og!uyAl8C+c8XM4|oj-vVPO{uK$bM`E7x6bd)2t((xDgo`4JsWI~24{vB+v z%p=8z-wWb5MD-3Ft8h%rXUsx^en>?P2!~Mcc_q+Zp$M+NR1)CCAohz_Mj59Erl-olz{sMR&!rf9*{@{zgD$HFGJONh8 zF-e{zSyJl{kT9_jguxoDOG4CVOx&c<_XtFkVx`FelP+?g64b<($!dRrSsk#;B5rUC z5?UUgf^K0_3i2Kj2HqM0V^sIRw$9i*!e2pi8#c`d5CS+!ew5QB!6Y*(+3XRpfFI#rt&ngk zbU)4*f2-8I_drePFN2kJY7RGQU0RF0ujAJV$Fn?Cy$*AxSdPP1v14QnmG(LU$g7*O zW(>EYX}5}D7KS!4lwx31uKM>LyKlh`13JRH$PV)1!Kkq39Vd~;*%;;emNENAz*V^Z z(HL8lC2w)4bU_P(~I9#Hdi6O++*}ZHz=m<;h!?ENzp{^-N2B(p9e}So={sL2N zwP9)CIwIc;W;Bw)pTNqv&570QXG{F>d1=hk+f5%HEnIS5Uz*z$8>Q7%EsEKA$8o*! z=Wv$hm%adUhlFi92Xg;-CN;^j0yMa>h0#%=!tzn0nm6Aup}Fmj@g=^m5w0P>#Ctoi zess*YLg=`@O^ncE-UV2a!J&q!Tw!XkG9{fRirz`WQqxAKR}TR{1uG7!@O}BqB)0J4o*wmBj*CsLYej6u@k_~4iN&w_;6PLax{>kb8ahthc!comZ+?<&2EE1Er zcQ4T=nO(NHk*1#G>xc54-thOtPu@6`az`?0i`jM@U?1FtX)-Gg%A87VV8uZ%aOwIL z2hXNytY0DV@}kMFz%$A$Y{0Im*Iy#KGK;ye0fJ?H;#w&|k^3!q>c?3=RO@^|n2$DCfq) z=aUt#_aW?L$a;_G=|C(zg{>9mCW9RpD7oQ~V`Md9E;dp;v+Oa!5*ex#r2r>Kcwu&w zI#^5^8i7R?%1nXSLjse*m5`M2Jdui{$i4K5@d|E2O$P747j5adrrZ5P>qm(z?XfVS zrkP@zFzHg}DeC`}_%^p|ceN7Mt+-IcZ2if?rFpuU@dPK7ZX3>6!j2*KO6F#*NZ9hT z~M3X` zHGnBGhcT0ra~rO*>cTvqYHCfHt0puRd%yD^q{qLqk^FXA%|j7C8<0xV=cu~q;hHAy zBv0RGb(n;W(uCEXbhw1g64y^o*bB#fO2V`}^ga>$&WT>`e9+(YOE{Lxm`a}ZfB6T( zJvW#}3J*B#Nfu3aG+Zw`j0??8acnKFyDCz+Bho55B!44iwOB87~>xs$Cv$Dr|Fd+oL7zsJJ}Y)X5tgTwk^$QQ?!AdGe+C8UM=fD;wO5 zGpL7a-<+l?`4pBY5?V-l#VVZL*oU)gxFj~inXu`$=_4I*pJMLE7YTE#C;!iwTQ^9U zLyj1O9HBSC*nZJL#{@k&rzx+?vpA8X7;v;2HqyFUggZ0CR*&oF}F_8JXkJq1nq11hfm_5h@ zy&@@jk&|;*V&XbXdETb|fMjOfFsG3F3G=w1&s#2NDz#3@ zR)5o@S_-Aw6h4k(OnmSgEo_rF#1ezYNJ4zy z0ZCg1Z%jF^ow#2V<(EM zq`y5v!$ws%=})(hDr=ea)4y`h_ZL#>-x8r|l=Rz{F_PQPRqxs9m-mwyU;X+on*KEa zp~$Xk@{W!`A}rjVs0vDweKEXE%5N5vdP>UaalJQ+OLn7^O26+r8cNbH;nU@xoqi!u zpC*u9{`18(tNd;HQ{_LhPx+&&k^V`k^lRhzspWsUDgXDG)PTVRpm+Jx`PlN$O$A%3F z|CQaE{_TKIN&l&maNlo1GU@+;n0lrEesSrO{u0HP^pC;xiRmAf4w{|*BgFOJPd`cl z=`Z>B(|?Xh|NSp&`d0xyCH*oCqCEr2r2k4W^-BLG;$qTY&Y`=I!zbT7Jjv)W42%Ju=oYj}0-76>qr{Q*ZC{9W;W$Ka|~C%tLYWf6oM+#rMjMC{FB%O&2WpY`WkkQi0z#{G5)v8i`8&9M@aB zORz4ecRDUG95&r)?h!m+uMl`D@y_|?QoG?PCURwI&)jBrqeg|&+r$eSSk1k>=#fx-;G*}6IWzD;S#O3r0G zkd~X?qG=f3u6`MY+ti+6c%9lU!#_00or!(4#-E|w&;x&r|4+&9>`hax+4>&*kJV@w zKCX&h@BAKOI4VnByP2c*4EygO3+W3m-%tyy-{TI?ejj~Q9ynq^XLQ}GxYByK({wOU z3#U9ps0R(ZYp19awj-TfhF&F4@s_Ja$~8!Z-CRH~n|NgLh3jZV$JbFAb@bZ3F=i9k zX?sYAg?VTV$>7W92+I?08(tTZXIbr?8EaJFll2Ww(`{6Raz=SwliAwa@K1G+*D!0P z{*|uEjQ7&fF#g-HDVK`XD{@@7%I07map6$XbBRBlOr;a*N{nQh6l}^XCgkD9kuv|{ zr1gMkNt-Mi-pSy7+eN*X4JYn@TvrX}il5_7rnDZHRl{j&FWgR+>8jxrwKpBqRl{=( z@^cqw{{Emo=6J+ zqx|JzE6v}E&zt~Gp!{_f`3Dzrrd7rme{`O1Q{QYfq{{H;4 z^H=02JzwxKxglLWA~y}=J|kVz6@heppDCO^a&wP3xw}ws-+V?)v;e_pF{$`)t~?n$ zLX#K#lX{PD6Hv%&hlezkM-lK5gD$7W6MN;Zq<9_^ej~mjm@)7D0^)g^bXpb>qnP(r zaqJ^aw!VCFdRh3_^fsSu())?^WS%0B-e3DA?q2oe0ddmwzPv+BlHP5YGU+{<+`{p9 z>OHdH*J$ zZ+f?g3v#eta_*S4?2k@@_eT7py@pUQzpIVoF(qT{^~tGUqJ8l ze)P1WYkD6$kWTM&IFsI21oTbsAj#x+=EZ6_xSAsdZ+gZgRbe_%&LMd-|q=#%=;4oeballI36~=Os(1Z=Av=? z87S*B7zc0*l@%8?c+1^QT|*+QEzA6PB-u8(`178I_qlwVK~Lg=zo2Y%TsYjXY9u56 zn**h*&ec1*>xT3CA}=Dg<;a3PeF9e&-$=G}v1AXI{be_XC?nE!xomrgT{xU)j148D zGSsE?4=N?<#4P$bIy;9{@XTH5DuRF9RhK<}!?MRZ<;Xkk4~eidV$BP@D$}*5AZ>ikRLN(>O6D#l&jcZt2}1_jE_NON<@i4+L-nWDc$n8dA|8 zOBokMhW{ys5Qd}}$QZNAFns{YT(MCSowYlO_L=^kSdQg4JLE5b*DHtm3$_6+%ExVC zk$((jkLjiWE64sZ6G!1SIqzNn%%i5KCnylfbsi-u~DpMgs5<;T52(HpmjFk{#7vhzIb5#+a+RyWuQ_Sc3iTILlGvDL)FTtt-Aut<0 z1GfzG7d#6N$?5=1&hr;Y)-NQMz)hd`7d(lhq*Y?uBnUP^xQo+9jgvqWw@J}e=@jj< zAtXgF@WZa?GX+DFF_}Eo2yAjRg5JrI)ZY4eK64~Z*GqD0B{?7R`-tBmVPF{uf!R_b zw+xp^{8#of99@=54iN-yD*iWy+!m5Sftx0NQd;2D2*Myr(j??6%2+aGGIFHZ4ZEM39Fq9&B?nea za->W$MU*KWe}U8wDIyrw=WPnI7mS(bxN@_{lXk*>Rv-JqKK8Oc_Pw3i;br|ctxuV8 z;>of#j7Q7MT6K1Svy4|ACAEEnTASEn%&WNRb)G)@Hfrq&vH;_gu{B!_>>?uDsCfP| zr{Rq>=O*08)(;4uiC;&{#Q)+@jbE*w9se+i-)rKR7w;wBw{5(%Pptp5eFKp=ZhmU}$8p8nl(n&E%4J|U8% zXC!9s;6|&?cm0<>&ek2^m3#TUU>dkrXy77B~EkO}{?tV#=5mdEENysw0w?!7`HwrMk|RRz>K}e8<$=h%`eV;<%8!q5@w*c5RXx zMuBQ$_=5k6gRrrjvbf?!Es_E$wC@DKzJq z!RrHbrn0|S&T|lo=@!*Mnh*uZw(!@O|Rf!IGLyde?%&+ocV37DVZi>R& ztQ~nC4{!4bo*CD^FK%J^fLJ&73KZN_+q7**HE;Ia1w-6N6A2lWvAxJ+0w zC2$PH$g|$_Vnrswzb2!3$Y`J;I!1_=p1r?x6`E+`3jFTg<@$b)KG(!8@B<7|9!Slb z8!qbbeh&{bvmAWobj^x3K;nkMn&1JgNqFA-HXlwUe$OEI1)&^-4sR8qDK}47e(9q* z8sptt6FbS<#Mj}SOm~`!J+y=r#YrZ|d4q#@)wU+)qea?zV)oVSOkAIZKTPo7(NxtV zYy%TMB1`Po@lH=7Va7cP8UN@bwa0)_!ZjuoaXsmhH=T&!*LZW-&On=<-6!L=~tozUL>g|TIwS1$3rgHpM# zVW`vI9+>}rdosDNq2E64OxqmTdoE?0E&p97NrsdWGXgi16Zy%8{)w*-AXmMK8vx=v z24ATde>S4Dccd`AZQI^=*%y(a75CPS5qy@=a`r!-Q$Jw+a1puZK$2Nfl|s4uIT~|9 z(O4;ihspp85@qrzD7*FF;nhOkx879R35|mh|F3(T+ymSXO&M&E!`F1OHNwO(k-|f= zL8wZq`FANNssGP)y|AgAEkMLl%!8LuS_qe?2y?`~>!mJs1M!?WTYQ`vuft92nT_g?t3cnqBK2{gh;NYizlAgumw+DFPS9Kf|1sG_Czv< zlGocLUahA)DIap=q-w{_8(k#TgqJ(z^l)eWuyJkOj{i44=P@?h{o`zD1PV6r^4k09 z^5P+3oHij?fAAMRc>h^;`i z1Ki2^vMH#Ir`MYvFX(!OwSAd!l#Fh=b(H6JF%6)gjsW)4avLscm+7zWej)J+#WU_1 zN`2tU-3|SnF9+xUjwEU2GB}NsTwFYNzfhmI@|lLgG>Njm7Z3rglNuu1aZ>$Xu;Uk! zBQG%dJ1%$%0h?Yu-`UAb@WEsQPd#o;gz2KXc+8T+@4j686KCQuZgX>+<1c4Yuies9 zdHjh~_4juj-}e44p7s=p+r~!6icw^*NOZ``Nyurc9`12|fA)i~objzfiQZc#T$F5R zSibvdLh?k&U3^qHp(mKB`*yu_Q52~GKIeUbE!Yn$M$f-(ZmxoGc%Ho=d3$0#IE z^7ULpsZ4LKc)Ja8bYFPv7@|+?luoj(KF{6ydmh1!M0U%FLhjZ{48&g?lEG_W zh|97PigDjA=_mAnV=`DGJ`kDcG|gi(C40vAsIMZ)n~kgq7e#4a=C{dhJO~2=u8c){ZNK^o2k@3dngE zXtE(m_Af(E_DjdZ-Tr%N^VpUzkycBj#8#RN&cR5xG+_VkWE1ly zl3S_$$ORJcA?3D{6l;At*H+}5hJDW5vWe`)TT!ev_w~n2&2@%1>}qOVNeTupeY&Z& z2Loq>+qB>)Z{%Fc`?+JKyhXlQ@LN$lx9-M04JYuPU0%b{t`mx{*mH2vn@v6amik#q zD*aYYeSg)s*jd~Fe|>bO5c_I>b(th@{7(Q~=p!Yv^hQm4IrrWSEMgPdrPIwU)oY3`E2{~}njqbb>M`EQ7@)aoVayqC55 zxbGTxk`27^Y55XxfD^ljUH(pfVu0u%+?S#Ly3%agZUjvB6vw?&)rZ-ExVK1+B)2`& zwzFSA=zlcw6X^ep>?7!iPo#fThJKF{G4u)ZQ)^{vVDB}MF>-F^K;m9kd|85x<+v|$ z_$xng9wNJjIVkmK(lu$hQ!)3ytIGmz!$h?DEuQL9x zr~PwNJOo68)5q5-)lgG)g1(9qTHfv-nX!WZ!E(2~!}}MkB2+8hnOjy-UhZxwcb_R+ z@lN&P`h{gH_AOjo9VuJUy|})5QF!3?a(6dpke;UnE?e<#SwsDT@W2<`o#g{xSh4TI z>ctK0c<(76_>6WM{s+-Z%H7T7v5FqIGj`2;u^AtR>E17MntTmJnJjDS%p2W(e$rna z^EN|qQXALI)fY^vI-;Ma{-jAY0*>>pJMdw$sV8*v@eaC~{39FuKip6^kl;CloJzvJcvi79)Zv|jk(PtA9#xp6l`BS|^-?44 zeWl->;GKydZLw6*ZOPFmOFbI>_R;eFhkm>Ddr-e0>h}o6X;t$8Dspo4!|HOEerxpm zCH=0{?_kAQrRLZ4dyjsi3)aU9I15>(|8ll*U-4_|K^M7y32s+4vJR zl-a)^{Yiye#wHD(Z{H!KN{>nqbwTGUOQ(h1;p|l{YkHlL!)ETE$m6~9`lHHtW4>Z; zW#0R{qPaVqxt-Bdc0}fG;2CpDVM+k?pZzEK`^$3u^#lBmmgP3>>5k^2JQK<0tM|Un z$!5`Wb*w^rY^GiqWAU84RF{Z*B@FG8N37kw^18YE6c5VtTu@$uSGbaHIB(Sa{kxRU zFz@KTzpFgAt-N_kUi}dEws3%3w}m~Km?yg&DAyucd556vb)bA5spsiO+_zy z&JF*Jx`ZZ1dZ&$neH)M%Pf#S_di^fa@5_{mNK@)B6Mu3~%FRG0es>iy?tQQSAsuOB zFBSV68GC`)pY>Y**mOC%Lk2oBjWuHdm0|&>0TSOh9-UB+P

    }f~bdjm6apt(*z%l z0{m;LbSX{V6`rNf`*LA?Pf~Uql)(75J{z@8Q@6}Utz*<)h`Mba81AojyHV@lFd@Kv zMnv9%N-flNOpy#lZIc~D^ee}ckK>+UboWQJh8E-_es`t#Cr5vKsN|Wo?=AlWX^7vw zhW2Rh%#ms?mV8+B`$Ngp9c`rNzEThzH@*kIMCpmCbQ5x!VE^N3lWUngPtkwqkLmoL zDVWbfSFlq4<&3p4Yn+|G&QQTkqTIW3eHy=1!q;QRuSLxHChR&f-;)*A(qAp^!&=kf zi^QIXo$%{~cISWfif@njKa89Bi+QavD{r$*yq)46_)V|8nSqGtdCbIVBM7Vl0f_~I z`sb%?6gW8I@a!W>UwP@ooXW9uUUQOt9c0Q{1E8C!Pjf6Zao;~E?_Sie*qkN+eWkgQ zd`QK$>{n@T$?qzOf93t^-zMfno|>gE+WV3*?-BRy#vBs&jmE5b(26<5Zz{feL$61| ztv0x=V!p?iSBtsPn0JY}-k6t(c>(4udRqU=rA}FSm}$(KZ)3M*aBcZ2{;(|kf-L-r zJ3fYAEOFku)3_7IOtG)b*ejL4jJ-@8y{I#GJq^xbla88>+?LhqsbcE2|5PS>j!m&Log`QBwa@&UfF*b z0_n(Vh>TuIE@|RD_uunkOBQ3&S>=q|yrLNOt<8ULyW4KIEoe{HH0fEUIXv~}B`Suq zDBVsCo)iZ}FrDfHGFdS4SXH+U-`Meocsy&4I?pU|5r zh`s53C?j3yT_f~n8hQtCgWhX^6up<9O{I621mI8TwFzQxdK<+XdOo3-Z|JoPnj8EX zz|iZL_(z(*H4=b7p|?s9v*>lPjFDrO$70^XAJQ<~@dncG`bJ`AZMR{MVjBYghktAD zWAw-0U<=>7*Vv1#z2A?G{U1Lv@EHpqws5tDD=n|96f5E(=>NTw~$aEv&cDv2fo5 zCfqkIj9NIAYxRPoHw%@_my&$HL1jyxzj)7Ot}J z>lUuH@JtKaEIiuEeJUMytuge!X<^F!f*%_9&L0?foVB-F`+3%0X<;f{ULSkN##dnB zhc~{waP}Tl+&Pek>eo;XzBM+rl?3e96LRENr##Aqy8-=vX+`!jmmL+QLB= z9<=3oy5)19wWs)8Zuu$dQ{ER@_g`5!%feK+1%2$d+II7c7G9icM=cClm~Y{}Z;dy%yeK;bj({XW^+94z=*j7DMk@3)frtZ3|afSZm=d3nyDx zV4=st#8(Zyrz~u>@IecgTX?;NGc7E&@Kg%}79RSFp?9)P|C`qSl!fape9*#MEu3TF z#TK4pVS$DD7QPoZ^k25{DGS$HxZ1*H7A~;xatli>Jk!DxEX=nsagU*&lG{CJcx1fq zDIMHTVah%FQI6Wz3^o27>`{A>g;f@|SlDJ^w}qi02EWq6Mhn+jxW~ePrC(}cO)9<= zBNjG(X!x0E`Re2SiPn?7-#Av%xAD46x*4Z<0xSA?LN=7lpe8Cz`Wurbp6%<6z2^7U z{Sph0GU=*H!HbRi$|DU-xzCwBFEo46q9t=@*DqYM*yGvrX?V4@vu_D4uD&_6sCw~r z^$Ucp8H;aNyyWJ^q3UIGt7{AiQtO*;7|vyKwP!8OMc-m(E_aa9+r|*DR@9sIfF&YSOXi6qC+1CmYyk zVPK@O7oBL}B3mwPBaFT3I0I{~`!Wlwjy3Kz!wqaa#=s&A_Z)5PJvRQvRQ_5hsaVHv zxb?5Ga$7XR@auWQgr8~c`vw~Kc@qrmE-|pjLeE5F&$n>6g#{LFvv`$XFnFsi?6$DY z!l;EaEi}2^eg5HLqjfjFBA4=)g2r~Zuc4QXC+6&*36+ZXa5S>rgpc05Wcy~*&5qOK z`84{5UT^$QO54Zj^bKeDOYxcF_1_66Dapo5g-yZil%~wt_$fDm+3tOL5&sO$-u(A= z|0KT*{Zu@k22=4%yr1;JN%!an~f*t?4JpjivRFbGOnMVmTbD& zac1%>Ln&i2Y3q%jNn-|2Odsd7Z@B*}f7$sY`Ss~#X7V?cuZM@JJWfFq=5SvVJ{wQW z*}vZL9gaq}+o$uMO*cEv!^ug8>i?CuzIh?`R9+s=FO#2#myh25pI&B$e)sup)BBZ5 zoB`YaH(Gn!%SIoV?x{vYs`rFE^Dxc#)Ou#~px%v`7vnmc-$Kt4%ptHA;$Gufgr8_s z<}ZX#B`&q=v5p1Kfk)xjxCw5G8;|D#<;3tk$5Oe$Vm5d_ zE40TmU3072bDqX`rsr%@CwZ94xiah$cdh4IQc#3{L$7pCZkRv6x;C_U_Knq{Stp-1 z@e8Mp4h@)AJ$K3Cd7(w@83~;^wuB3R2-o{>3Ta);za>x)+0=0L~AfDAwQ%vMJfb;q2|kl$iWQKVE$5aB~-5i zFC_`aekOQlYF>Ogo`h>4HjzS!Nu*=8O6okNU}$9Lz?3EPs>e>RuB#tAec_GOW2Y=x zJb&SJW2Y~vpS>s~E}?>kx-nzR(jyP!Sg~;N;XTThUUztpirLGIM|pK-7}JN?$Gy_z zKJJw!Xn3utzHv$IEg{Qj^}JBQZ|LfFG{bA&o_h!et3_vdGmA>G<(q{HPy8`a+|WG zVR8MW(G8*L3u>!p&r4f{)k|vURo6}$J#XyjGtZv?Ss#gctj2G?wxax>U!Kt=b@i!o zWT+d`EE?O_)s(uHs+RQZg^MWkEIBp7F8G*x?|`Z1C0*$0^6pbUl}nZ^lGrAVUbLto zbU`icOuFopWTd|O#(k4ZUCjSx^^zx+7O4&Ds2^r zzD1rY_zRXS@tk#Sy$Vz+A!nUq9qI~;MqfL3$&EM4tbxaq!Bg_4)qAZWI4^PIE zaqmSY%e@yFPslXTbt(QrksD`US4~f1-3wFpX*be7cs!x#?Aq(9fodcy8_aj|Fej0-LZU&g_a zin0sC*QQ)k_Gskda2dWS`z4jrL5yOb%v9G6lv1Wfn<6t6N85CDkea5AuMpwS_-9-) z;idMX?<`#Gp&RhfW@q6mJw?#EoC)rs*k-3YKVqDIOAS;!9z)IJxn%zQx@yu6AGPJP z>t`!bKBhks?8xGI)yqtJQi1T-^l5wH@l0u`^=OL{SqvHDSXo<*K2uBLBqK5bm)>w; zb-k%GCW_0dZ)#XrTYV$#NZr{}9nDPWE1uS=Pet>TSI=9>oMQF7Oh#$G3g;o{lvmH6 zji5;Lr|v0(<`e!-_3RqKyI|4m>ja@Ze0gL__}b_tQ?wX8)9Po}i@S5lwCUGg9+?)I z998*H`;<$Z@MYnP|5tlwA0Jm$=l?TllBP-1Ch1cWN|}~G!$aF?nnI95CVhYw3Jhsl z3Zj#TnKYqkGR#cc6f1@YrK_kF(XEOSS#U!@%_{76U6qKeTV33Z2)pdAXw z-0l8E`Mp2)+?%;GO?mLQ`^WB!^UBHRobUOb@A;m`d+xb+zPH6o-_W#S^QMi&c%t5P z!o+w=w{E|>srmA#){t0RG8T1?F`Y}bqsNZ^p51M|!R+lt zUoyC@ZBL>n9;{m#a@eN+?Qr&AyPaKSkP`iS!6tTQnf`rms+ug?S4({ttcLjUyt zdvqn`!kg}5b!&I_)t)qCrsSPsY`uoKZms%~#`^iCpbngI`nPxY2KGbg_{Mm9|1MV6 zzGQj>nzXld?B2AK0kU_uFXgIuDuo;+h|ZaF-+o`RKjXHpHa1l&_r{aym8ryDD#T;n{+#i`+W4sm!Udz z)QZMbjcC%Ths2wvSlb>3&A#rA-QB@Wz3e5OX7J>s_O&HCx?Ekv@53%+iJ@1zxn&N5~^_}0B5%>E_;(q@Malg-sknl>5&*oN6zjMx5E~TyT zrcv$ZrCvV6NmG2#@72E^kB6$y&hG5~I2B(5eBKi5A1}Gj=ZdD0r}10UrIxc!?KVn0 z=*Fdo%AJlcG7X=bZ3>j%`1oE;zj2-?2Hl(VKL!6KH;O$YTKi7!I@$TaeveOrc4 z1W^z z3UQu(!rb{gwD)8_NviG5me=|^`GTk5d%8bc-URU=<;Z#ZiNpC}xHT<>i*E{;q^? zy87n!$2k2RbQSTpOYS&0bPB%FM5Z22ySz@mG4kb>t99)BzKlG6bYv!o2c;O!(@&T? z{w5h`K{4T8jqiw$&+8ABnAh&;S)#L$ER{5#K+vMUb(`T-kFi-h=DH*UC!fExlbR+@aaCKjC}s~ z6aR@TKYxQzoV)Q)Z_hJNH`#n%8b3BL^YINkm^bUQ({%kRLW!-UZOuuWLz>^+>yP{w zhKok?!m4+!n_O<`CtS=;cNz<#KX3T4JX3acp5B2viQOZJzZM?PHTc4!Nrvb6d84z@ z3`3Ayzw?*dN}hwHx90Hp`IURfmwrV2k!+dbmn=`VzkJE}JC8@SgXAf?hJ8&AkDp&- zEeMb2!h}zygLov<&94ZJrsNCU1~5cRJ}O_*3oOmBV}D z@#))T0=d&U&B4vvGHGSxmyUA&a!%xwGYFrY_Plb+6UGntWDzy0ljr>MR8PMfb>)fX z=NoU%<{L(j&CpKZ8+`5Pm!DG(LD2 z&*Am|lTIG5zZ~i5cU|Y zSN0KKSbY9|Q`{TBx&7v^m%skG^(v!nU+>sdd7hJ=$9?tk`c?DFUtVs%25GBTYqQBe zo4(&_Tl~GmJfYvF>)*=l*?gY94%L@d`=tz@U*AC=kMbTS9?mISbgyiMrL*+ooeo8( zbxP^-rTg=%F4CKKL{82NZMyQGS36F}?@ExQlb|dhNm7 zmaRI>8b!sG>G(jpCf2RAQ4TT&e-m(Ndz|y!&>Xvuw0wJdYeFFtffiy@XK4dwa6z7t zfi)Z7ACoK-WM3(s?if#VYSxOx9>_UETNw&N}N9KLaAm1j|pq}quw%wF%9+RcgABvmi%gYRa5xU^etxh2-F=G)0xtJa^D$#h4} zmL{i+)-2i$P8nT(T8rgNKw}8lye*b*5Sj+a$^om(;O*?A175sl0Cw~A9q22X?bkZd zj@_B(?8#l)8qrQb^YwqdX!?yqGpCd;-vy#+5QbUQSK1<-{c0IEFixE6N0^gC+%T+c zPwDX&3n6O{Hvo)o`;>5!{E%1f>gjJ0co70*XB^I_W2n7oBOkrJKC{Ublit7taaa~~|fX(ClWvr9J^~O|{g6*^eX1ETm z&odEp*7{)c6r1Aa!q%k)d8Qye%ka*<`5OKx?Y|U-{PDnoBC`OV1u?LGrdd!|J`t!W zG8GtBuc)78D(dD;%ubZFR)uDnWwA=Lq<)d92vnGv69tKUF6^OF_NtVBJ91UN=B^yS z2XNw-o#sT%C8yiC)v3_%y8-hn{N0XD>=Dd*FrxM<=WtJY8TWIFSW}3OQ>)g7MMAi94W=U+m zsji=A<~(C-r8-S=U)YXF$!euelQlM+VlK(Oc(#p~R~4G7*laT&eG8L$36*!0yvmm# zpYZ3w!Ghga!u#i zxbn%pA5>W%0Y&+=OaBU}GOLR7G>#)`AJw6%WQHkOKQmJ3tCwxdQrdF%R9g4U$xmSR4W9TK+7?c}#<&0eHL``I! zEvFL{U!Z)ZDNh%e@>sqpub*McCurlP#ilB-{G_tfmto3MzHztyUO5vySrYxsMq5tT z23yXppxmy@-SVo6m_M^jRSU?tuc}*+2vpB9)v>dglV_RgK+rfV5Pf3lLFi{dug=gp zlqp)DKbc1xFRd;z)#+k051GZs@=XD96I{{tiS{qQ-L8NBxU<&YlNYMu=}~cgxha08 zF!9)I)--f14;GssYs2D}d8V?y!pwfAXtHo3-c?vI zm-|Cd?NW910uw&G&|Hl1tP2JzOl@7SK%J?a zTsSd*yke}3woqFiq@b#mB_^1zr@hyhV0|5Dy(`W9$%+Y&r&T6S86_$r?abh1-4+x4mw3G2zPj&0Z^a*R8glz68n@-{zLH zrQ9?eJkP8dINz-5T4Bzs?__@8Xy#1Lo+utK8Y@f`w0h$pLU|9kx+mOn4uDkO90b)L z_kyTohCr<^Pk?^c>Xv)YyDjy8Q28GR#rY(t{Qv$QcYV6co-bE{lD866`pxgP{6+7x z`QHi3{U@mLRKBX1K0nQzHd!^XXneugyhKH7d8o{}4%AW)@r=8=eGeSH+vaP3zm<~& zrPtR#YU3}0$`hz8HkGjoQ`u6^+$=MdUvNfCqN=Wx$NzPvCbrtt)Q3z>;M|kOTNr^= zjiqMgQ`ebu(>vH_Y-gX*!agHnoEgq|(mHC;FA1o;Nms|y%WXS`KWXW2{gl<~%b@i9 zmK#5GzfJ!lDEB2dUS3;bYU@|A$F0eXqr{9>U30XaKEKJfNysf{or{xh{&R*Lai$(r zc?Io3%VAwrZ<{$zU)|>nAKWD znVNqQwTF|hl6+O)5*7ld>hDjX;wZxcdZJSnpNpb%z3eO%?G?zld9(8$s#c1*%2UpvKc6sPT0-=y$CT+x&kE%9W1T`Y!|JYTWpyN9=ff9F)B6 zkJ|Bi<)|IMH~)!^e-~8xnlD)XPlD2C<`?;g>jtIAyFu}N8k9y~0mb(Zpxm!P)hqm% zjobFPnD$`JtG|f$sLS*-?X7WjkoLIUZI2K5M zxaH1x-0JlZD9#0cYW02`RQ{5`u=T&{%QpTnDEYI#V);{`%6|crYy7I^dn+iueW2ui z7!=>rT-#c0bVI z&f7`1pI!#lE=3g{fe#WJL4a_4x@5h#} z7L@#`8_#Wrge!N@O@9QGJD$Z~Jau2P+2s!&v+{H01>N*xwX3Boc`@;C= z9eG8xDTg!P0nmBj4Zt& z-`tR%V+xMVH9=mOx)gt}Qy#M&dKOLVHe;oLaWXk8U`mSvb_{7OGJXtcCO3`t%;Hs> zry2hx0doo7e_oYM;~d$?Q@(sQ_dRntGn->hZz*LQ&DMOK%%8{`(>`zDN;`fc?)piw zQ{zeFN^!Zd*#U+^)0 z@6tI|ULUB?dqGE@`XW|}&WwM~w7BL;XGdnOD>LDfIppSh#g%u~Tq`dOzLq?LyjGX* z2~h5z-TbD^=C9(rPG@pGUP2wKsH1z$n{f8ZSMcn>N-sw7Ey;;%>@%P z5_zr4GfE!0u*{s$}=)2gxtKLE1p>ZWq1z z@1fGf=SPJPzd`;7-vGbAJii_)chIe0aGtI2uRw)n&bQ-f87Nl+D%1wf01vqIiG=~f zj4>~P%o;Oyk>xK3MZX%H1$Mjif@;bk zz5^=sbC+Iqf$l?>a34B9bAOUh>)3mgF=X5Wf7yAqyq~SGatc@4c$L=X>^bSw=P>!? zLM~qll)K-J|Fs)GW0jQ?Xe~3XFP&y~KD*dNo;uxZPA@Ux*coPROVG>@EH$MQC7}S< z-MP<{v@R_#TZYyI%y#^JN%~Iuyfo7+JXU0ipJk8nRI!uRN&16IH|!Ni zH>4ZVl`ks|nnAWfwqI`fS=d_Q>~DUuY?5=x@#3+fgswwcOA^#oV{4qgllw8K+Af-B z@8Qk`TQ9Kn`~@gid7V9JacKljHm3F_s5<3#;Q15t27IKnR)7>`Uc(lde^coPgR(rSeZi` zf!-xLa>SE~(}(bJSis(-EK7#Qb4`8F%nK}>c08Zh%$XVTq*IuF5la8jjaJY9++_7X z?=q|ZWuSPj-)!S)Q1S9JIh%eU`^uo_4Qiip${6PhQQ~XA!pi$c9&jr3YfyCO`5hg< zG@twWdgj+v0kZ<%)o<&Y9fvW8r@Ur1XIt}3u%&|aq?~=x)G4tu|Ax^)(=WO*ULfV$%S3M4})^&HrxE0K$TbiD(lfsWE^t&3b)z(O`t+I zf=>Qdxz2~kKkD*b*<$l|g34#_Uy95j;$wLGj1!&N+xcnE^@KI}G}evM=Iph+2>BD+ zt-Rekto;7~6{@(-((QT~WWAhzd`<8?Ku1y1snyr^iuVv7a^-B?Y2`P)-K{?;p6|5U z^&s7D=^q3|uR41c*S|9c<>!cywt zkNeMtj?k`hqwq>UFaIR*je9fsv*Oybw-j?|6)-J$uZ*m?%TqQGFfqKx@{?7c1b6@%{xn2+LPjx6?RzIecSHa44m zZwb%F*#9PZHl}MJW4Jt7nwZ^M5(!inn`*9G)m)3K*&kPP4^Z8_(4JXwEz=o+og*ct z^^SlU#E+{iN2g*Z{t;;3^U5dekNfik=H>HTt;Ec0o|{-&nr}*v6;rPwGfAEqcLq$& zEv{VjQ2b1MNHqE{H}22l)Pei>YUXS;=YG}PlU3I(oD3B6%$PH`rIm%I@>q$PQ_mmJK(2me?_*8Jd;0m~u70hKeriK>(K{ML@P8iP^$v8@IC0xnw7=%{e7wb2e?Ju`+m1z&wQC%=`MPk3a65ExPNQbnP&oa4IX68mAe~+WDdJ= zq1F+h&cz=D3&2q~ouK2!KVsLLjUQER&YwVqIzjE%egG=}2mUbN@_7oVik@b!etxC7?C2_U>5-7R zXn3`$KU`;49b9A19jG^Fcda$c(n;>oZ|2M-VU`7YvYwNUQ*CFD@c^>U(^)n5Pc59+ zaIaJJh5Urt=@9Lt$GB>TF}L0mpSOB09<}v-1XOy}r6p!X`e}1c?627ee~V{i-^hAK z)`hHPR9w8{uB`8YlWzV;{@lu2{3Y9dXFp-%cYw;{jCJO`&S)mS7BIiS5Az#0yJi(R z@vl7@Fa!MVV<|c--5>YosdApxRtHwZssiWKbHxl)1g5Tu?idN9@9o?IsO(8s-^S)IqjsDW zf76bW5l}L#{??|~fhw~RR4IQBO5S%sxu1d3*`8<4 z8Q1%uDWTmYbCR_3YTgNHcCI@i?RP6_OX(DG+vQeJ+8%Q8Q(*94tlq7l+`B-9{sdG# ze+sJJi;uhY)3chEoaZ-p7MeS`#unGx=S#K?X3++m>o(H{i)WZQ?8|iTDqV+>7pN;R zb-Yn{b{F+qPn`3zrW@2{@|5>DWUdH!z(wdl52_PDd!B8_t73?oclZ6o*p2hn}gcoRReTlJk#2H(GS)B2-!>|2Z((dP+wdQtc6@{iE&3l9N zQAHj566)p14}n$eUAV?o#dybueyC!u^PbOy>h9#{mQF4)%j!LJyG2X2|3x_heVF${ z{jQ9Negh<@u_&y>j}~R~iSEfaIWm;y%y;P1m7#Pw{l-VmUrw2;vy;F-aSS3SS5Kvf z=IcMdvzexR(e?WaIluhS3Wt>S&-*QnpD#Es!xDaPrtUwvmuCuRF@EmM(7yB$!a?3c zzbixg_(wUbSjM}H@6OPI_p_G3H|FD;fOZtV$RA|*KJ`b)s^K2zV21WpXv1eR|9!Lx zXtmJFeEJnWsP~gOukz8(hSngRePwKhW}r3r{cizwScX^7-D1sb$+3OCgW>OW zHYI$RlxzFr{qgJCy3<=?47S^$oK#DP<+8Rt-97t#&24VByta=V9%DY_NyO}D3QL}9 zl3OgFH_)eM*PYXq<+kXCwnSTdcMnFV)oT-Tp(MFBG`)FZSF$)MQ*CI~-87Jn_ogtf?Qg}mXQbL+ zW~0Z}DXG`V)TP^|(!5A1%n#W`_t&(|C&jL|;=Xc9&Q!yT%sD96MTy87h{jwx#qYlW z+ym8_GBnAa?lh+&nUS0|dgzTC3eN?l`+|_{#iatx7O0Wo_GtL-IV5ISr|@@o4?9qlx|DMHglEH zHR{g`+;XnM#%?zTWv^*>enxN1UwA3oFm~Ne-2T}{U9kskI+V)>BupQ5#3pVYNVeC4L*t2b?3 zQ)lbcpBKR{`?dYuY1jBFzrKNgLr*uFH1pGc4pFxDVbH!mnd(bkXO=s$EggNm*O|*_ zv?SYlQ$5UJFSAMHMdI<@Zj{m=HygWCiMDh{SBCcfykP`KrwB?X$n>eriv@980l2%eAZQr)3*{j_{rm=B#bSsvYyVK!R zIu;IJyQ!%uyz>V3NsW!6lT$W_dC>7(pfO4t?CR-rZAACP!}>xS;KG}^eG%5}9Rqv<4*G8Nuv)v*7!xcxLX zqj6U}O(jv}4GFk*)B2{yrX5qwLr?sgT5aiNB)maU(nUSFyr8kaH`UeM`C6@_vNt!W z?K2%lD{Wj2O-Q5sWqu>ejMJM_>U8(f2(Q*C8oWCC2`aufNS!haC<7FXOlOHHOdh4Yl5FsP!AwSZgwkX4d34GL!ymg}#$ES?$&CHCuAK zJ1+mgpX+swX)o_{^nF*~K5j|&zu{qh73D{-U@)2u1&sqK=7!q%4b2QILQNaJf{}EY z3C1i?;)FIXnL`9<0tbP+j{o3?N3E}`?#p?4ZF*Il%{a6 z6q+Q_tKnT8Xlz{1HNjoermknvD>WXP3!)o(V|*@SS8rR7`A|WW4OMb~lw17HXs3*f zb9=Pk^t1N!36#El(G7`2&g!p=LpqZ_P3mn0(W}w}SV5dnqp6&po-Nqu6e9 zrwwpb+e~GlvU>reUXPP_h6RJeA&Uum44=NH}_3_vKIdbmb1UtsmEvX8auFG zl+28!H?TaLtG4af5LsOxvb(HhCW^i>YT9AH(&{xKJ5bEP@9Enc=Z+zlr-Pd*W6pHx z)MZb1FIRtafr%#ilW{gAJ!ZY3EBG_KH6M*`7#MK2)pqljTq8SsP@}0tyra95t8bUH zL52HLmCF@~C#pu-oH4P#LunugA zFj$;#G(;0^`U(%#WlD7ScJ^`ZxR5bzj!&BjCoUeu0)q`2$1X`@FoqO0d z>n=1JO+*>5$zAkeK_-S?jFk?XcL5=r`x3+F80@Wyw>UX{qh?^i5_?qBPM@!E=9aCq z{X~<)8JAqO7N2R3I!e&O(xF42YLZUONQof5%TyV+(P=-YJ9vHD-&%dWPLb)hc8%_J zKJDBT%|GXR%uA5UoU}TcwnX=eJ*gG@x_ehZUBS}R-M6A=^@`Oi?1Ge&o@AwDw7m+4 zrdMh|w`_a-Ou6Oc3dv3HPwNb8zh6X-CJb!ui~S6_GP?LM&8wwrM7~Ncm1T4>Cs)Md1p zLZ)TLmZ=G_Z7aJQKCTlppUZDf_i(Vs8bNui1p{1D0_QU^Mp>Z11&;d~9kpM-gU@b_m%ac0SS7VpQBnn!*70DwpLbAqc6q!J};1B4L6+RY0cV`nC_j5H);R# zT0rj->7A{*2Y#^PW9J$3?Y9M=>3r7G^rUZ9FuP9TmD9V1PF@iTGVd-qiPL;<+0ykt z^UaYA&HFxVhLQgD&&Ax*7D+!fxMXum9 z{FPVcn+04877?;K4L(5~Jl~kn(+m02{Cav}C!KsAdijLn@#s$2C*Ork_;4E8m~rk*~as)93f-P8s5Hy;rs; zYr1stI_Xbj^5y3D;<-2_SLG?c>-}^`IRBiy)1-?o=jEM>L(dI+`aAq?x+B*gej+=c zBixfOzMS-v%5Y_QWq9c2^U9l|JN*5J75BMw^1|MlAnHnPlC`;H_tV)I4rBN8pI_YtNChrzvf|` zWK5&S)Lpn=;c572&#i52!@43IPH}|O-`mY8N;Hv9Dwgi&xsC!+y*3a}a%QxhvCG4} zn3b#3sdYV^b;sJG7p`t==)CZv_SF~0F1)brqPottp}LOxHSKHLSBF;DU3Af<;qVrl z%l_vSIA*`l$VovDH^cw!CF+#RnIrbU*ZfoX`?inQ0`(2Cczylqj)sfcJL=olTyR0W zvtw;2*3n)UUwdKfq7GW%wX6Pyg`K)qzi)kBUw^l)ZEs)Oxw@`%^+ln0EJp8lG^}p# zSiSaw2C9ES9ku1ua=V`6>LcCuE`leIJT7D;*c0!aF8I6U|9@QkcWHL2mY$}aqnb~- z>+#e;do%Dqq6PRN#rgQ5>vI=EwVhq6y7Nd^lW7dZ{Wuy&mvQ}3xAw&0&J<` z_Zoy-!R^)9Z@7Ul_?6T7X$4`wEo<4ip1TBFhtP$~@dpTpz&U4F9c0IP-!jg;l^0xd zCihl^!{GDy_3OVnsaG4g~2=V zPXB_{=OYukY%8zg_a=AV2oIRRJ8c5~9exb@B)E1Zx)B}(KZ9bwta{n7Q*jj(LmJ{+Ykgh#;x zoXPEMMF;S)9@>iV7}%ab2g0%u8{_H7QH8;6w{cF|hJ5gk?uMW6D0t2vBB!0Qz%L%A z?u5s{i#|zS!m=Iu)BE||EMeL8JnuoqMhE<0@-y%d9t0=!L$(-g1ulHp_C*lf;f7@s z^Tx;MUwC8>a^VyFwkS?Nf?vg}U&g>4V>WLq__!M$1Dn6*(!tNV;Zd;dNlQQOSI2;5Y{DuvWg1^H%bMUWd`lFWKD#VLE1YY?~OZVG@l%2@azQvk? ztw`bR_*TM)z#rn3cM=@_Ya1Q~f9HnH)AZ-JtxVzl_yl=}z|`MZ{z34HzqR2A__cAK z0l+g3Ry_y41nGX(zbpE&8>l?MU)F!;rUWiRr%7nskYf8Ur( z@LJzw=kbg9QRuSa*!ctci|`nD_JqwF0zdLY=3N(MfwG+_nZo`bv*tjT-NRquCke}@ z;ZyivH!{KYe>NseI01eOuXSJ?d=c;T3%KMb^cOsSJB6}exDS62y6`UieT4ls31zEr z_P-z#y08ZSEMdR>LD?mI6n_G`@G1Q4o6rHAah$pk7QPLCK4IZue3-D`uAuAY*<{&mN!f@9^uO3tz-X2v35W^8%)oa0Hb7yaUP$dbah1*YVrWVdz5H#~V{#(6gy0 zyb}K`bYVeB!2F!BumoSU2c1FBhMw?gd?j>Y(d>XZpRjNaJ}iFlFYp@YvV-?5w(BC$ zWy9|NIsBfPui!UFb4?_fnUMKlhbZKTTin2bf6JA@CXm#PY@P9)e~|D5`1y^tT-jB6Y76>9mrbNu*f^@~r%gc5-jVQq_yly}efT>G z3uj)99KynH;YF7Xql>UVG)!3bh~C>A;B{T}2W1~9Ojvjuc7O&&2R)lW!sGZO(1q38 z8GnR@tKG0L<%WgNxnbD``tWtM8~n24^QZ0f$3EH${B8$hjIiwM3?`6ISoqnSX-|G@ zJ__zj!B2P)d?<~cqJy6O8{vKUF6hEx`~mTRFXHbe?6+?tyEi|>4?`FJ2mZ^1{q}8S z_vW|wr=bgHVdG|ku;0Fo?A{b&$7T|`umYdIpD_k{_HBec_!{WKcicL+pLi- zo6q4l!y|kWez_o^!$U2 zS9pYT@%h+p5tiZS6Bg#ZkA6`Y^z6+DKaO7qT`2o7I|=*k#Q1H<$hOR_chg_+2tR;- zkg)KB_{RwQZNtcp%(vdpS_)nGjt`(GVc`V+Wx~Rre$bA8zpWVAk(v9UfSG-Ox`0ji z8p6Wc@C}3yfw$aa^ZISW$Y#uiAI6>&Ji?9mG~oz%*S(A%Qf6E_T=@h@%|Tz;H> zN4F3-fp_W)e&}CqcnFOA)P{xc{F$o{_)EO%XnxLq27eGa6JYa;mfi|p@NbqL25Wx7 zyi-}={7D-Qg8Bc)hReX0+_3p&z#PMie-a$~6=R05@GoCtjU_w|o^^u0AUp^@iPv@3 z{Aa+d$19uwANmb-gGY8mR=-RcgzLc`yy`m$Ui@1dUI%W)t6svm8}0&sfGv^Q+o&V> zOT2i>UO^^40=*tg;2oLZBX0OnaDmA)E$~!=x8M&Dz7zZ zD|{#T4ZQm3D0m#-0Kc%P*oK9FQ<7)G&}B>JJ9z2#EV!Z+U7^c{(&zCv5*`D~=U`ug za2Q;Vj}X>(vGo0{)*CHejSbTTVSO)3--?orqoFePQh0^+eGz?IL}A$xl?_paWm`vf zbrhCu8QF|cSSXt?3JYcPMPZ@ryC^JtC;lb+PTx@cw&>Jl96WuY4F|zJZa4vc#SM>v zrHd?28F;Z9HgCHBl`XKET7~e%_)>fseiJ^4pNGE?&ygCxEVX!@i*Ex<@Nc^RO)bD4 zhw9)5vukDTS}{&eqGMj$`F268nPJ|^ZCS2c*Eq1JC%BgvWce}UCCgW@3@s1x0()PK z*8?wEzP;tL6%EUS`qgNxt%r9ZFIm1no?6~`X=(8~eo+?R)84Z`h=AVICChmQ_bsW8 zE`Ax9TCu0QBiWbg>rAiUb?~V5a zd-VU3_g_{ikQi4pTi(IaJ#R6Y`X zr1p`}BMpz35Z7?@8hzl%17i;yeZYimeE73tpFR3nC16tu2tU~R;K2vS9xQuk=R<=J gjXreZq2On_J~RB8@y|^D@z`+f2rO^9S80L&2WyMeXaE2J diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 8f7e034..ea6d4ef 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,7 +62,7 @@ $(function() {

    vk_mem_alloc.h
    -Go to the documentation of this file.
    1 //
    2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
    3 //
    4 // Permission is hereby granted, free of charge, to any person obtaining a copy
    5 // of this software and associated documentation files (the "Software"), to deal
    6 // in the Software without restriction, including without limitation the rights
    7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    8 // copies of the Software, and to permit persons to whom the Software is
    9 // furnished to do so, subject to the following conditions:
    10 //
    11 // The above copyright notice and this permission notice shall be included in
    12 // all copies or substantial portions of the Software.
    13 //
    14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    20 // THE SOFTWARE.
    21 //
    22 
    23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
    24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
    25 
    26 #ifdef __cplusplus
    27 extern "C" {
    28 #endif
    29 
    1258 #include <vulkan/vulkan.h>
    1259 
    1260 #if !defined(VMA_DEDICATED_ALLOCATION)
    1261  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
    1262  #define VMA_DEDICATED_ALLOCATION 1
    1263  #else
    1264  #define VMA_DEDICATED_ALLOCATION 0
    1265  #endif
    1266 #endif
    1267 
    1277 VK_DEFINE_HANDLE(VmaAllocator)
    1278 
    1279 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
    1281  VmaAllocator allocator,
    1282  uint32_t memoryType,
    1283  VkDeviceMemory memory,
    1284  VkDeviceSize size);
    1286 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
    1287  VmaAllocator allocator,
    1288  uint32_t memoryType,
    1289  VkDeviceMemory memory,
    1290  VkDeviceSize size);
    1291 
    1299 typedef struct VmaDeviceMemoryCallbacks {
    1301  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
    1303  PFN_vmaFreeDeviceMemoryFunction pfnFree;
    1304 } VmaDeviceMemoryCallbacks;
    1305 
    1307 typedef enum VmaAllocatorCreateFlagBits {
    1312  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
    1334  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
    1335 
    1336  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1337 } VmaAllocatorCreateFlagBits;
    1338 typedef VkFlags VmaAllocatorCreateFlags;
    1339 
    1344 typedef struct VmaVulkanFunctions {
    1345  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
    1346  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
    1347  PFN_vkAllocateMemory vkAllocateMemory;
    1348  PFN_vkFreeMemory vkFreeMemory;
    1349  PFN_vkMapMemory vkMapMemory;
    1350  PFN_vkUnmapMemory vkUnmapMemory;
    1351  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
    1352  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
    1353  PFN_vkBindBufferMemory vkBindBufferMemory;
    1354  PFN_vkBindImageMemory vkBindImageMemory;
    1355  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
    1356  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
    1357  PFN_vkCreateBuffer vkCreateBuffer;
    1358  PFN_vkDestroyBuffer vkDestroyBuffer;
    1359  PFN_vkCreateImage vkCreateImage;
    1360  PFN_vkDestroyImage vkDestroyImage;
    1361 #if VMA_DEDICATED_ALLOCATION
    1362  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
    1363  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
    1364 #endif
    1365 } VmaVulkanFunctions;
    1366 
    1368 typedef enum VmaRecordFlagBits {
    1374  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
    1375 
    1376  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1377 } VmaRecordFlagBits;
    1378 typedef VkFlags VmaRecordFlags;
    1379 
    1380 /*
    1381 Define this macro to 0/1 to disable/enable support for recording functionality,
    1382 available through VmaAllocatorCreateInfo::pRecordSettings.
    1383 */
    1384 #ifndef VMA_RECORDING_ENABLED
    1385  #ifdef _WIN32
    1386  #define VMA_RECORDING_ENABLED 1
    1387  #else
    1388  #define VMA_RECORDING_ENABLED 0
    1389  #endif
    1390 #endif
    1391 
    1393 typedef struct VmaRecordSettings
    1394 {
    1396  VmaRecordFlags flags;
    1404  const char* pFilePath;
    1405 } VmaRecordSettings;
    1406 
    1408 typedef struct VmaAllocatorCreateInfo
    1409 {
    1411  VmaAllocatorCreateFlags flags;
    1413 
    1414  VkPhysicalDevice physicalDevice;
    1416 
    1417  VkDevice device;
    1419 
    1420  VkDeviceSize preferredLargeHeapBlockSize;
    1422 
    1423  const VkAllocationCallbacks* pAllocationCallbacks;
    1425 
    1426  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
    1440  uint32_t frameInUseCount;
    1464  const VkDeviceSize* pHeapSizeLimit;
    1476  const VmaVulkanFunctions* pVulkanFunctions;
    1483  const VmaRecordSettings* pRecordSettings;
    1484 } VmaAllocatorCreateInfo;
    1485 
    1487 VkResult vmaCreateAllocator(
    1488  const VmaAllocatorCreateInfo* pCreateInfo,
    1489  VmaAllocator* pAllocator);
    1490 
    1492 void vmaDestroyAllocator(
    1493  VmaAllocator allocator);
    1494 
    1499 void vmaGetPhysicalDeviceProperties(
    1500  VmaAllocator allocator,
    1501  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
    1502 
    1507 void vmaGetMemoryProperties(
    1508  VmaAllocator allocator,
    1509  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
    1510 
    1517 void vmaGetMemoryTypeProperties(
    1518  VmaAllocator allocator,
    1519  uint32_t memoryTypeIndex,
    1520  VkMemoryPropertyFlags* pFlags);
    1521 
    1530 void vmaSetCurrentFrameIndex(
    1531  VmaAllocator allocator,
    1532  uint32_t frameIndex);
    1533 
    1536 typedef struct VmaStatInfo
    1537 {
    1539  uint32_t blockCount;
    1541  uint32_t allocationCount;
    1543  uint32_t unusedRangeCount;
    1545  VkDeviceSize usedBytes;
    1547  VkDeviceSize unusedBytes;
    1548  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
    1549  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
    1550 } VmaStatInfo;
    1551 
    1553 typedef struct VmaStats
    1554 {
    1555  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
    1556  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
    1557  VmaStatInfo total;
    1558 } VmaStats;
    1559 
    1561 void vmaCalculateStats(
    1562  VmaAllocator allocator,
    1563  VmaStats* pStats);
    1564 
    1565 #define VMA_STATS_STRING_ENABLED 1
    1566 
    1567 #if VMA_STATS_STRING_ENABLED
    1568 
    1570 
    1572 void vmaBuildStatsString(
    1573  VmaAllocator allocator,
    1574  char** ppStatsString,
    1575  VkBool32 detailedMap);
    1576 
    1577 void vmaFreeStatsString(
    1578  VmaAllocator allocator,
    1579  char* pStatsString);
    1580 
    1581 #endif // #if VMA_STATS_STRING_ENABLED
    1582 
    1591 VK_DEFINE_HANDLE(VmaPool)
    1592 
    1593 typedef enum VmaMemoryUsage
    1594 {
    1598  VMA_MEMORY_USAGE_UNKNOWN = 0,
    1615  VMA_MEMORY_USAGE_GPU_ONLY = 1,
    1625  VMA_MEMORY_USAGE_CPU_ONLY = 2,
    1632  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
    1641  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
    1642  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
    1643 } VmaMemoryUsage;
    1644 
    1646 typedef enum VmaAllocationCreateFlagBits {
    1658  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
    1659 
    1669  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
    1682  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
    1695  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
    1702  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
    1708  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
    1709 
    1710  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1711 } VmaAllocationCreateFlagBits;
    1712 typedef VkFlags VmaAllocationCreateFlags;
    1713 
    1714 typedef struct VmaAllocationCreateInfo
    1715 {
    1717  VmaAllocationCreateFlags flags;
    1723  VmaMemoryUsage usage;
    1728  VkMemoryPropertyFlags requiredFlags;
    1733  VkMemoryPropertyFlags preferredFlags;
    1741  uint32_t memoryTypeBits;
    1747  VmaPool pool;
    1754  void* pUserData;
    1755 } VmaAllocationCreateInfo;
    1756 
    1773 VkResult vmaFindMemoryTypeIndex(
    1774  VmaAllocator allocator,
    1775  uint32_t memoryTypeBits,
    1776  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1777  uint32_t* pMemoryTypeIndex);
    1778 
    1791 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    1792  VmaAllocator allocator,
    1793  const VkBufferCreateInfo* pBufferCreateInfo,
    1794  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1795  uint32_t* pMemoryTypeIndex);
    1796 
    1809 VkResult vmaFindMemoryTypeIndexForImageInfo(
    1810  VmaAllocator allocator,
    1811  const VkImageCreateInfo* pImageCreateInfo,
    1812  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1813  uint32_t* pMemoryTypeIndex);
    1814 
    1816 typedef enum VmaPoolCreateFlagBits {
    1834  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
    1835 
    1836  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1837 } VmaPoolCreateFlagBits;
    1838 typedef VkFlags VmaPoolCreateFlags;
    1839 
    1842 typedef struct VmaPoolCreateInfo {
    1845  uint32_t memoryTypeIndex;
    1848  VmaPoolCreateFlags flags;
    1853  VkDeviceSize blockSize;
    1858  size_t minBlockCount;
    1866  size_t maxBlockCount;
    1880  uint32_t frameInUseCount;
    1881 } VmaPoolCreateInfo;
    1882 
    1885 typedef struct VmaPoolStats {
    1888  VkDeviceSize size;
    1891  VkDeviceSize unusedSize;
    1894  size_t allocationCount;
    1897  size_t unusedRangeCount;
    1904  VkDeviceSize unusedRangeSizeMax;
    1905 } VmaPoolStats;
    1906 
    1913 VkResult vmaCreatePool(
    1914  VmaAllocator allocator,
    1915  const VmaPoolCreateInfo* pCreateInfo,
    1916  VmaPool* pPool);
    1917 
    1920 void vmaDestroyPool(
    1921  VmaAllocator allocator,
    1922  VmaPool pool);
    1923 
    1930 void vmaGetPoolStats(
    1931  VmaAllocator allocator,
    1932  VmaPool pool,
    1933  VmaPoolStats* pPoolStats);
    1934 
    1941 void vmaMakePoolAllocationsLost(
    1942  VmaAllocator allocator,
    1943  VmaPool pool,
    1944  size_t* pLostAllocationCount);
    1945 
    1960 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
    1961 
    1986 VK_DEFINE_HANDLE(VmaAllocation)
    1987 
    1988 
    1990 typedef struct VmaAllocationInfo {
    1995  uint32_t memoryType;
    2004  VkDeviceMemory deviceMemory;
    2009  VkDeviceSize offset;
    2014  VkDeviceSize size;
    2023  void* pMappedData;
    2028  void* pUserData;
    2029 } VmaAllocationInfo;
    2030 
    2041 VkResult vmaAllocateMemory(
    2042  VmaAllocator allocator,
    2043  const VkMemoryRequirements* pVkMemoryRequirements,
    2044  const VmaAllocationCreateInfo* pCreateInfo,
    2045  VmaAllocation* pAllocation,
    2046  VmaAllocationInfo* pAllocationInfo);
    2047 
    2054 VkResult vmaAllocateMemoryForBuffer(
    2055  VmaAllocator allocator,
    2056  VkBuffer buffer,
    2057  const VmaAllocationCreateInfo* pCreateInfo,
    2058  VmaAllocation* pAllocation,
    2059  VmaAllocationInfo* pAllocationInfo);
    2060 
    2062 VkResult vmaAllocateMemoryForImage(
    2063  VmaAllocator allocator,
    2064  VkImage image,
    2065  const VmaAllocationCreateInfo* pCreateInfo,
    2066  VmaAllocation* pAllocation,
    2067  VmaAllocationInfo* pAllocationInfo);
    2068 
    2070 void vmaFreeMemory(
    2071  VmaAllocator allocator,
    2072  VmaAllocation allocation);
    2073 
    2090 void vmaGetAllocationInfo(
    2091  VmaAllocator allocator,
    2092  VmaAllocation allocation,
    2093  VmaAllocationInfo* pAllocationInfo);
    2094 
    2109 VkBool32 vmaTouchAllocation(
    2110  VmaAllocator allocator,
    2111  VmaAllocation allocation);
    2112 
    2126 void vmaSetAllocationUserData(
    2127  VmaAllocator allocator,
    2128  VmaAllocation allocation,
    2129  void* pUserData);
    2130 
    2141 void vmaCreateLostAllocation(
    2142  VmaAllocator allocator,
    2143  VmaAllocation* pAllocation);
    2144 
    2179 VkResult vmaMapMemory(
    2180  VmaAllocator allocator,
    2181  VmaAllocation allocation,
    2182  void** ppData);
    2183 
    2188 void vmaUnmapMemory(
    2189  VmaAllocator allocator,
    2190  VmaAllocation allocation);
    2191 
    2204 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2205 
    2218 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2219 
    2236 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
    2237 
    2239 typedef struct VmaDefragmentationInfo {
    2244  VkDeviceSize maxBytesToMove;
    2249  uint32_t maxAllocationsToMove;
    2250 } VmaDefragmentationInfo;
    2251 
    2253 typedef struct VmaDefragmentationStats {
    2255  VkDeviceSize bytesMoved;
    2257  VkDeviceSize bytesFreed;
    2259  uint32_t allocationsMoved;
    2261  uint32_t deviceMemoryBlocksFreed;
    2262 } VmaDefragmentationStats;
    2263 
    2346 VkResult vmaDefragment(
    2347  VmaAllocator allocator,
    2348  VmaAllocation* pAllocations,
    2349  size_t allocationCount,
    2350  VkBool32* pAllocationsChanged,
    2351  const VmaDefragmentationInfo *pDefragmentationInfo,
    2352  VmaDefragmentationStats* pDefragmentationStats);
    2353 
    2366 VkResult vmaBindBufferMemory(
    2367  VmaAllocator allocator,
    2368  VmaAllocation allocation,
    2369  VkBuffer buffer);
    2370 
    2383 VkResult vmaBindImageMemory(
    2384  VmaAllocator allocator,
    2385  VmaAllocation allocation,
    2386  VkImage image);
    2387 
    2414 VkResult vmaCreateBuffer(
    2415  VmaAllocator allocator,
    2416  const VkBufferCreateInfo* pBufferCreateInfo,
    2417  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2418  VkBuffer* pBuffer,
    2419  VmaAllocation* pAllocation,
    2420  VmaAllocationInfo* pAllocationInfo);
    2421 
    2433 void vmaDestroyBuffer(
    2434  VmaAllocator allocator,
    2435  VkBuffer buffer,
    2436  VmaAllocation allocation);
    2437 
    2439 VkResult vmaCreateImage(
    2440  VmaAllocator allocator,
    2441  const VkImageCreateInfo* pImageCreateInfo,
    2442  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2443  VkImage* pImage,
    2444  VmaAllocation* pAllocation,
    2445  VmaAllocationInfo* pAllocationInfo);
    2446 
    2458 void vmaDestroyImage(
    2459  VmaAllocator allocator,
    2460  VkImage image,
    2461  VmaAllocation allocation);
    2462 
    2463 #ifdef __cplusplus
    2464 }
    2465 #endif
    2466 
    2467 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
    2468 
    2469 // For Visual Studio IntelliSense.
    2470 #if defined(__cplusplus) && defined(__INTELLISENSE__)
    2471 #define VMA_IMPLEMENTATION
    2472 #endif
    2473 
    2474 #ifdef VMA_IMPLEMENTATION
    2475 #undef VMA_IMPLEMENTATION
    2476 
    2477 #include <cstdint>
    2478 #include <cstdlib>
    2479 #include <cstring>
    2480 
    2481 /*******************************************************************************
    2482 CONFIGURATION SECTION
    2483 
    2484 Define some of these macros before each #include of this header or change them
    2485 here if you need other then default behavior depending on your environment.
    2486 */
    2487 
    2488 /*
    2489 Define this macro to 1 to make the library fetch pointers to Vulkan functions
    2490 internally, like:
    2491 
    2492  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    2493 
    2494 Define to 0 if you are going to provide you own pointers to Vulkan functions via
    2495 VmaAllocatorCreateInfo::pVulkanFunctions.
    2496 */
    2497 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
    2498 #define VMA_STATIC_VULKAN_FUNCTIONS 1
    2499 #endif
    2500 
    2501 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
    2502 //#define VMA_USE_STL_CONTAINERS 1
    2503 
    2504 /* Set this macro to 1 to make the library including and using STL containers:
    2505 std::pair, std::vector, std::list, std::unordered_map.
    2506 
    2507 Set it to 0 or undefined to make the library using its own implementation of
    2508 the containers.
    2509 */
    2510 #if VMA_USE_STL_CONTAINERS
    2511  #define VMA_USE_STL_VECTOR 1
    2512  #define VMA_USE_STL_UNORDERED_MAP 1
    2513  #define VMA_USE_STL_LIST 1
    2514 #endif
    2515 
    2516 #if VMA_USE_STL_VECTOR
    2517  #include <vector>
    2518 #endif
    2519 
    2520 #if VMA_USE_STL_UNORDERED_MAP
    2521  #include <unordered_map>
    2522 #endif
    2523 
    2524 #if VMA_USE_STL_LIST
    2525  #include <list>
    2526 #endif
    2527 
    2528 /*
    2529 Following headers are used in this CONFIGURATION section only, so feel free to
    2530 remove them if not needed.
    2531 */
    2532 #include <cassert> // for assert
    2533 #include <algorithm> // for min, max
    2534 #include <mutex> // for std::mutex
    2535 #include <atomic> // for std::atomic
    2536 
    2537 #ifndef VMA_NULL
    2538  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
    2539  #define VMA_NULL nullptr
    2540 #endif
    2541 
    2542 #if defined(__APPLE__) || defined(__ANDROID__)
    2543 #include <cstdlib>
    2544 void *aligned_alloc(size_t alignment, size_t size)
    2545 {
    2546  // alignment must be >= sizeof(void*)
    2547  if(alignment < sizeof(void*))
    2548  {
    2549  alignment = sizeof(void*);
    2550  }
    2551 
    2552  void *pointer;
    2553  if(posix_memalign(&pointer, alignment, size) == 0)
    2554  return pointer;
    2555  return VMA_NULL;
    2556 }
    2557 #endif
    2558 
    2559 // If your compiler is not compatible with C++11 and definition of
    2560 // aligned_alloc() function is missing, uncommeting following line may help:
    2561 
    2562 //#include <malloc.h>
    2563 
    2564 // Normal assert to check for programmer's errors, especially in Debug configuration.
    2565 #ifndef VMA_ASSERT
    2566  #ifdef _DEBUG
    2567  #define VMA_ASSERT(expr) assert(expr)
    2568  #else
    2569  #define VMA_ASSERT(expr)
    2570  #endif
    2571 #endif
    2572 
    2573 // Assert that will be called very often, like inside data structures e.g. operator[].
    2574 // Making it non-empty can make program slow.
    2575 #ifndef VMA_HEAVY_ASSERT
    2576  #ifdef _DEBUG
    2577  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
    2578  #else
    2579  #define VMA_HEAVY_ASSERT(expr)
    2580  #endif
    2581 #endif
    2582 
    2583 #ifndef VMA_ALIGN_OF
    2584  #define VMA_ALIGN_OF(type) (__alignof(type))
    2585 #endif
    2586 
    2587 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
    2588  #if defined(_WIN32)
    2589  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
    2590  #else
    2591  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
    2592  #endif
    2593 #endif
    2594 
    2595 #ifndef VMA_SYSTEM_FREE
    2596  #if defined(_WIN32)
    2597  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
    2598  #else
    2599  #define VMA_SYSTEM_FREE(ptr) free(ptr)
    2600  #endif
    2601 #endif
    2602 
    2603 #ifndef VMA_MIN
    2604  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
    2605 #endif
    2606 
    2607 #ifndef VMA_MAX
    2608  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
    2609 #endif
    2610 
    2611 #ifndef VMA_SWAP
    2612  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
    2613 #endif
    2614 
    2615 #ifndef VMA_SORT
    2616  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
    2617 #endif
    2618 
    2619 #ifndef VMA_DEBUG_LOG
    2620  #define VMA_DEBUG_LOG(format, ...)
    2621  /*
    2622  #define VMA_DEBUG_LOG(format, ...) do { \
    2623  printf(format, __VA_ARGS__); \
    2624  printf("\n"); \
    2625  } while(false)
    2626  */
    2627 #endif
    2628 
    2629 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
    2630 #if VMA_STATS_STRING_ENABLED
    2631  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
    2632  {
    2633  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
    2634  }
    2635  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
    2636  {
    2637  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
    2638  }
    2639  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
    2640  {
    2641  snprintf(outStr, strLen, "%p", ptr);
    2642  }
    2643 #endif
    2644 
    2645 #ifndef VMA_MUTEX
    2646  class VmaMutex
    2647  {
    2648  public:
    2649  VmaMutex() { }
    2650  ~VmaMutex() { }
    2651  void Lock() { m_Mutex.lock(); }
    2652  void Unlock() { m_Mutex.unlock(); }
    2653  private:
    2654  std::mutex m_Mutex;
    2655  };
    2656  #define VMA_MUTEX VmaMutex
    2657 #endif
    2658 
    2659 /*
    2660 If providing your own implementation, you need to implement a subset of std::atomic:
    2661 
    2662 - Constructor(uint32_t desired)
    2663 - uint32_t load() const
    2664 - void store(uint32_t desired)
    2665 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
    2666 */
    2667 #ifndef VMA_ATOMIC_UINT32
    2668  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
    2669 #endif
    2670 
    2671 #ifndef VMA_BEST_FIT
    2672 
    2684  #define VMA_BEST_FIT (1)
    2685 #endif
    2686 
    2687 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
    2688 
    2692  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
    2693 #endif
    2694 
    2695 #ifndef VMA_DEBUG_ALIGNMENT
    2696 
    2700  #define VMA_DEBUG_ALIGNMENT (1)
    2701 #endif
    2702 
    2703 #ifndef VMA_DEBUG_MARGIN
    2704 
    2708  #define VMA_DEBUG_MARGIN (0)
    2709 #endif
    2710 
    2711 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
    2712 
    2716  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
    2717 #endif
    2718 
    2719 #ifndef VMA_DEBUG_DETECT_CORRUPTION
    2720 
    2725  #define VMA_DEBUG_DETECT_CORRUPTION (0)
    2726 #endif
    2727 
    2728 #ifndef VMA_DEBUG_GLOBAL_MUTEX
    2729 
    2733  #define VMA_DEBUG_GLOBAL_MUTEX (0)
    2734 #endif
    2735 
    2736 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
    2737 
    2741  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
    2742 #endif
    2743 
    2744 #ifndef VMA_SMALL_HEAP_MAX_SIZE
    2745  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
    2747 #endif
    2748 
    2749 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
    2750  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
    2752 #endif
    2753 
    2754 #ifndef VMA_CLASS_NO_COPY
    2755  #define VMA_CLASS_NO_COPY(className) \
    2756  private: \
    2757  className(const className&) = delete; \
    2758  className& operator=(const className&) = delete;
    2759 #endif
    2760 
    2761 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
    2762 
    2763 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
    2764 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
    2765 
    2766 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
    2767 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
    2768 
    2769 /*******************************************************************************
    2770 END OF CONFIGURATION
    2771 */
    2772 
    2773 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
    2774  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
    2775 
    2776 // Returns number of bits set to 1 in (v).
    2777 static inline uint32_t VmaCountBitsSet(uint32_t v)
    2778 {
    2779  uint32_t c = v - ((v >> 1) & 0x55555555);
    2780  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
    2781  c = ((c >> 4) + c) & 0x0F0F0F0F;
    2782  c = ((c >> 8) + c) & 0x00FF00FF;
    2783  c = ((c >> 16) + c) & 0x0000FFFF;
    2784  return c;
    2785 }
    2786 
    2787 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
    2788 // Use types like uint32_t, uint64_t as T.
    2789 template <typename T>
    2790 static inline T VmaAlignUp(T val, T align)
    2791 {
    2792  return (val + align - 1) / align * align;
    2793 }
    2794 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
    2795 // Use types like uint32_t, uint64_t as T.
    2796 template <typename T>
    2797 static inline T VmaAlignDown(T val, T align)
    2798 {
    2799  return val / align * align;
    2800 }
    2801 
    2802 // Division with mathematical rounding to nearest number.
    2803 template <typename T>
    2804 inline T VmaRoundDiv(T x, T y)
    2805 {
    2806  return (x + (y / (T)2)) / y;
    2807 }
    2808 
    2809 static inline bool VmaStrIsEmpty(const char* pStr)
    2810 {
    2811  return pStr == VMA_NULL || *pStr == '\0';
    2812 }
    2813 
    2814 #ifndef VMA_SORT
    2815 
    2816 template<typename Iterator, typename Compare>
    2817 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
    2818 {
    2819  Iterator centerValue = end; --centerValue;
    2820  Iterator insertIndex = beg;
    2821  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
    2822  {
    2823  if(cmp(*memTypeIndex, *centerValue))
    2824  {
    2825  if(insertIndex != memTypeIndex)
    2826  {
    2827  VMA_SWAP(*memTypeIndex, *insertIndex);
    2828  }
    2829  ++insertIndex;
    2830  }
    2831  }
    2832  if(insertIndex != centerValue)
    2833  {
    2834  VMA_SWAP(*insertIndex, *centerValue);
    2835  }
    2836  return insertIndex;
    2837 }
    2838 
    2839 template<typename Iterator, typename Compare>
    2840 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
    2841 {
    2842  if(beg < end)
    2843  {
    2844  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
    2845  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
    2846  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
    2847  }
    2848 }
    2849 
    2850 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
    2851 
    2852 #endif // #ifndef VMA_SORT
    2853 
    2854 /*
    2855 Returns true if two memory blocks occupy overlapping pages.
    2856 ResourceA must be in less memory offset than ResourceB.
    2857 
    2858 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
    2859 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
    2860 */
    2861 static inline bool VmaBlocksOnSamePage(
    2862  VkDeviceSize resourceAOffset,
    2863  VkDeviceSize resourceASize,
    2864  VkDeviceSize resourceBOffset,
    2865  VkDeviceSize pageSize)
    2866 {
    2867  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
    2868  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
    2869  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
    2870  VkDeviceSize resourceBStart = resourceBOffset;
    2871  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
    2872  return resourceAEndPage == resourceBStartPage;
    2873 }
    2874 
    2875 enum VmaSuballocationType
    2876 {
    2877  VMA_SUBALLOCATION_TYPE_FREE = 0,
    2878  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
    2879  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
    2880  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
    2881  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
    2882  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
    2883  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
    2884 };
    2885 
    2886 /*
    2887 Returns true if given suballocation types could conflict and must respect
    2888 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
    2889 or linear image and another one is optimal image. If type is unknown, behave
    2890 conservatively.
    2891 */
    2892 static inline bool VmaIsBufferImageGranularityConflict(
    2893  VmaSuballocationType suballocType1,
    2894  VmaSuballocationType suballocType2)
    2895 {
    2896  if(suballocType1 > suballocType2)
    2897  {
    2898  VMA_SWAP(suballocType1, suballocType2);
    2899  }
    2900 
    2901  switch(suballocType1)
    2902  {
    2903  case VMA_SUBALLOCATION_TYPE_FREE:
    2904  return false;
    2905  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
    2906  return true;
    2907  case VMA_SUBALLOCATION_TYPE_BUFFER:
    2908  return
    2909  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2910  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2911  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
    2912  return
    2913  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2914  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
    2915  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2916  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
    2917  return
    2918  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2919  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
    2920  return false;
    2921  default:
    2922  VMA_ASSERT(0);
    2923  return true;
    2924  }
    2925 }
    2926 
    2927 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
    2928 {
    2929  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
    2930  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2931  for(size_t i = 0; i < numberCount; ++i, ++pDst)
    2932  {
    2933  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
    2934  }
    2935 }
    2936 
    2937 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
    2938 {
    2939  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
    2940  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2941  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
    2942  {
    2943  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
    2944  {
    2945  return false;
    2946  }
    2947  }
    2948  return true;
    2949 }
    2950 
    2951 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
    2952 struct VmaMutexLock
    2953 {
    2954  VMA_CLASS_NO_COPY(VmaMutexLock)
    2955 public:
    2956  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
    2957  m_pMutex(useMutex ? &mutex : VMA_NULL)
    2958  {
    2959  if(m_pMutex)
    2960  {
    2961  m_pMutex->Lock();
    2962  }
    2963  }
    2964 
    2965  ~VmaMutexLock()
    2966  {
    2967  if(m_pMutex)
    2968  {
    2969  m_pMutex->Unlock();
    2970  }
    2971  }
    2972 
    2973 private:
    2974  VMA_MUTEX* m_pMutex;
    2975 };
    2976 
    2977 #if VMA_DEBUG_GLOBAL_MUTEX
    2978  static VMA_MUTEX gDebugGlobalMutex;
    2979  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
    2980 #else
    2981  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
    2982 #endif
    2983 
    2984 // Minimum size of a free suballocation to register it in the free suballocation collection.
    2985 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
    2986 
    2987 /*
    2988 Performs binary search and returns iterator to first element that is greater or
    2989 equal to (key), according to comparison (cmp).
    2990 
    2991 Cmp should return true if first argument is less than second argument.
    2992 
    2993 Returned value is the found element, if present in the collection or place where
    2994 new element with value (key) should be inserted.
    2995 */
    2996 template <typename IterT, typename KeyT, typename CmpT>
    2997 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
    2998 {
    2999  size_t down = 0, up = (end - beg);
    3000  while(down < up)
    3001  {
    3002  const size_t mid = (down + up) / 2;
    3003  if(cmp(*(beg+mid), key))
    3004  {
    3005  down = mid + 1;
    3006  }
    3007  else
    3008  {
    3009  up = mid;
    3010  }
    3011  }
    3012  return beg + down;
    3013 }
    3014 
    3016 // Memory allocation
    3017 
    3018 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
    3019 {
    3020  if((pAllocationCallbacks != VMA_NULL) &&
    3021  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
    3022  {
    3023  return (*pAllocationCallbacks->pfnAllocation)(
    3024  pAllocationCallbacks->pUserData,
    3025  size,
    3026  alignment,
    3027  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    3028  }
    3029  else
    3030  {
    3031  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
    3032  }
    3033 }
    3034 
    3035 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
    3036 {
    3037  if((pAllocationCallbacks != VMA_NULL) &&
    3038  (pAllocationCallbacks->pfnFree != VMA_NULL))
    3039  {
    3040  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
    3041  }
    3042  else
    3043  {
    3044  VMA_SYSTEM_FREE(ptr);
    3045  }
    3046 }
    3047 
    3048 template<typename T>
    3049 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
    3050 {
    3051  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
    3052 }
    3053 
    3054 template<typename T>
    3055 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
    3056 {
    3057  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
    3058 }
    3059 
    3060 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
    3061 
    3062 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
    3063 
    3064 template<typename T>
    3065 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
    3066 {
    3067  ptr->~T();
    3068  VmaFree(pAllocationCallbacks, ptr);
    3069 }
    3070 
    3071 template<typename T>
    3072 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
    3073 {
    3074  if(ptr != VMA_NULL)
    3075  {
    3076  for(size_t i = count; i--; )
    3077  {
    3078  ptr[i].~T();
    3079  }
    3080  VmaFree(pAllocationCallbacks, ptr);
    3081  }
    3082 }
    3083 
    3084 // STL-compatible allocator.
    3085 template<typename T>
    3086 class VmaStlAllocator
    3087 {
    3088 public:
    3089  const VkAllocationCallbacks* const m_pCallbacks;
    3090  typedef T value_type;
    3091 
    3092  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
    3093  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
    3094 
    3095  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
    3096  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
    3097 
    3098  template<typename U>
    3099  bool operator==(const VmaStlAllocator<U>& rhs) const
    3100  {
    3101  return m_pCallbacks == rhs.m_pCallbacks;
    3102  }
    3103  template<typename U>
    3104  bool operator!=(const VmaStlAllocator<U>& rhs) const
    3105  {
    3106  return m_pCallbacks != rhs.m_pCallbacks;
    3107  }
    3108 
    3109  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
    3110 };
    3111 
    3112 #if VMA_USE_STL_VECTOR
    3113 
    3114 #define VmaVector std::vector
    3115 
    3116 template<typename T, typename allocatorT>
    3117 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
    3118 {
    3119  vec.insert(vec.begin() + index, item);
    3120 }
    3121 
    3122 template<typename T, typename allocatorT>
    3123 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
    3124 {
    3125  vec.erase(vec.begin() + index);
    3126 }
    3127 
    3128 #else // #if VMA_USE_STL_VECTOR
    3129 
    3130 /* Class with interface compatible with subset of std::vector.
    3131 T must be POD because constructors and destructors are not called and memcpy is
    3132 used for these objects. */
    3133 template<typename T, typename AllocatorT>
    3134 class VmaVector
    3135 {
    3136 public:
    3137  typedef T value_type;
    3138 
    3139  VmaVector(const AllocatorT& allocator) :
    3140  m_Allocator(allocator),
    3141  m_pArray(VMA_NULL),
    3142  m_Count(0),
    3143  m_Capacity(0)
    3144  {
    3145  }
    3146 
    3147  VmaVector(size_t count, const AllocatorT& allocator) :
    3148  m_Allocator(allocator),
    3149  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
    3150  m_Count(count),
    3151  m_Capacity(count)
    3152  {
    3153  }
    3154 
    3155  VmaVector(const VmaVector<T, AllocatorT>& src) :
    3156  m_Allocator(src.m_Allocator),
    3157  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
    3158  m_Count(src.m_Count),
    3159  m_Capacity(src.m_Count)
    3160  {
    3161  if(m_Count != 0)
    3162  {
    3163  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
    3164  }
    3165  }
    3166 
    3167  ~VmaVector()
    3168  {
    3169  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3170  }
    3171 
    3172  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
    3173  {
    3174  if(&rhs != this)
    3175  {
    3176  resize(rhs.m_Count);
    3177  if(m_Count != 0)
    3178  {
    3179  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
    3180  }
    3181  }
    3182  return *this;
    3183  }
    3184 
    3185  bool empty() const { return m_Count == 0; }
    3186  size_t size() const { return m_Count; }
    3187  T* data() { return m_pArray; }
    3188  const T* data() const { return m_pArray; }
    3189 
    3190  T& operator[](size_t index)
    3191  {
    3192  VMA_HEAVY_ASSERT(index < m_Count);
    3193  return m_pArray[index];
    3194  }
    3195  const T& operator[](size_t index) const
    3196  {
    3197  VMA_HEAVY_ASSERT(index < m_Count);
    3198  return m_pArray[index];
    3199  }
    3200 
    3201  T& front()
    3202  {
    3203  VMA_HEAVY_ASSERT(m_Count > 0);
    3204  return m_pArray[0];
    3205  }
    3206  const T& front() const
    3207  {
    3208  VMA_HEAVY_ASSERT(m_Count > 0);
    3209  return m_pArray[0];
    3210  }
    3211  T& back()
    3212  {
    3213  VMA_HEAVY_ASSERT(m_Count > 0);
    3214  return m_pArray[m_Count - 1];
    3215  }
    3216  const T& back() const
    3217  {
    3218  VMA_HEAVY_ASSERT(m_Count > 0);
    3219  return m_pArray[m_Count - 1];
    3220  }
    3221 
    3222  void reserve(size_t newCapacity, bool freeMemory = false)
    3223  {
    3224  newCapacity = VMA_MAX(newCapacity, m_Count);
    3225 
    3226  if((newCapacity < m_Capacity) && !freeMemory)
    3227  {
    3228  newCapacity = m_Capacity;
    3229  }
    3230 
    3231  if(newCapacity != m_Capacity)
    3232  {
    3233  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
    3234  if(m_Count != 0)
    3235  {
    3236  memcpy(newArray, m_pArray, m_Count * sizeof(T));
    3237  }
    3238  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3239  m_Capacity = newCapacity;
    3240  m_pArray = newArray;
    3241  }
    3242  }
    3243 
    3244  void resize(size_t newCount, bool freeMemory = false)
    3245  {
    3246  size_t newCapacity = m_Capacity;
    3247  if(newCount > m_Capacity)
    3248  {
    3249  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
    3250  }
    3251  else if(freeMemory)
    3252  {
    3253  newCapacity = newCount;
    3254  }
    3255 
    3256  if(newCapacity != m_Capacity)
    3257  {
    3258  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
    3259  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
    3260  if(elementsToCopy != 0)
    3261  {
    3262  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
    3263  }
    3264  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3265  m_Capacity = newCapacity;
    3266  m_pArray = newArray;
    3267  }
    3268 
    3269  m_Count = newCount;
    3270  }
    3271 
    3272  void clear(bool freeMemory = false)
    3273  {
    3274  resize(0, freeMemory);
    3275  }
    3276 
    3277  void insert(size_t index, const T& src)
    3278  {
    3279  VMA_HEAVY_ASSERT(index <= m_Count);
    3280  const size_t oldCount = size();
    3281  resize(oldCount + 1);
    3282  if(index < oldCount)
    3283  {
    3284  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
    3285  }
    3286  m_pArray[index] = src;
    3287  }
    3288 
    3289  void remove(size_t index)
    3290  {
    3291  VMA_HEAVY_ASSERT(index < m_Count);
    3292  const size_t oldCount = size();
    3293  if(index < oldCount - 1)
    3294  {
    3295  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
    3296  }
    3297  resize(oldCount - 1);
    3298  }
    3299 
    3300  void push_back(const T& src)
    3301  {
    3302  const size_t newIndex = size();
    3303  resize(newIndex + 1);
    3304  m_pArray[newIndex] = src;
    3305  }
    3306 
    3307  void pop_back()
    3308  {
    3309  VMA_HEAVY_ASSERT(m_Count > 0);
    3310  resize(size() - 1);
    3311  }
    3312 
    3313  void push_front(const T& src)
    3314  {
    3315  insert(0, src);
    3316  }
    3317 
    3318  void pop_front()
    3319  {
    3320  VMA_HEAVY_ASSERT(m_Count > 0);
    3321  remove(0);
    3322  }
    3323 
    3324  typedef T* iterator;
    3325 
    3326  iterator begin() { return m_pArray; }
    3327  iterator end() { return m_pArray + m_Count; }
    3328 
    3329 private:
    3330  AllocatorT m_Allocator;
    3331  T* m_pArray;
    3332  size_t m_Count;
    3333  size_t m_Capacity;
    3334 };
    3335 
    3336 template<typename T, typename allocatorT>
    3337 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
    3338 {
    3339  vec.insert(index, item);
    3340 }
    3341 
    3342 template<typename T, typename allocatorT>
    3343 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
    3344 {
    3345  vec.remove(index);
    3346 }
    3347 
    3348 #endif // #if VMA_USE_STL_VECTOR
    3349 
    3350 template<typename CmpLess, typename VectorT>
    3351 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
    3352 {
    3353  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    3354  vector.data(),
    3355  vector.data() + vector.size(),
    3356  value,
    3357  CmpLess()) - vector.data();
    3358  VmaVectorInsert(vector, indexToInsert, value);
    3359  return indexToInsert;
    3360 }
    3361 
    3362 template<typename CmpLess, typename VectorT>
    3363 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
    3364 {
    3365  CmpLess comparator;
    3366  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3367  vector.begin(),
    3368  vector.end(),
    3369  value,
    3370  comparator);
    3371  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
    3372  {
    3373  size_t indexToRemove = it - vector.begin();
    3374  VmaVectorRemove(vector, indexToRemove);
    3375  return true;
    3376  }
    3377  return false;
    3378 }
    3379 
    3380 template<typename CmpLess, typename VectorT>
    3381 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
    3382 {
    3383  CmpLess comparator;
    3384  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3385  vector.data(),
    3386  vector.data() + vector.size(),
    3387  value,
    3388  comparator);
    3389  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
    3390  {
    3391  return it - vector.begin();
    3392  }
    3393  else
    3394  {
    3395  return vector.size();
    3396  }
    3397 }
    3398 
    3400 // class VmaPoolAllocator
    3401 
    3402 /*
    3403 Allocator for objects of type T using a list of arrays (pools) to speed up
    3404 allocation. Number of elements that can be allocated is not bounded because
    3405 allocator can create multiple blocks.
    3406 */
    3407 template<typename T>
    3408 class VmaPoolAllocator
    3409 {
    3410  VMA_CLASS_NO_COPY(VmaPoolAllocator)
    3411 public:
    3412  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
    3413  ~VmaPoolAllocator();
    3414  void Clear();
    3415  T* Alloc();
    3416  void Free(T* ptr);
    3417 
    3418 private:
    3419  union Item
    3420  {
    3421  uint32_t NextFreeIndex;
    3422  T Value;
    3423  };
    3424 
    3425  struct ItemBlock
    3426  {
    3427  Item* pItems;
    3428  uint32_t FirstFreeIndex;
    3429  };
    3430 
    3431  const VkAllocationCallbacks* m_pAllocationCallbacks;
    3432  size_t m_ItemsPerBlock;
    3433  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
    3434 
    3435  ItemBlock& CreateNewBlock();
    3436 };
    3437 
    3438 template<typename T>
    3439 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
    3440  m_pAllocationCallbacks(pAllocationCallbacks),
    3441  m_ItemsPerBlock(itemsPerBlock),
    3442  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
    3443 {
    3444  VMA_ASSERT(itemsPerBlock > 0);
    3445 }
    3446 
    3447 template<typename T>
    3448 VmaPoolAllocator<T>::~VmaPoolAllocator()
    3449 {
    3450  Clear();
    3451 }
    3452 
    3453 template<typename T>
    3454 void VmaPoolAllocator<T>::Clear()
    3455 {
    3456  for(size_t i = m_ItemBlocks.size(); i--; )
    3457  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
    3458  m_ItemBlocks.clear();
    3459 }
    3460 
    3461 template<typename T>
    3462 T* VmaPoolAllocator<T>::Alloc()
    3463 {
    3464  for(size_t i = m_ItemBlocks.size(); i--; )
    3465  {
    3466  ItemBlock& block = m_ItemBlocks[i];
    3467  // This block has some free items: Use first one.
    3468  if(block.FirstFreeIndex != UINT32_MAX)
    3469  {
    3470  Item* const pItem = &block.pItems[block.FirstFreeIndex];
    3471  block.FirstFreeIndex = pItem->NextFreeIndex;
    3472  return &pItem->Value;
    3473  }
    3474  }
    3475 
    3476  // No block has free item: Create new one and use it.
    3477  ItemBlock& newBlock = CreateNewBlock();
    3478  Item* const pItem = &newBlock.pItems[0];
    3479  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
    3480  return &pItem->Value;
    3481 }
    3482 
    3483 template<typename T>
    3484 void VmaPoolAllocator<T>::Free(T* ptr)
    3485 {
    3486  // Search all memory blocks to find ptr.
    3487  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
    3488  {
    3489  ItemBlock& block = m_ItemBlocks[i];
    3490 
    3491  // Casting to union.
    3492  Item* pItemPtr;
    3493  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
    3494 
    3495  // Check if pItemPtr is in address range of this block.
    3496  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
    3497  {
    3498  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
    3499  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
    3500  block.FirstFreeIndex = index;
    3501  return;
    3502  }
    3503  }
    3504  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
    3505 }
    3506 
    3507 template<typename T>
    3508 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
    3509 {
    3510  ItemBlock newBlock = {
    3511  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
    3512 
    3513  m_ItemBlocks.push_back(newBlock);
    3514 
    3515  // Setup singly-linked list of all free items in this block.
    3516  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
    3517  newBlock.pItems[i].NextFreeIndex = i + 1;
    3518  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
    3519  return m_ItemBlocks.back();
    3520 }
    3521 
    3523 // class VmaRawList, VmaList
    3524 
    3525 #if VMA_USE_STL_LIST
    3526 
    3527 #define VmaList std::list
    3528 
    3529 #else // #if VMA_USE_STL_LIST
    3530 
    3531 template<typename T>
    3532 struct VmaListItem
    3533 {
    3534  VmaListItem* pPrev;
    3535  VmaListItem* pNext;
    3536  T Value;
    3537 };
    3538 
    3539 // Doubly linked list.
    3540 template<typename T>
    3541 class VmaRawList
    3542 {
    3543  VMA_CLASS_NO_COPY(VmaRawList)
    3544 public:
    3545  typedef VmaListItem<T> ItemType;
    3546 
    3547  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
    3548  ~VmaRawList();
    3549  void Clear();
    3550 
    3551  size_t GetCount() const { return m_Count; }
    3552  bool IsEmpty() const { return m_Count == 0; }
    3553 
    3554  ItemType* Front() { return m_pFront; }
    3555  const ItemType* Front() const { return m_pFront; }
    3556  ItemType* Back() { return m_pBack; }
    3557  const ItemType* Back() const { return m_pBack; }
    3558 
    3559  ItemType* PushBack();
    3560  ItemType* PushFront();
    3561  ItemType* PushBack(const T& value);
    3562  ItemType* PushFront(const T& value);
    3563  void PopBack();
    3564  void PopFront();
    3565 
    3566  // Item can be null - it means PushBack.
    3567  ItemType* InsertBefore(ItemType* pItem);
    3568  // Item can be null - it means PushFront.
    3569  ItemType* InsertAfter(ItemType* pItem);
    3570 
    3571  ItemType* InsertBefore(ItemType* pItem, const T& value);
    3572  ItemType* InsertAfter(ItemType* pItem, const T& value);
    3573 
    3574  void Remove(ItemType* pItem);
    3575 
    3576 private:
    3577  const VkAllocationCallbacks* const m_pAllocationCallbacks;
    3578  VmaPoolAllocator<ItemType> m_ItemAllocator;
    3579  ItemType* m_pFront;
    3580  ItemType* m_pBack;
    3581  size_t m_Count;
    3582 };
    3583 
    3584 template<typename T>
    3585 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
    3586  m_pAllocationCallbacks(pAllocationCallbacks),
    3587  m_ItemAllocator(pAllocationCallbacks, 128),
    3588  m_pFront(VMA_NULL),
    3589  m_pBack(VMA_NULL),
    3590  m_Count(0)
    3591 {
    3592 }
    3593 
    3594 template<typename T>
    3595 VmaRawList<T>::~VmaRawList()
    3596 {
    3597  // Intentionally not calling Clear, because that would be unnecessary
    3598  // computations to return all items to m_ItemAllocator as free.
    3599 }
    3600 
    3601 template<typename T>
    3602 void VmaRawList<T>::Clear()
    3603 {
    3604  if(IsEmpty() == false)
    3605  {
    3606  ItemType* pItem = m_pBack;
    3607  while(pItem != VMA_NULL)
    3608  {
    3609  ItemType* const pPrevItem = pItem->pPrev;
    3610  m_ItemAllocator.Free(pItem);
    3611  pItem = pPrevItem;
    3612  }
    3613  m_pFront = VMA_NULL;
    3614  m_pBack = VMA_NULL;
    3615  m_Count = 0;
    3616  }
    3617 }
    3618 
    3619 template<typename T>
    3620 VmaListItem<T>* VmaRawList<T>::PushBack()
    3621 {
    3622  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3623  pNewItem->pNext = VMA_NULL;
    3624  if(IsEmpty())
    3625  {
    3626  pNewItem->pPrev = VMA_NULL;
    3627  m_pFront = pNewItem;
    3628  m_pBack = pNewItem;
    3629  m_Count = 1;
    3630  }
    3631  else
    3632  {
    3633  pNewItem->pPrev = m_pBack;
    3634  m_pBack->pNext = pNewItem;
    3635  m_pBack = pNewItem;
    3636  ++m_Count;
    3637  }
    3638  return pNewItem;
    3639 }
    3640 
    3641 template<typename T>
    3642 VmaListItem<T>* VmaRawList<T>::PushFront()
    3643 {
    3644  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3645  pNewItem->pPrev = VMA_NULL;
    3646  if(IsEmpty())
    3647  {
    3648  pNewItem->pNext = VMA_NULL;
    3649  m_pFront = pNewItem;
    3650  m_pBack = pNewItem;
    3651  m_Count = 1;
    3652  }
    3653  else
    3654  {
    3655  pNewItem->pNext = m_pFront;
    3656  m_pFront->pPrev = pNewItem;
    3657  m_pFront = pNewItem;
    3658  ++m_Count;
    3659  }
    3660  return pNewItem;
    3661 }
    3662 
    3663 template<typename T>
    3664 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
    3665 {
    3666  ItemType* const pNewItem = PushBack();
    3667  pNewItem->Value = value;
    3668  return pNewItem;
    3669 }
    3670 
    3671 template<typename T>
    3672 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
    3673 {
    3674  ItemType* const pNewItem = PushFront();
    3675  pNewItem->Value = value;
    3676  return pNewItem;
    3677 }
    3678 
    3679 template<typename T>
    3680 void VmaRawList<T>::PopBack()
    3681 {
    3682  VMA_HEAVY_ASSERT(m_Count > 0);
    3683  ItemType* const pBackItem = m_pBack;
    3684  ItemType* const pPrevItem = pBackItem->pPrev;
    3685  if(pPrevItem != VMA_NULL)
    3686  {
    3687  pPrevItem->pNext = VMA_NULL;
    3688  }
    3689  m_pBack = pPrevItem;
    3690  m_ItemAllocator.Free(pBackItem);
    3691  --m_Count;
    3692 }
    3693 
    3694 template<typename T>
    3695 void VmaRawList<T>::PopFront()
    3696 {
    3697  VMA_HEAVY_ASSERT(m_Count > 0);
    3698  ItemType* const pFrontItem = m_pFront;
    3699  ItemType* const pNextItem = pFrontItem->pNext;
    3700  if(pNextItem != VMA_NULL)
    3701  {
    3702  pNextItem->pPrev = VMA_NULL;
    3703  }
    3704  m_pFront = pNextItem;
    3705  m_ItemAllocator.Free(pFrontItem);
    3706  --m_Count;
    3707 }
    3708 
    3709 template<typename T>
    3710 void VmaRawList<T>::Remove(ItemType* pItem)
    3711 {
    3712  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
    3713  VMA_HEAVY_ASSERT(m_Count > 0);
    3714 
    3715  if(pItem->pPrev != VMA_NULL)
    3716  {
    3717  pItem->pPrev->pNext = pItem->pNext;
    3718  }
    3719  else
    3720  {
    3721  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3722  m_pFront = pItem->pNext;
    3723  }
    3724 
    3725  if(pItem->pNext != VMA_NULL)
    3726  {
    3727  pItem->pNext->pPrev = pItem->pPrev;
    3728  }
    3729  else
    3730  {
    3731  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3732  m_pBack = pItem->pPrev;
    3733  }
    3734 
    3735  m_ItemAllocator.Free(pItem);
    3736  --m_Count;
    3737 }
    3738 
    3739 template<typename T>
    3740 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
    3741 {
    3742  if(pItem != VMA_NULL)
    3743  {
    3744  ItemType* const prevItem = pItem->pPrev;
    3745  ItemType* const newItem = m_ItemAllocator.Alloc();
    3746  newItem->pPrev = prevItem;
    3747  newItem->pNext = pItem;
    3748  pItem->pPrev = newItem;
    3749  if(prevItem != VMA_NULL)
    3750  {
    3751  prevItem->pNext = newItem;
    3752  }
    3753  else
    3754  {
    3755  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3756  m_pFront = newItem;
    3757  }
    3758  ++m_Count;
    3759  return newItem;
    3760  }
    3761  else
    3762  return PushBack();
    3763 }
    3764 
    3765 template<typename T>
    3766 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
    3767 {
    3768  if(pItem != VMA_NULL)
    3769  {
    3770  ItemType* const nextItem = pItem->pNext;
    3771  ItemType* const newItem = m_ItemAllocator.Alloc();
    3772  newItem->pNext = nextItem;
    3773  newItem->pPrev = pItem;
    3774  pItem->pNext = newItem;
    3775  if(nextItem != VMA_NULL)
    3776  {
    3777  nextItem->pPrev = newItem;
    3778  }
    3779  else
    3780  {
    3781  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3782  m_pBack = newItem;
    3783  }
    3784  ++m_Count;
    3785  return newItem;
    3786  }
    3787  else
    3788  return PushFront();
    3789 }
    3790 
    3791 template<typename T>
    3792 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
    3793 {
    3794  ItemType* const newItem = InsertBefore(pItem);
    3795  newItem->Value = value;
    3796  return newItem;
    3797 }
    3798 
    3799 template<typename T>
    3800 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
    3801 {
    3802  ItemType* const newItem = InsertAfter(pItem);
    3803  newItem->Value = value;
    3804  return newItem;
    3805 }
    3806 
    3807 template<typename T, typename AllocatorT>
    3808 class VmaList
    3809 {
    3810  VMA_CLASS_NO_COPY(VmaList)
    3811 public:
    3812  class iterator
    3813  {
    3814  public:
    3815  iterator() :
    3816  m_pList(VMA_NULL),
    3817  m_pItem(VMA_NULL)
    3818  {
    3819  }
    3820 
    3821  T& operator*() const
    3822  {
    3823  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3824  return m_pItem->Value;
    3825  }
    3826  T* operator->() const
    3827  {
    3828  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3829  return &m_pItem->Value;
    3830  }
    3831 
    3832  iterator& operator++()
    3833  {
    3834  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3835  m_pItem = m_pItem->pNext;
    3836  return *this;
    3837  }
    3838  iterator& operator--()
    3839  {
    3840  if(m_pItem != VMA_NULL)
    3841  {
    3842  m_pItem = m_pItem->pPrev;
    3843  }
    3844  else
    3845  {
    3846  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3847  m_pItem = m_pList->Back();
    3848  }
    3849  return *this;
    3850  }
    3851 
    3852  iterator operator++(int)
    3853  {
    3854  iterator result = *this;
    3855  ++*this;
    3856  return result;
    3857  }
    3858  iterator operator--(int)
    3859  {
    3860  iterator result = *this;
    3861  --*this;
    3862  return result;
    3863  }
    3864 
    3865  bool operator==(const iterator& rhs) const
    3866  {
    3867  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3868  return m_pItem == rhs.m_pItem;
    3869  }
    3870  bool operator!=(const iterator& rhs) const
    3871  {
    3872  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3873  return m_pItem != rhs.m_pItem;
    3874  }
    3875 
    3876  private:
    3877  VmaRawList<T>* m_pList;
    3878  VmaListItem<T>* m_pItem;
    3879 
    3880  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
    3881  m_pList(pList),
    3882  m_pItem(pItem)
    3883  {
    3884  }
    3885 
    3886  friend class VmaList<T, AllocatorT>;
    3887  };
    3888 
    3889  class const_iterator
    3890  {
    3891  public:
    3892  const_iterator() :
    3893  m_pList(VMA_NULL),
    3894  m_pItem(VMA_NULL)
    3895  {
    3896  }
    3897 
    3898  const_iterator(const iterator& src) :
    3899  m_pList(src.m_pList),
    3900  m_pItem(src.m_pItem)
    3901  {
    3902  }
    3903 
    3904  const T& operator*() const
    3905  {
    3906  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3907  return m_pItem->Value;
    3908  }
    3909  const T* operator->() const
    3910  {
    3911  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3912  return &m_pItem->Value;
    3913  }
    3914 
    3915  const_iterator& operator++()
    3916  {
    3917  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3918  m_pItem = m_pItem->pNext;
    3919  return *this;
    3920  }
    3921  const_iterator& operator--()
    3922  {
    3923  if(m_pItem != VMA_NULL)
    3924  {
    3925  m_pItem = m_pItem->pPrev;
    3926  }
    3927  else
    3928  {
    3929  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3930  m_pItem = m_pList->Back();
    3931  }
    3932  return *this;
    3933  }
    3934 
    3935  const_iterator operator++(int)
    3936  {
    3937  const_iterator result = *this;
    3938  ++*this;
    3939  return result;
    3940  }
    3941  const_iterator operator--(int)
    3942  {
    3943  const_iterator result = *this;
    3944  --*this;
    3945  return result;
    3946  }
    3947 
    3948  bool operator==(const const_iterator& rhs) const
    3949  {
    3950  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3951  return m_pItem == rhs.m_pItem;
    3952  }
    3953  bool operator!=(const const_iterator& rhs) const
    3954  {
    3955  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3956  return m_pItem != rhs.m_pItem;
    3957  }
    3958 
    3959  private:
    3960  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
    3961  m_pList(pList),
    3962  m_pItem(pItem)
    3963  {
    3964  }
    3965 
    3966  const VmaRawList<T>* m_pList;
    3967  const VmaListItem<T>* m_pItem;
    3968 
    3969  friend class VmaList<T, AllocatorT>;
    3970  };
    3971 
    3972  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
    3973 
    3974  bool empty() const { return m_RawList.IsEmpty(); }
    3975  size_t size() const { return m_RawList.GetCount(); }
    3976 
    3977  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
    3978  iterator end() { return iterator(&m_RawList, VMA_NULL); }
    3979 
    3980  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
    3981  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
    3982 
    3983  void clear() { m_RawList.Clear(); }
    3984  void push_back(const T& value) { m_RawList.PushBack(value); }
    3985  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
    3986  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
    3987 
    3988 private:
    3989  VmaRawList<T> m_RawList;
    3990 };
    3991 
    3992 #endif // #if VMA_USE_STL_LIST
    3993 
    3995 // class VmaMap
    3996 
    3997 // Unused in this version.
    3998 #if 0
    3999 
    4000 #if VMA_USE_STL_UNORDERED_MAP
    4001 
    4002 #define VmaPair std::pair
    4003 
    4004 #define VMA_MAP_TYPE(KeyT, ValueT) \
    4005  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
    4006 
    4007 #else // #if VMA_USE_STL_UNORDERED_MAP
    4008 
    4009 template<typename T1, typename T2>
    4010 struct VmaPair
    4011 {
    4012  T1 first;
    4013  T2 second;
    4014 
    4015  VmaPair() : first(), second() { }
    4016  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
    4017 };
    4018 
    4019 /* Class compatible with subset of interface of std::unordered_map.
    4020 KeyT, ValueT must be POD because they will be stored in VmaVector.
    4021 */
    4022 template<typename KeyT, typename ValueT>
    4023 class VmaMap
    4024 {
    4025 public:
    4026  typedef VmaPair<KeyT, ValueT> PairType;
    4027  typedef PairType* iterator;
    4028 
    4029  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
    4030 
    4031  iterator begin() { return m_Vector.begin(); }
    4032  iterator end() { return m_Vector.end(); }
    4033 
    4034  void insert(const PairType& pair);
    4035  iterator find(const KeyT& key);
    4036  void erase(iterator it);
    4037 
    4038 private:
    4039  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
    4040 };
    4041 
    4042 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
    4043 
    4044 template<typename FirstT, typename SecondT>
    4045 struct VmaPairFirstLess
    4046 {
    4047  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
    4048  {
    4049  return lhs.first < rhs.first;
    4050  }
    4051  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
    4052  {
    4053  return lhs.first < rhsFirst;
    4054  }
    4055 };
    4056 
    4057 template<typename KeyT, typename ValueT>
    4058 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
    4059 {
    4060  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    4061  m_Vector.data(),
    4062  m_Vector.data() + m_Vector.size(),
    4063  pair,
    4064  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
    4065  VmaVectorInsert(m_Vector, indexToInsert, pair);
    4066 }
    4067 
    4068 template<typename KeyT, typename ValueT>
    4069 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
    4070 {
    4071  PairType* it = VmaBinaryFindFirstNotLess(
    4072  m_Vector.data(),
    4073  m_Vector.data() + m_Vector.size(),
    4074  key,
    4075  VmaPairFirstLess<KeyT, ValueT>());
    4076  if((it != m_Vector.end()) && (it->first == key))
    4077  {
    4078  return it;
    4079  }
    4080  else
    4081  {
    4082  return m_Vector.end();
    4083  }
    4084 }
    4085 
    4086 template<typename KeyT, typename ValueT>
    4087 void VmaMap<KeyT, ValueT>::erase(iterator it)
    4088 {
    4089  VmaVectorRemove(m_Vector, it - m_Vector.begin());
    4090 }
    4091 
    4092 #endif // #if VMA_USE_STL_UNORDERED_MAP
    4093 
    4094 #endif // #if 0
    4095 
    4097 
    4098 class VmaDeviceMemoryBlock;
    4099 
    4100 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
    4101 
    4102 struct VmaAllocation_T
    4103 {
    4104  VMA_CLASS_NO_COPY(VmaAllocation_T)
    4105 private:
    4106  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
    4107 
    4108  enum FLAGS
    4109  {
    4110  FLAG_USER_DATA_STRING = 0x01,
    4111  };
    4112 
    4113 public:
    4114  enum ALLOCATION_TYPE
    4115  {
    4116  ALLOCATION_TYPE_NONE,
    4117  ALLOCATION_TYPE_BLOCK,
    4118  ALLOCATION_TYPE_DEDICATED,
    4119  };
    4120 
    4121  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
    4122  m_Alignment(1),
    4123  m_Size(0),
    4124  m_pUserData(VMA_NULL),
    4125  m_LastUseFrameIndex(currentFrameIndex),
    4126  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
    4127  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
    4128  m_MapCount(0),
    4129  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
    4130  {
    4131 #if VMA_STATS_STRING_ENABLED
    4132  m_CreationFrameIndex = currentFrameIndex;
    4133  m_BufferImageUsage = 0;
    4134 #endif
    4135  }
    4136 
    4137  ~VmaAllocation_T()
    4138  {
    4139  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
    4140 
    4141  // Check if owned string was freed.
    4142  VMA_ASSERT(m_pUserData == VMA_NULL);
    4143  }
    4144 
    4145  void InitBlockAllocation(
    4146  VmaPool hPool,
    4147  VmaDeviceMemoryBlock* block,
    4148  VkDeviceSize offset,
    4149  VkDeviceSize alignment,
    4150  VkDeviceSize size,
    4151  VmaSuballocationType suballocationType,
    4152  bool mapped,
    4153  bool canBecomeLost)
    4154  {
    4155  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4156  VMA_ASSERT(block != VMA_NULL);
    4157  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4158  m_Alignment = alignment;
    4159  m_Size = size;
    4160  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4161  m_SuballocationType = (uint8_t)suballocationType;
    4162  m_BlockAllocation.m_hPool = hPool;
    4163  m_BlockAllocation.m_Block = block;
    4164  m_BlockAllocation.m_Offset = offset;
    4165  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
    4166  }
    4167 
    4168  void InitLost()
    4169  {
    4170  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4171  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
    4172  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4173  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
    4174  m_BlockAllocation.m_Block = VMA_NULL;
    4175  m_BlockAllocation.m_Offset = 0;
    4176  m_BlockAllocation.m_CanBecomeLost = true;
    4177  }
    4178 
    4179  void ChangeBlockAllocation(
    4180  VmaAllocator hAllocator,
    4181  VmaDeviceMemoryBlock* block,
    4182  VkDeviceSize offset);
    4183 
    4184  // pMappedData not null means allocation is created with MAPPED flag.
    4185  void InitDedicatedAllocation(
    4186  uint32_t memoryTypeIndex,
    4187  VkDeviceMemory hMemory,
    4188  VmaSuballocationType suballocationType,
    4189  void* pMappedData,
    4190  VkDeviceSize size)
    4191  {
    4192  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4193  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
    4194  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
    4195  m_Alignment = 0;
    4196  m_Size = size;
    4197  m_SuballocationType = (uint8_t)suballocationType;
    4198  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4199  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
    4200  m_DedicatedAllocation.m_hMemory = hMemory;
    4201  m_DedicatedAllocation.m_pMappedData = pMappedData;
    4202  }
    4203 
    4204  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
    4205  VkDeviceSize GetAlignment() const { return m_Alignment; }
    4206  VkDeviceSize GetSize() const { return m_Size; }
    4207  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
    4208  void* GetUserData() const { return m_pUserData; }
    4209  void SetUserData(VmaAllocator hAllocator, void* pUserData);
    4210  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
    4211 
    4212  VmaDeviceMemoryBlock* GetBlock() const
    4213  {
    4214  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    4215  return m_BlockAllocation.m_Block;
    4216  }
    4217  VkDeviceSize GetOffset() const;
    4218  VkDeviceMemory GetMemory() const;
    4219  uint32_t GetMemoryTypeIndex() const;
    4220  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
    4221  void* GetMappedData() const;
    4222  bool CanBecomeLost() const;
    4223  VmaPool GetPool() const;
    4224 
    4225  uint32_t GetLastUseFrameIndex() const
    4226  {
    4227  return m_LastUseFrameIndex.load();
    4228  }
    4229  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
    4230  {
    4231  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
    4232  }
    4233  /*
    4234  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
    4235  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
    4236  - Else, returns false.
    4237 
    4238  If hAllocation is already lost, assert - you should not call it then.
    4239  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
    4240  */
    4241  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4242 
    4243  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
    4244  {
    4245  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
    4246  outInfo.blockCount = 1;
    4247  outInfo.allocationCount = 1;
    4248  outInfo.unusedRangeCount = 0;
    4249  outInfo.usedBytes = m_Size;
    4250  outInfo.unusedBytes = 0;
    4251  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
    4252  outInfo.unusedRangeSizeMin = UINT64_MAX;
    4253  outInfo.unusedRangeSizeMax = 0;
    4254  }
    4255 
    4256  void BlockAllocMap();
    4257  void BlockAllocUnmap();
    4258  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
    4259  void DedicatedAllocUnmap(VmaAllocator hAllocator);
    4260 
    4261 #if VMA_STATS_STRING_ENABLED
    4262  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
    4263  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
    4264 
    4265  void InitBufferImageUsage(uint32_t bufferImageUsage)
    4266  {
    4267  VMA_ASSERT(m_BufferImageUsage == 0);
    4268  m_BufferImageUsage = bufferImageUsage;
    4269  }
    4270 
    4271  void PrintParameters(class VmaJsonWriter& json) const;
    4272 #endif
    4273 
    4274 private:
    4275  VkDeviceSize m_Alignment;
    4276  VkDeviceSize m_Size;
    4277  void* m_pUserData;
    4278  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
    4279  uint8_t m_Type; // ALLOCATION_TYPE
    4280  uint8_t m_SuballocationType; // VmaSuballocationType
    4281  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
    4282  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
    4283  uint8_t m_MapCount;
    4284  uint8_t m_Flags; // enum FLAGS
    4285 
    4286  // Allocation out of VmaDeviceMemoryBlock.
    4287  struct BlockAllocation
    4288  {
    4289  VmaPool m_hPool; // Null if belongs to general memory.
    4290  VmaDeviceMemoryBlock* m_Block;
    4291  VkDeviceSize m_Offset;
    4292  bool m_CanBecomeLost;
    4293  };
    4294 
    4295  // Allocation for an object that has its own private VkDeviceMemory.
    4296  struct DedicatedAllocation
    4297  {
    4298  uint32_t m_MemoryTypeIndex;
    4299  VkDeviceMemory m_hMemory;
    4300  void* m_pMappedData; // Not null means memory is mapped.
    4301  };
    4302 
    4303  union
    4304  {
    4305  // Allocation out of VmaDeviceMemoryBlock.
    4306  BlockAllocation m_BlockAllocation;
    4307  // Allocation for an object that has its own private VkDeviceMemory.
    4308  DedicatedAllocation m_DedicatedAllocation;
    4309  };
    4310 
    4311 #if VMA_STATS_STRING_ENABLED
    4312  uint32_t m_CreationFrameIndex;
    4313  uint32_t m_BufferImageUsage; // 0 if unknown.
    4314 #endif
    4315 
    4316  void FreeUserDataString(VmaAllocator hAllocator);
    4317 };
    4318 
    4319 /*
    4320 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
    4321 allocated memory block or free.
    4322 */
    4323 struct VmaSuballocation
    4324 {
    4325  VkDeviceSize offset;
    4326  VkDeviceSize size;
    4327  VmaAllocation hAllocation;
    4328  VmaSuballocationType type;
    4329 };
    4330 
    4331 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
    4332 
    4333 // Cost of one additional allocation lost, as equivalent in bytes.
    4334 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
    4335 
    4336 /*
    4337 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
    4338 
    4339 If canMakeOtherLost was false:
    4340 - item points to a FREE suballocation.
    4341 - itemsToMakeLostCount is 0.
    4342 
    4343 If canMakeOtherLost was true:
    4344 - item points to first of sequence of suballocations, which are either FREE,
    4345  or point to VmaAllocations that can become lost.
    4346 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
    4347  the requested allocation to succeed.
    4348 */
    4349 struct VmaAllocationRequest
    4350 {
    4351  VkDeviceSize offset;
    4352  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
    4353  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
    4354  VmaSuballocationList::iterator item;
    4355  size_t itemsToMakeLostCount;
    4356 
    4357  VkDeviceSize CalcCost() const
    4358  {
    4359  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
    4360  }
    4361 };
    4362 
    4363 /*
    4364 Data structure used for bookkeeping of allocations and unused ranges of memory
    4365 in a single VkDeviceMemory block.
    4366 */
    4367 class VmaBlockMetadata
    4368 {
    4369  VMA_CLASS_NO_COPY(VmaBlockMetadata)
    4370 public:
    4371  VmaBlockMetadata(VmaAllocator hAllocator);
    4372  ~VmaBlockMetadata();
    4373  void Init(VkDeviceSize size);
    4374 
    4375  // Validates all data structures inside this object. If not valid, returns false.
    4376  bool Validate() const;
    4377  VkDeviceSize GetSize() const { return m_Size; }
    4378  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
    4379  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
    4380  VkDeviceSize GetUnusedRangeSizeMax() const;
    4381  // Returns true if this block is empty - contains only single free suballocation.
    4382  bool IsEmpty() const;
    4383 
    4384  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
    4385  void AddPoolStats(VmaPoolStats& inoutStats) const;
    4386 
    4387 #if VMA_STATS_STRING_ENABLED
    4388  void PrintDetailedMap(class VmaJsonWriter& json) const;
    4389 #endif
    4390 
    4391  // Tries to find a place for suballocation with given parameters inside this block.
    4392  // If succeeded, fills pAllocationRequest and returns true.
    4393  // If failed, returns false.
    4394  bool CreateAllocationRequest(
    4395  uint32_t currentFrameIndex,
    4396  uint32_t frameInUseCount,
    4397  VkDeviceSize bufferImageGranularity,
    4398  VkDeviceSize allocSize,
    4399  VkDeviceSize allocAlignment,
    4400  VmaSuballocationType allocType,
    4401  bool canMakeOtherLost,
    4402  VmaAllocationRequest* pAllocationRequest);
    4403 
    4404  bool MakeRequestedAllocationsLost(
    4405  uint32_t currentFrameIndex,
    4406  uint32_t frameInUseCount,
    4407  VmaAllocationRequest* pAllocationRequest);
    4408 
    4409  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4410 
    4411  VkResult CheckCorruption(const void* pBlockData);
    4412 
    4413  // Makes actual allocation based on request. Request must already be checked and valid.
    4414  void Alloc(
    4415  const VmaAllocationRequest& request,
    4416  VmaSuballocationType type,
    4417  VkDeviceSize allocSize,
    4418  VmaAllocation hAllocation);
    4419 
    4420  // Frees suballocation assigned to given memory region.
    4421  void Free(const VmaAllocation allocation);
    4422  void FreeAtOffset(VkDeviceSize offset);
    4423 
    4424 private:
    4425  VkDeviceSize m_Size;
    4426  uint32_t m_FreeCount;
    4427  VkDeviceSize m_SumFreeSize;
    4428  VmaSuballocationList m_Suballocations;
    4429  // Suballocations that are free and have size greater than certain threshold.
    4430  // Sorted by size, ascending.
    4431  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
    4432 
    4433  bool ValidateFreeSuballocationList() const;
    4434 
    4435  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
    4436  // If yes, fills pOffset and returns true. If no, returns false.
    4437  bool CheckAllocation(
    4438  uint32_t currentFrameIndex,
    4439  uint32_t frameInUseCount,
    4440  VkDeviceSize bufferImageGranularity,
    4441  VkDeviceSize allocSize,
    4442  VkDeviceSize allocAlignment,
    4443  VmaSuballocationType allocType,
    4444  VmaSuballocationList::const_iterator suballocItem,
    4445  bool canMakeOtherLost,
    4446  VkDeviceSize* pOffset,
    4447  size_t* itemsToMakeLostCount,
    4448  VkDeviceSize* pSumFreeSize,
    4449  VkDeviceSize* pSumItemSize) const;
    4450  // Given free suballocation, it merges it with following one, which must also be free.
    4451  void MergeFreeWithNext(VmaSuballocationList::iterator item);
    4452  // Releases given suballocation, making it free.
    4453  // Merges it with adjacent free suballocations if applicable.
    4454  // Returns iterator to new free suballocation at this place.
    4455  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
    4456  // Given free suballocation, it inserts it into sorted list of
    4457  // m_FreeSuballocationsBySize if it's suitable.
    4458  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
    4459  // Given free suballocation, it removes it from sorted list of
    4460  // m_FreeSuballocationsBySize if it's suitable.
    4461  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
    4462 };
    4463 
    4464 /*
    4465 Represents a single block of device memory (`VkDeviceMemory`) with all the
    4466 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
    4467 
    4468 Thread-safety: This class must be externally synchronized.
    4469 */
    4470 class VmaDeviceMemoryBlock
    4471 {
    4472  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
    4473 public:
    4474  VmaBlockMetadata m_Metadata;
    4475 
    4476  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
    4477 
    4478  ~VmaDeviceMemoryBlock()
    4479  {
    4480  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
    4481  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    4482  }
    4483 
    4484  // Always call after construction.
    4485  void Init(
    4486  uint32_t newMemoryTypeIndex,
    4487  VkDeviceMemory newMemory,
    4488  VkDeviceSize newSize,
    4489  uint32_t id);
    4490  // Always call before destruction.
    4491  void Destroy(VmaAllocator allocator);
    4492 
    4493  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
    4494  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4495  uint32_t GetId() const { return m_Id; }
    4496  void* GetMappedData() const { return m_pMappedData; }
    4497 
    4498  // Validates all data structures inside this object. If not valid, returns false.
    4499  bool Validate() const;
    4500 
    4501  VkResult CheckCorruption(VmaAllocator hAllocator);
    4502 
    4503  // ppData can be null.
    4504  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
    4505  void Unmap(VmaAllocator hAllocator, uint32_t count);
    4506 
    4507  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4508  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4509 
    4510  VkResult BindBufferMemory(
    4511  const VmaAllocator hAllocator,
    4512  const VmaAllocation hAllocation,
    4513  VkBuffer hBuffer);
    4514  VkResult BindImageMemory(
    4515  const VmaAllocator hAllocator,
    4516  const VmaAllocation hAllocation,
    4517  VkImage hImage);
    4518 
    4519 private:
    4520  uint32_t m_MemoryTypeIndex;
    4521  uint32_t m_Id;
    4522  VkDeviceMemory m_hMemory;
    4523 
    4524  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
    4525  // Also protects m_MapCount, m_pMappedData.
    4526  VMA_MUTEX m_Mutex;
    4527  uint32_t m_MapCount;
    4528  void* m_pMappedData;
    4529 };
    4530 
    4531 struct VmaPointerLess
    4532 {
    4533  bool operator()(const void* lhs, const void* rhs) const
    4534  {
    4535  return lhs < rhs;
    4536  }
    4537 };
    4538 
    4539 class VmaDefragmentator;
    4540 
    4541 /*
    4542 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
    4543 Vulkan memory type.
    4544 
    4545 Synchronized internally with a mutex.
    4546 */
    4547 struct VmaBlockVector
    4548 {
    4549  VMA_CLASS_NO_COPY(VmaBlockVector)
    4550 public:
    4551  VmaBlockVector(
    4552  VmaAllocator hAllocator,
    4553  uint32_t memoryTypeIndex,
    4554  VkDeviceSize preferredBlockSize,
    4555  size_t minBlockCount,
    4556  size_t maxBlockCount,
    4557  VkDeviceSize bufferImageGranularity,
    4558  uint32_t frameInUseCount,
    4559  bool isCustomPool);
    4560  ~VmaBlockVector();
    4561 
    4562  VkResult CreateMinBlocks();
    4563 
    4564  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4565  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
    4566  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
    4567  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
    4568 
    4569  void GetPoolStats(VmaPoolStats* pStats);
    4570 
    4571  bool IsEmpty() const { return m_Blocks.empty(); }
    4572  bool IsCorruptionDetectionEnabled() const;
    4573 
    4574  VkResult Allocate(
    4575  VmaPool hCurrentPool,
    4576  uint32_t currentFrameIndex,
    4577  VkDeviceSize size,
    4578  VkDeviceSize alignment,
    4579  const VmaAllocationCreateInfo& createInfo,
    4580  VmaSuballocationType suballocType,
    4581  VmaAllocation* pAllocation);
    4582 
    4583  void Free(
    4584  VmaAllocation hAllocation);
    4585 
    4586  // Adds statistics of this BlockVector to pStats.
    4587  void AddStats(VmaStats* pStats);
    4588 
    4589 #if VMA_STATS_STRING_ENABLED
    4590  void PrintDetailedMap(class VmaJsonWriter& json);
    4591 #endif
    4592 
    4593  void MakePoolAllocationsLost(
    4594  uint32_t currentFrameIndex,
    4595  size_t* pLostAllocationCount);
    4596  VkResult CheckCorruption();
    4597 
    4598  VmaDefragmentator* EnsureDefragmentator(
    4599  VmaAllocator hAllocator,
    4600  uint32_t currentFrameIndex);
    4601 
    4602  VkResult Defragment(
    4603  VmaDefragmentationStats* pDefragmentationStats,
    4604  VkDeviceSize& maxBytesToMove,
    4605  uint32_t& maxAllocationsToMove);
    4606 
    4607  void DestroyDefragmentator();
    4608 
    4609 private:
    4610  friend class VmaDefragmentator;
    4611 
    4612  const VmaAllocator m_hAllocator;
    4613  const uint32_t m_MemoryTypeIndex;
    4614  const VkDeviceSize m_PreferredBlockSize;
    4615  const size_t m_MinBlockCount;
    4616  const size_t m_MaxBlockCount;
    4617  const VkDeviceSize m_BufferImageGranularity;
    4618  const uint32_t m_FrameInUseCount;
    4619  const bool m_IsCustomPool;
    4620  VMA_MUTEX m_Mutex;
    4621  // Incrementally sorted by sumFreeSize, ascending.
    4622  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
    4623  /* There can be at most one allocation that is completely empty - a
    4624  hysteresis to avoid pessimistic case of alternating creation and destruction
    4625  of a VkDeviceMemory. */
    4626  bool m_HasEmptyBlock;
    4627  VmaDefragmentator* m_pDefragmentator;
    4628  uint32_t m_NextBlockId;
    4629 
    4630  VkDeviceSize CalcMaxBlockSize() const;
    4631 
    4632  // Finds and removes given block from vector.
    4633  void Remove(VmaDeviceMemoryBlock* pBlock);
    4634 
    4635  // Performs single step in sorting m_Blocks. They may not be fully sorted
    4636  // after this call.
    4637  void IncrementallySortBlocks();
    4638 
    4639  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
    4640 };
    4641 
    4642 struct VmaPool_T
    4643 {
    4644  VMA_CLASS_NO_COPY(VmaPool_T)
    4645 public:
    4646  VmaBlockVector m_BlockVector;
    4647 
    4648  VmaPool_T(
    4649  VmaAllocator hAllocator,
    4650  const VmaPoolCreateInfo& createInfo);
    4651  ~VmaPool_T();
    4652 
    4653  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
    4654  uint32_t GetId() const { return m_Id; }
    4655  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
    4656 
    4657 #if VMA_STATS_STRING_ENABLED
    4658  //void PrintDetailedMap(class VmaStringBuilder& sb);
    4659 #endif
    4660 
    4661 private:
    4662  uint32_t m_Id;
    4663 };
    4664 
    4665 class VmaDefragmentator
    4666 {
    4667  VMA_CLASS_NO_COPY(VmaDefragmentator)
    4668 private:
    4669  const VmaAllocator m_hAllocator;
    4670  VmaBlockVector* const m_pBlockVector;
    4671  uint32_t m_CurrentFrameIndex;
    4672  VkDeviceSize m_BytesMoved;
    4673  uint32_t m_AllocationsMoved;
    4674 
    4675  struct AllocationInfo
    4676  {
    4677  VmaAllocation m_hAllocation;
    4678  VkBool32* m_pChanged;
    4679 
    4680  AllocationInfo() :
    4681  m_hAllocation(VK_NULL_HANDLE),
    4682  m_pChanged(VMA_NULL)
    4683  {
    4684  }
    4685  };
    4686 
    4687  struct AllocationInfoSizeGreater
    4688  {
    4689  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
    4690  {
    4691  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
    4692  }
    4693  };
    4694 
    4695  // Used between AddAllocation and Defragment.
    4696  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4697 
    4698  struct BlockInfo
    4699  {
    4700  VmaDeviceMemoryBlock* m_pBlock;
    4701  bool m_HasNonMovableAllocations;
    4702  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4703 
    4704  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
    4705  m_pBlock(VMA_NULL),
    4706  m_HasNonMovableAllocations(true),
    4707  m_Allocations(pAllocationCallbacks),
    4708  m_pMappedDataForDefragmentation(VMA_NULL)
    4709  {
    4710  }
    4711 
    4712  void CalcHasNonMovableAllocations()
    4713  {
    4714  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
    4715  const size_t defragmentAllocCount = m_Allocations.size();
    4716  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
    4717  }
    4718 
    4719  void SortAllocationsBySizeDescecnding()
    4720  {
    4721  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
    4722  }
    4723 
    4724  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
    4725  void Unmap(VmaAllocator hAllocator);
    4726 
    4727  private:
    4728  // Not null if mapped for defragmentation only, not originally mapped.
    4729  void* m_pMappedDataForDefragmentation;
    4730  };
    4731 
    4732  struct BlockPointerLess
    4733  {
    4734  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
    4735  {
    4736  return pLhsBlockInfo->m_pBlock < pRhsBlock;
    4737  }
    4738  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4739  {
    4740  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
    4741  }
    4742  };
    4743 
    4744  // 1. Blocks with some non-movable allocations go first.
    4745  // 2. Blocks with smaller sumFreeSize go first.
    4746  struct BlockInfoCompareMoveDestination
    4747  {
    4748  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4749  {
    4750  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
    4751  {
    4752  return true;
    4753  }
    4754  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
    4755  {
    4756  return false;
    4757  }
    4758  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
    4759  {
    4760  return true;
    4761  }
    4762  return false;
    4763  }
    4764  };
    4765 
    4766  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
    4767  BlockInfoVector m_Blocks;
    4768 
    4769  VkResult DefragmentRound(
    4770  VkDeviceSize maxBytesToMove,
    4771  uint32_t maxAllocationsToMove);
    4772 
    4773  static bool MoveMakesSense(
    4774  size_t dstBlockIndex, VkDeviceSize dstOffset,
    4775  size_t srcBlockIndex, VkDeviceSize srcOffset);
    4776 
    4777 public:
    4778  VmaDefragmentator(
    4779  VmaAllocator hAllocator,
    4780  VmaBlockVector* pBlockVector,
    4781  uint32_t currentFrameIndex);
    4782 
    4783  ~VmaDefragmentator();
    4784 
    4785  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
    4786  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
    4787 
    4788  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
    4789 
    4790  VkResult Defragment(
    4791  VkDeviceSize maxBytesToMove,
    4792  uint32_t maxAllocationsToMove);
    4793 };
    4794 
    4795 #if VMA_RECORDING_ENABLED
    4796 
    4797 class VmaRecorder
    4798 {
    4799 public:
    4800  VmaRecorder();
    4801  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
    4802  ~VmaRecorder();
    4803 
    4804  void RecordCreateAllocator(uint32_t frameIndex);
    4805  void RecordDestroyAllocator(uint32_t frameIndex);
    4806  void RecordCreatePool(uint32_t frameIndex,
    4807  const VmaPoolCreateInfo& createInfo,
    4808  VmaPool pool);
    4809  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
    4810  void RecordAllocateMemory(uint32_t frameIndex,
    4811  const VkMemoryRequirements& vkMemReq,
    4812  const VmaAllocationCreateInfo& createInfo,
    4813  VmaAllocation allocation);
    4814  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    4815  const VkMemoryRequirements& vkMemReq,
    4816  bool requiresDedicatedAllocation,
    4817  bool prefersDedicatedAllocation,
    4818  const VmaAllocationCreateInfo& createInfo,
    4819  VmaAllocation allocation);
    4820  void RecordAllocateMemoryForImage(uint32_t frameIndex,
    4821  const VkMemoryRequirements& vkMemReq,
    4822  bool requiresDedicatedAllocation,
    4823  bool prefersDedicatedAllocation,
    4824  const VmaAllocationCreateInfo& createInfo,
    4825  VmaAllocation allocation);
    4826  void RecordFreeMemory(uint32_t frameIndex,
    4827  VmaAllocation allocation);
    4828  void RecordSetAllocationUserData(uint32_t frameIndex,
    4829  VmaAllocation allocation,
    4830  const void* pUserData);
    4831  void RecordCreateLostAllocation(uint32_t frameIndex,
    4832  VmaAllocation allocation);
    4833  void RecordMapMemory(uint32_t frameIndex,
    4834  VmaAllocation allocation);
    4835  void RecordUnmapMemory(uint32_t frameIndex,
    4836  VmaAllocation allocation);
    4837  void RecordFlushAllocation(uint32_t frameIndex,
    4838  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4839  void RecordInvalidateAllocation(uint32_t frameIndex,
    4840  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4841  void RecordCreateBuffer(uint32_t frameIndex,
    4842  const VkBufferCreateInfo& bufCreateInfo,
    4843  const VmaAllocationCreateInfo& allocCreateInfo,
    4844  VmaAllocation allocation);
    4845  void RecordCreateImage(uint32_t frameIndex,
    4846  const VkImageCreateInfo& imageCreateInfo,
    4847  const VmaAllocationCreateInfo& allocCreateInfo,
    4848  VmaAllocation allocation);
    4849  void RecordDestroyBuffer(uint32_t frameIndex,
    4850  VmaAllocation allocation);
    4851  void RecordDestroyImage(uint32_t frameIndex,
    4852  VmaAllocation allocation);
    4853  void RecordTouchAllocation(uint32_t frameIndex,
    4854  VmaAllocation allocation);
    4855  void RecordGetAllocationInfo(uint32_t frameIndex,
    4856  VmaAllocation allocation);
    4857 
    4858 private:
    4859  struct CallParams
    4860  {
    4861  uint32_t threadId;
    4862  double time;
    4863  };
    4864 
    4865  class UserDataString
    4866  {
    4867  public:
    4868  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
    4869  const char* GetString() const { return m_Str; }
    4870 
    4871  private:
    4872  char m_PtrStr[17];
    4873  const char* m_Str;
    4874  };
    4875 
    4876  bool m_UseMutex;
    4877  VmaRecordFlags m_Flags;
    4878  FILE* m_File;
    4879  VMA_MUTEX m_FileMutex;
    4880  int64_t m_Freq;
    4881  int64_t m_StartCounter;
    4882 
    4883  void GetBasicParams(CallParams& outParams);
    4884  void Flush();
    4885 };
    4886 
    4887 #endif // #if VMA_RECORDING_ENABLED
    4888 
    4889 // Main allocator object.
    4890 struct VmaAllocator_T
    4891 {
    4892  VMA_CLASS_NO_COPY(VmaAllocator_T)
    4893 public:
    4894  bool m_UseMutex;
    4895  bool m_UseKhrDedicatedAllocation;
    4896  VkDevice m_hDevice;
    4897  bool m_AllocationCallbacksSpecified;
    4898  VkAllocationCallbacks m_AllocationCallbacks;
    4899  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
    4900 
    4901  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
    4902  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
    4903  VMA_MUTEX m_HeapSizeLimitMutex;
    4904 
    4905  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
    4906  VkPhysicalDeviceMemoryProperties m_MemProps;
    4907 
    4908  // Default pools.
    4909  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
    4910 
    4911  // Each vector is sorted by memory (handle value).
    4912  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
    4913  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
    4914  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
    4915 
    4916  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
    4917  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
    4918  ~VmaAllocator_T();
    4919 
    4920  const VkAllocationCallbacks* GetAllocationCallbacks() const
    4921  {
    4922  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
    4923  }
    4924  const VmaVulkanFunctions& GetVulkanFunctions() const
    4925  {
    4926  return m_VulkanFunctions;
    4927  }
    4928 
    4929  VkDeviceSize GetBufferImageGranularity() const
    4930  {
    4931  return VMA_MAX(
    4932  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
    4933  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
    4934  }
    4935 
    4936  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
    4937  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
    4938 
    4939  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
    4940  {
    4941  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
    4942  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
    4943  }
    4944  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
    4945  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
    4946  {
    4947  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
    4948  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    4949  }
    4950  // Minimum alignment for all allocations in specific memory type.
    4951  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
    4952  {
    4953  return IsMemoryTypeNonCoherent(memTypeIndex) ?
    4954  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
    4955  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
    4956  }
    4957 
    4958  bool IsIntegratedGpu() const
    4959  {
    4960  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
    4961  }
    4962 
    4963 #if VMA_RECORDING_ENABLED
    4964  VmaRecorder* GetRecorder() const { return m_pRecorder; }
    4965 #endif
    4966 
    4967  void GetBufferMemoryRequirements(
    4968  VkBuffer hBuffer,
    4969  VkMemoryRequirements& memReq,
    4970  bool& requiresDedicatedAllocation,
    4971  bool& prefersDedicatedAllocation) const;
    4972  void GetImageMemoryRequirements(
    4973  VkImage hImage,
    4974  VkMemoryRequirements& memReq,
    4975  bool& requiresDedicatedAllocation,
    4976  bool& prefersDedicatedAllocation) const;
    4977 
    4978  // Main allocation function.
    4979  VkResult AllocateMemory(
    4980  const VkMemoryRequirements& vkMemReq,
    4981  bool requiresDedicatedAllocation,
    4982  bool prefersDedicatedAllocation,
    4983  VkBuffer dedicatedBuffer,
    4984  VkImage dedicatedImage,
    4985  const VmaAllocationCreateInfo& createInfo,
    4986  VmaSuballocationType suballocType,
    4987  VmaAllocation* pAllocation);
    4988 
    4989  // Main deallocation function.
    4990  void FreeMemory(const VmaAllocation allocation);
    4991 
    4992  void CalculateStats(VmaStats* pStats);
    4993 
    4994 #if VMA_STATS_STRING_ENABLED
    4995  void PrintDetailedMap(class VmaJsonWriter& json);
    4996 #endif
    4997 
    4998  VkResult Defragment(
    4999  VmaAllocation* pAllocations,
    5000  size_t allocationCount,
    5001  VkBool32* pAllocationsChanged,
    5002  const VmaDefragmentationInfo* pDefragmentationInfo,
    5003  VmaDefragmentationStats* pDefragmentationStats);
    5004 
    5005  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
    5006  bool TouchAllocation(VmaAllocation hAllocation);
    5007 
    5008  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
    5009  void DestroyPool(VmaPool pool);
    5010  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
    5011 
    5012  void SetCurrentFrameIndex(uint32_t frameIndex);
    5013  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
    5014 
    5015  void MakePoolAllocationsLost(
    5016  VmaPool hPool,
    5017  size_t* pLostAllocationCount);
    5018  VkResult CheckPoolCorruption(VmaPool hPool);
    5019  VkResult CheckCorruption(uint32_t memoryTypeBits);
    5020 
    5021  void CreateLostAllocation(VmaAllocation* pAllocation);
    5022 
    5023  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
    5024  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
    5025 
    5026  VkResult Map(VmaAllocation hAllocation, void** ppData);
    5027  void Unmap(VmaAllocation hAllocation);
    5028 
    5029  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
    5030  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
    5031 
    5032  void FlushOrInvalidateAllocation(
    5033  VmaAllocation hAllocation,
    5034  VkDeviceSize offset, VkDeviceSize size,
    5035  VMA_CACHE_OPERATION op);
    5036 
    5037  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
    5038 
    5039 private:
    5040  VkDeviceSize m_PreferredLargeHeapBlockSize;
    5041 
    5042  VkPhysicalDevice m_PhysicalDevice;
    5043  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
    5044 
    5045  VMA_MUTEX m_PoolsMutex;
    5046  // Protected by m_PoolsMutex. Sorted by pointer value.
    5047  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
    5048  uint32_t m_NextPoolId;
    5049 
    5050  VmaVulkanFunctions m_VulkanFunctions;
    5051 
    5052 #if VMA_RECORDING_ENABLED
    5053  VmaRecorder* m_pRecorder;
    5054 #endif
    5055 
    5056  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
    5057 
    5058  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
    5059 
    5060  VkResult AllocateMemoryOfType(
    5061  VkDeviceSize size,
    5062  VkDeviceSize alignment,
    5063  bool dedicatedAllocation,
    5064  VkBuffer dedicatedBuffer,
    5065  VkImage dedicatedImage,
    5066  const VmaAllocationCreateInfo& createInfo,
    5067  uint32_t memTypeIndex,
    5068  VmaSuballocationType suballocType,
    5069  VmaAllocation* pAllocation);
    5070 
    5071  // Allocates and registers new VkDeviceMemory specifically for single allocation.
    5072  VkResult AllocateDedicatedMemory(
    5073  VkDeviceSize size,
    5074  VmaSuballocationType suballocType,
    5075  uint32_t memTypeIndex,
    5076  bool map,
    5077  bool isUserDataString,
    5078  void* pUserData,
    5079  VkBuffer dedicatedBuffer,
    5080  VkImage dedicatedImage,
    5081  VmaAllocation* pAllocation);
    5082 
    5083  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
    5084  void FreeDedicatedMemory(VmaAllocation allocation);
    5085 };
    5086 
    5088 // Memory allocation #2 after VmaAllocator_T definition
    5089 
    5090 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
    5091 {
    5092  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
    5093 }
    5094 
    5095 static void VmaFree(VmaAllocator hAllocator, void* ptr)
    5096 {
    5097  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
    5098 }
    5099 
    5100 template<typename T>
    5101 static T* VmaAllocate(VmaAllocator hAllocator)
    5102 {
    5103  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
    5104 }
    5105 
    5106 template<typename T>
    5107 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
    5108 {
    5109  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
    5110 }
    5111 
    5112 template<typename T>
    5113 static void vma_delete(VmaAllocator hAllocator, T* ptr)
    5114 {
    5115  if(ptr != VMA_NULL)
    5116  {
    5117  ptr->~T();
    5118  VmaFree(hAllocator, ptr);
    5119  }
    5120 }
    5121 
    5122 template<typename T>
    5123 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
    5124 {
    5125  if(ptr != VMA_NULL)
    5126  {
    5127  for(size_t i = count; i--; )
    5128  ptr[i].~T();
    5129  VmaFree(hAllocator, ptr);
    5130  }
    5131 }
    5132 
    5134 // VmaStringBuilder
    5135 
    5136 #if VMA_STATS_STRING_ENABLED
    5137 
    5138 class VmaStringBuilder
    5139 {
    5140 public:
    5141  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
    5142  size_t GetLength() const { return m_Data.size(); }
    5143  const char* GetData() const { return m_Data.data(); }
    5144 
    5145  void Add(char ch) { m_Data.push_back(ch); }
    5146  void Add(const char* pStr);
    5147  void AddNewLine() { Add('\n'); }
    5148  void AddNumber(uint32_t num);
    5149  void AddNumber(uint64_t num);
    5150  void AddPointer(const void* ptr);
    5151 
    5152 private:
    5153  VmaVector< char, VmaStlAllocator<char> > m_Data;
    5154 };
    5155 
    5156 void VmaStringBuilder::Add(const char* pStr)
    5157 {
    5158  const size_t strLen = strlen(pStr);
    5159  if(strLen > 0)
    5160  {
    5161  const size_t oldCount = m_Data.size();
    5162  m_Data.resize(oldCount + strLen);
    5163  memcpy(m_Data.data() + oldCount, pStr, strLen);
    5164  }
    5165 }
    5166 
    5167 void VmaStringBuilder::AddNumber(uint32_t num)
    5168 {
    5169  char buf[11];
    5170  VmaUint32ToStr(buf, sizeof(buf), num);
    5171  Add(buf);
    5172 }
    5173 
    5174 void VmaStringBuilder::AddNumber(uint64_t num)
    5175 {
    5176  char buf[21];
    5177  VmaUint64ToStr(buf, sizeof(buf), num);
    5178  Add(buf);
    5179 }
    5180 
    5181 void VmaStringBuilder::AddPointer(const void* ptr)
    5182 {
    5183  char buf[21];
    5184  VmaPtrToStr(buf, sizeof(buf), ptr);
    5185  Add(buf);
    5186 }
    5187 
    5188 #endif // #if VMA_STATS_STRING_ENABLED
    5189 
    5191 // VmaJsonWriter
    5192 
    5193 #if VMA_STATS_STRING_ENABLED
    5194 
    5195 class VmaJsonWriter
    5196 {
    5197  VMA_CLASS_NO_COPY(VmaJsonWriter)
    5198 public:
    5199  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
    5200  ~VmaJsonWriter();
    5201 
    5202  void BeginObject(bool singleLine = false);
    5203  void EndObject();
    5204 
    5205  void BeginArray(bool singleLine = false);
    5206  void EndArray();
    5207 
    5208  void WriteString(const char* pStr);
    5209  void BeginString(const char* pStr = VMA_NULL);
    5210  void ContinueString(const char* pStr);
    5211  void ContinueString(uint32_t n);
    5212  void ContinueString(uint64_t n);
    5213  void ContinueString_Pointer(const void* ptr);
    5214  void EndString(const char* pStr = VMA_NULL);
    5215 
    5216  void WriteNumber(uint32_t n);
    5217  void WriteNumber(uint64_t n);
    5218  void WriteBool(bool b);
    5219  void WriteNull();
    5220 
    5221 private:
    5222  static const char* const INDENT;
    5223 
    5224  enum COLLECTION_TYPE
    5225  {
    5226  COLLECTION_TYPE_OBJECT,
    5227  COLLECTION_TYPE_ARRAY,
    5228  };
    5229  struct StackItem
    5230  {
    5231  COLLECTION_TYPE type;
    5232  uint32_t valueCount;
    5233  bool singleLineMode;
    5234  };
    5235 
    5236  VmaStringBuilder& m_SB;
    5237  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
    5238  bool m_InsideString;
    5239 
    5240  void BeginValue(bool isString);
    5241  void WriteIndent(bool oneLess = false);
    5242 };
    5243 
    5244 const char* const VmaJsonWriter::INDENT = " ";
    5245 
    5246 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
    5247  m_SB(sb),
    5248  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
    5249  m_InsideString(false)
    5250 {
    5251 }
    5252 
    5253 VmaJsonWriter::~VmaJsonWriter()
    5254 {
    5255  VMA_ASSERT(!m_InsideString);
    5256  VMA_ASSERT(m_Stack.empty());
    5257 }
    5258 
    5259 void VmaJsonWriter::BeginObject(bool singleLine)
    5260 {
    5261  VMA_ASSERT(!m_InsideString);
    5262 
    5263  BeginValue(false);
    5264  m_SB.Add('{');
    5265 
    5266  StackItem item;
    5267  item.type = COLLECTION_TYPE_OBJECT;
    5268  item.valueCount = 0;
    5269  item.singleLineMode = singleLine;
    5270  m_Stack.push_back(item);
    5271 }
    5272 
    5273 void VmaJsonWriter::EndObject()
    5274 {
    5275  VMA_ASSERT(!m_InsideString);
    5276 
    5277  WriteIndent(true);
    5278  m_SB.Add('}');
    5279 
    5280  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
    5281  m_Stack.pop_back();
    5282 }
    5283 
    5284 void VmaJsonWriter::BeginArray(bool singleLine)
    5285 {
    5286  VMA_ASSERT(!m_InsideString);
    5287 
    5288  BeginValue(false);
    5289  m_SB.Add('[');
    5290 
    5291  StackItem item;
    5292  item.type = COLLECTION_TYPE_ARRAY;
    5293  item.valueCount = 0;
    5294  item.singleLineMode = singleLine;
    5295  m_Stack.push_back(item);
    5296 }
    5297 
    5298 void VmaJsonWriter::EndArray()
    5299 {
    5300  VMA_ASSERT(!m_InsideString);
    5301 
    5302  WriteIndent(true);
    5303  m_SB.Add(']');
    5304 
    5305  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
    5306  m_Stack.pop_back();
    5307 }
    5308 
    5309 void VmaJsonWriter::WriteString(const char* pStr)
    5310 {
    5311  BeginString(pStr);
    5312  EndString();
    5313 }
    5314 
    5315 void VmaJsonWriter::BeginString(const char* pStr)
    5316 {
    5317  VMA_ASSERT(!m_InsideString);
    5318 
    5319  BeginValue(true);
    5320  m_SB.Add('"');
    5321  m_InsideString = true;
    5322  if(pStr != VMA_NULL && pStr[0] != '\0')
    5323  {
    5324  ContinueString(pStr);
    5325  }
    5326 }
    5327 
    5328 void VmaJsonWriter::ContinueString(const char* pStr)
    5329 {
    5330  VMA_ASSERT(m_InsideString);
    5331 
    5332  const size_t strLen = strlen(pStr);
    5333  for(size_t i = 0; i < strLen; ++i)
    5334  {
    5335  char ch = pStr[i];
    5336  if(ch == '\\')
    5337  {
    5338  m_SB.Add("\\\\");
    5339  }
    5340  else if(ch == '"')
    5341  {
    5342  m_SB.Add("\\\"");
    5343  }
    5344  else if(ch >= 32)
    5345  {
    5346  m_SB.Add(ch);
    5347  }
    5348  else switch(ch)
    5349  {
    5350  case '\b':
    5351  m_SB.Add("\\b");
    5352  break;
    5353  case '\f':
    5354  m_SB.Add("\\f");
    5355  break;
    5356  case '\n':
    5357  m_SB.Add("\\n");
    5358  break;
    5359  case '\r':
    5360  m_SB.Add("\\r");
    5361  break;
    5362  case '\t':
    5363  m_SB.Add("\\t");
    5364  break;
    5365  default:
    5366  VMA_ASSERT(0 && "Character not currently supported.");
    5367  break;
    5368  }
    5369  }
    5370 }
    5371 
    5372 void VmaJsonWriter::ContinueString(uint32_t n)
    5373 {
    5374  VMA_ASSERT(m_InsideString);
    5375  m_SB.AddNumber(n);
    5376 }
    5377 
    5378 void VmaJsonWriter::ContinueString(uint64_t n)
    5379 {
    5380  VMA_ASSERT(m_InsideString);
    5381  m_SB.AddNumber(n);
    5382 }
    5383 
    5384 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
    5385 {
    5386  VMA_ASSERT(m_InsideString);
    5387  m_SB.AddPointer(ptr);
    5388 }
    5389 
    5390 void VmaJsonWriter::EndString(const char* pStr)
    5391 {
    5392  VMA_ASSERT(m_InsideString);
    5393  if(pStr != VMA_NULL && pStr[0] != '\0')
    5394  {
    5395  ContinueString(pStr);
    5396  }
    5397  m_SB.Add('"');
    5398  m_InsideString = false;
    5399 }
    5400 
    5401 void VmaJsonWriter::WriteNumber(uint32_t n)
    5402 {
    5403  VMA_ASSERT(!m_InsideString);
    5404  BeginValue(false);
    5405  m_SB.AddNumber(n);
    5406 }
    5407 
    5408 void VmaJsonWriter::WriteNumber(uint64_t n)
    5409 {
    5410  VMA_ASSERT(!m_InsideString);
    5411  BeginValue(false);
    5412  m_SB.AddNumber(n);
    5413 }
    5414 
    5415 void VmaJsonWriter::WriteBool(bool b)
    5416 {
    5417  VMA_ASSERT(!m_InsideString);
    5418  BeginValue(false);
    5419  m_SB.Add(b ? "true" : "false");
    5420 }
    5421 
    5422 void VmaJsonWriter::WriteNull()
    5423 {
    5424  VMA_ASSERT(!m_InsideString);
    5425  BeginValue(false);
    5426  m_SB.Add("null");
    5427 }
    5428 
    5429 void VmaJsonWriter::BeginValue(bool isString)
    5430 {
    5431  if(!m_Stack.empty())
    5432  {
    5433  StackItem& currItem = m_Stack.back();
    5434  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5435  currItem.valueCount % 2 == 0)
    5436  {
    5437  VMA_ASSERT(isString);
    5438  }
    5439 
    5440  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5441  currItem.valueCount % 2 != 0)
    5442  {
    5443  m_SB.Add(": ");
    5444  }
    5445  else if(currItem.valueCount > 0)
    5446  {
    5447  m_SB.Add(", ");
    5448  WriteIndent();
    5449  }
    5450  else
    5451  {
    5452  WriteIndent();
    5453  }
    5454  ++currItem.valueCount;
    5455  }
    5456 }
    5457 
    5458 void VmaJsonWriter::WriteIndent(bool oneLess)
    5459 {
    5460  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
    5461  {
    5462  m_SB.AddNewLine();
    5463 
    5464  size_t count = m_Stack.size();
    5465  if(count > 0 && oneLess)
    5466  {
    5467  --count;
    5468  }
    5469  for(size_t i = 0; i < count; ++i)
    5470  {
    5471  m_SB.Add(INDENT);
    5472  }
    5473  }
    5474 }
    5475 
    5476 #endif // #if VMA_STATS_STRING_ENABLED
    5477 
    5479 
    5480 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
    5481 {
    5482  if(IsUserDataString())
    5483  {
    5484  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
    5485 
    5486  FreeUserDataString(hAllocator);
    5487 
    5488  if(pUserData != VMA_NULL)
    5489  {
    5490  const char* const newStrSrc = (char*)pUserData;
    5491  const size_t newStrLen = strlen(newStrSrc);
    5492  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
    5493  memcpy(newStrDst, newStrSrc, newStrLen + 1);
    5494  m_pUserData = newStrDst;
    5495  }
    5496  }
    5497  else
    5498  {
    5499  m_pUserData = pUserData;
    5500  }
    5501 }
    5502 
    5503 void VmaAllocation_T::ChangeBlockAllocation(
    5504  VmaAllocator hAllocator,
    5505  VmaDeviceMemoryBlock* block,
    5506  VkDeviceSize offset)
    5507 {
    5508  VMA_ASSERT(block != VMA_NULL);
    5509  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5510 
    5511  // Move mapping reference counter from old block to new block.
    5512  if(block != m_BlockAllocation.m_Block)
    5513  {
    5514  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
    5515  if(IsPersistentMap())
    5516  ++mapRefCount;
    5517  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
    5518  block->Map(hAllocator, mapRefCount, VMA_NULL);
    5519  }
    5520 
    5521  m_BlockAllocation.m_Block = block;
    5522  m_BlockAllocation.m_Offset = offset;
    5523 }
    5524 
    5525 VkDeviceSize VmaAllocation_T::GetOffset() const
    5526 {
    5527  switch(m_Type)
    5528  {
    5529  case ALLOCATION_TYPE_BLOCK:
    5530  return m_BlockAllocation.m_Offset;
    5531  case ALLOCATION_TYPE_DEDICATED:
    5532  return 0;
    5533  default:
    5534  VMA_ASSERT(0);
    5535  return 0;
    5536  }
    5537 }
    5538 
    5539 VkDeviceMemory VmaAllocation_T::GetMemory() const
    5540 {
    5541  switch(m_Type)
    5542  {
    5543  case ALLOCATION_TYPE_BLOCK:
    5544  return m_BlockAllocation.m_Block->GetDeviceMemory();
    5545  case ALLOCATION_TYPE_DEDICATED:
    5546  return m_DedicatedAllocation.m_hMemory;
    5547  default:
    5548  VMA_ASSERT(0);
    5549  return VK_NULL_HANDLE;
    5550  }
    5551 }
    5552 
    5553 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
    5554 {
    5555  switch(m_Type)
    5556  {
    5557  case ALLOCATION_TYPE_BLOCK:
    5558  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
    5559  case ALLOCATION_TYPE_DEDICATED:
    5560  return m_DedicatedAllocation.m_MemoryTypeIndex;
    5561  default:
    5562  VMA_ASSERT(0);
    5563  return UINT32_MAX;
    5564  }
    5565 }
    5566 
    5567 void* VmaAllocation_T::GetMappedData() const
    5568 {
    5569  switch(m_Type)
    5570  {
    5571  case ALLOCATION_TYPE_BLOCK:
    5572  if(m_MapCount != 0)
    5573  {
    5574  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
    5575  VMA_ASSERT(pBlockData != VMA_NULL);
    5576  return (char*)pBlockData + m_BlockAllocation.m_Offset;
    5577  }
    5578  else
    5579  {
    5580  return VMA_NULL;
    5581  }
    5582  break;
    5583  case ALLOCATION_TYPE_DEDICATED:
    5584  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
    5585  return m_DedicatedAllocation.m_pMappedData;
    5586  default:
    5587  VMA_ASSERT(0);
    5588  return VMA_NULL;
    5589  }
    5590 }
    5591 
    5592 bool VmaAllocation_T::CanBecomeLost() const
    5593 {
    5594  switch(m_Type)
    5595  {
    5596  case ALLOCATION_TYPE_BLOCK:
    5597  return m_BlockAllocation.m_CanBecomeLost;
    5598  case ALLOCATION_TYPE_DEDICATED:
    5599  return false;
    5600  default:
    5601  VMA_ASSERT(0);
    5602  return false;
    5603  }
    5604 }
    5605 
    5606 VmaPool VmaAllocation_T::GetPool() const
    5607 {
    5608  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5609  return m_BlockAllocation.m_hPool;
    5610 }
    5611 
    5612 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    5613 {
    5614  VMA_ASSERT(CanBecomeLost());
    5615 
    5616  /*
    5617  Warning: This is a carefully designed algorithm.
    5618  Do not modify unless you really know what you're doing :)
    5619  */
    5620  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
    5621  for(;;)
    5622  {
    5623  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    5624  {
    5625  VMA_ASSERT(0);
    5626  return false;
    5627  }
    5628  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
    5629  {
    5630  return false;
    5631  }
    5632  else // Last use time earlier than current time.
    5633  {
    5634  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
    5635  {
    5636  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
    5637  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
    5638  return true;
    5639  }
    5640  }
    5641  }
    5642 }
    5643 
    5644 #if VMA_STATS_STRING_ENABLED
    5645 
    5646 // Correspond to values of enum VmaSuballocationType.
    5647 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
    5648  "FREE",
    5649  "UNKNOWN",
    5650  "BUFFER",
    5651  "IMAGE_UNKNOWN",
    5652  "IMAGE_LINEAR",
    5653  "IMAGE_OPTIMAL",
    5654 };
    5655 
    5656 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
    5657 {
    5658  json.WriteString("Type");
    5659  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
    5660 
    5661  json.WriteString("Size");
    5662  json.WriteNumber(m_Size);
    5663 
    5664  if(m_pUserData != VMA_NULL)
    5665  {
    5666  json.WriteString("UserData");
    5667  if(IsUserDataString())
    5668  {
    5669  json.WriteString((const char*)m_pUserData);
    5670  }
    5671  else
    5672  {
    5673  json.BeginString();
    5674  json.ContinueString_Pointer(m_pUserData);
    5675  json.EndString();
    5676  }
    5677  }
    5678 
    5679  json.WriteString("CreationFrameIndex");
    5680  json.WriteNumber(m_CreationFrameIndex);
    5681 
    5682  json.WriteString("LastUseFrameIndex");
    5683  json.WriteNumber(GetLastUseFrameIndex());
    5684 
    5685  if(m_BufferImageUsage != 0)
    5686  {
    5687  json.WriteString("Usage");
    5688  json.WriteNumber(m_BufferImageUsage);
    5689  }
    5690 }
    5691 
    5692 #endif
    5693 
    5694 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
    5695 {
    5696  VMA_ASSERT(IsUserDataString());
    5697  if(m_pUserData != VMA_NULL)
    5698  {
    5699  char* const oldStr = (char*)m_pUserData;
    5700  const size_t oldStrLen = strlen(oldStr);
    5701  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
    5702  m_pUserData = VMA_NULL;
    5703  }
    5704 }
    5705 
    5706 void VmaAllocation_T::BlockAllocMap()
    5707 {
    5708  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5709 
    5710  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5711  {
    5712  ++m_MapCount;
    5713  }
    5714  else
    5715  {
    5716  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
    5717  }
    5718 }
    5719 
    5720 void VmaAllocation_T::BlockAllocUnmap()
    5721 {
    5722  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5723 
    5724  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5725  {
    5726  --m_MapCount;
    5727  }
    5728  else
    5729  {
    5730  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
    5731  }
    5732 }
    5733 
    5734 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
    5735 {
    5736  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5737 
    5738  if(m_MapCount != 0)
    5739  {
    5740  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5741  {
    5742  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
    5743  *ppData = m_DedicatedAllocation.m_pMappedData;
    5744  ++m_MapCount;
    5745  return VK_SUCCESS;
    5746  }
    5747  else
    5748  {
    5749  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
    5750  return VK_ERROR_MEMORY_MAP_FAILED;
    5751  }
    5752  }
    5753  else
    5754  {
    5755  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    5756  hAllocator->m_hDevice,
    5757  m_DedicatedAllocation.m_hMemory,
    5758  0, // offset
    5759  VK_WHOLE_SIZE,
    5760  0, // flags
    5761  ppData);
    5762  if(result == VK_SUCCESS)
    5763  {
    5764  m_DedicatedAllocation.m_pMappedData = *ppData;
    5765  m_MapCount = 1;
    5766  }
    5767  return result;
    5768  }
    5769 }
    5770 
    5771 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
    5772 {
    5773  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5774 
    5775  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5776  {
    5777  --m_MapCount;
    5778  if(m_MapCount == 0)
    5779  {
    5780  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
    5781  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
    5782  hAllocator->m_hDevice,
    5783  m_DedicatedAllocation.m_hMemory);
    5784  }
    5785  }
    5786  else
    5787  {
    5788  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
    5789  }
    5790 }
    5791 
    5792 #if VMA_STATS_STRING_ENABLED
    5793 
    5794 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
    5795 {
    5796  json.BeginObject();
    5797 
    5798  json.WriteString("Blocks");
    5799  json.WriteNumber(stat.blockCount);
    5800 
    5801  json.WriteString("Allocations");
    5802  json.WriteNumber(stat.allocationCount);
    5803 
    5804  json.WriteString("UnusedRanges");
    5805  json.WriteNumber(stat.unusedRangeCount);
    5806 
    5807  json.WriteString("UsedBytes");
    5808  json.WriteNumber(stat.usedBytes);
    5809 
    5810  json.WriteString("UnusedBytes");
    5811  json.WriteNumber(stat.unusedBytes);
    5812 
    5813  if(stat.allocationCount > 1)
    5814  {
    5815  json.WriteString("AllocationSize");
    5816  json.BeginObject(true);
    5817  json.WriteString("Min");
    5818  json.WriteNumber(stat.allocationSizeMin);
    5819  json.WriteString("Avg");
    5820  json.WriteNumber(stat.allocationSizeAvg);
    5821  json.WriteString("Max");
    5822  json.WriteNumber(stat.allocationSizeMax);
    5823  json.EndObject();
    5824  }
    5825 
    5826  if(stat.unusedRangeCount > 1)
    5827  {
    5828  json.WriteString("UnusedRangeSize");
    5829  json.BeginObject(true);
    5830  json.WriteString("Min");
    5831  json.WriteNumber(stat.unusedRangeSizeMin);
    5832  json.WriteString("Avg");
    5833  json.WriteNumber(stat.unusedRangeSizeAvg);
    5834  json.WriteString("Max");
    5835  json.WriteNumber(stat.unusedRangeSizeMax);
    5836  json.EndObject();
    5837  }
    5838 
    5839  json.EndObject();
    5840 }
    5841 
    5842 #endif // #if VMA_STATS_STRING_ENABLED
    5843 
    5844 struct VmaSuballocationItemSizeLess
    5845 {
    5846  bool operator()(
    5847  const VmaSuballocationList::iterator lhs,
    5848  const VmaSuballocationList::iterator rhs) const
    5849  {
    5850  return lhs->size < rhs->size;
    5851  }
    5852  bool operator()(
    5853  const VmaSuballocationList::iterator lhs,
    5854  VkDeviceSize rhsSize) const
    5855  {
    5856  return lhs->size < rhsSize;
    5857  }
    5858 };
    5859 
    5861 // class VmaBlockMetadata
    5862 
    5863 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
    5864  m_Size(0),
    5865  m_FreeCount(0),
    5866  m_SumFreeSize(0),
    5867  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
    5868  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
    5869 {
    5870 }
    5871 
    5872 VmaBlockMetadata::~VmaBlockMetadata()
    5873 {
    5874 }
    5875 
    5876 void VmaBlockMetadata::Init(VkDeviceSize size)
    5877 {
    5878  m_Size = size;
    5879  m_FreeCount = 1;
    5880  m_SumFreeSize = size;
    5881 
    5882  VmaSuballocation suballoc = {};
    5883  suballoc.offset = 0;
    5884  suballoc.size = size;
    5885  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    5886  suballoc.hAllocation = VK_NULL_HANDLE;
    5887 
    5888  m_Suballocations.push_back(suballoc);
    5889  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
    5890  --suballocItem;
    5891  m_FreeSuballocationsBySize.push_back(suballocItem);
    5892 }
    5893 
    5894 bool VmaBlockMetadata::Validate() const
    5895 {
    5896  if(m_Suballocations.empty())
    5897  {
    5898  return false;
    5899  }
    5900 
    5901  // Expected offset of new suballocation as calculates from previous ones.
    5902  VkDeviceSize calculatedOffset = 0;
    5903  // Expected number of free suballocations as calculated from traversing their list.
    5904  uint32_t calculatedFreeCount = 0;
    5905  // Expected sum size of free suballocations as calculated from traversing their list.
    5906  VkDeviceSize calculatedSumFreeSize = 0;
    5907  // Expected number of free suballocations that should be registered in
    5908  // m_FreeSuballocationsBySize calculated from traversing their list.
    5909  size_t freeSuballocationsToRegister = 0;
    5910  // True if previous visited suballocation was free.
    5911  bool prevFree = false;
    5912 
    5913  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5914  suballocItem != m_Suballocations.cend();
    5915  ++suballocItem)
    5916  {
    5917  const VmaSuballocation& subAlloc = *suballocItem;
    5918 
    5919  // Actual offset of this suballocation doesn't match expected one.
    5920  if(subAlloc.offset != calculatedOffset)
    5921  {
    5922  return false;
    5923  }
    5924 
    5925  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
    5926  // Two adjacent free suballocations are invalid. They should be merged.
    5927  if(prevFree && currFree)
    5928  {
    5929  return false;
    5930  }
    5931 
    5932  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
    5933  {
    5934  return false;
    5935  }
    5936 
    5937  if(currFree)
    5938  {
    5939  calculatedSumFreeSize += subAlloc.size;
    5940  ++calculatedFreeCount;
    5941  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    5942  {
    5943  ++freeSuballocationsToRegister;
    5944  }
    5945 
    5946  // Margin required between allocations - every free space must be at least that large.
    5947  if(subAlloc.size < VMA_DEBUG_MARGIN)
    5948  {
    5949  return false;
    5950  }
    5951  }
    5952  else
    5953  {
    5954  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
    5955  {
    5956  return false;
    5957  }
    5958  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
    5959  {
    5960  return false;
    5961  }
    5962 
    5963  // Margin required between allocations - previous allocation must be free.
    5964  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
    5965  {
    5966  return false;
    5967  }
    5968  }
    5969 
    5970  calculatedOffset += subAlloc.size;
    5971  prevFree = currFree;
    5972  }
    5973 
    5974  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
    5975  // match expected one.
    5976  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
    5977  {
    5978  return false;
    5979  }
    5980 
    5981  VkDeviceSize lastSize = 0;
    5982  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
    5983  {
    5984  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
    5985 
    5986  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
    5987  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
    5988  {
    5989  return false;
    5990  }
    5991  // They must be sorted by size ascending.
    5992  if(suballocItem->size < lastSize)
    5993  {
    5994  return false;
    5995  }
    5996 
    5997  lastSize = suballocItem->size;
    5998  }
    5999 
    6000  // Check if totals match calculacted values.
    6001  if(!ValidateFreeSuballocationList() ||
    6002  (calculatedOffset != m_Size) ||
    6003  (calculatedSumFreeSize != m_SumFreeSize) ||
    6004  (calculatedFreeCount != m_FreeCount))
    6005  {
    6006  return false;
    6007  }
    6008 
    6009  return true;
    6010 }
    6011 
    6012 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
    6013 {
    6014  if(!m_FreeSuballocationsBySize.empty())
    6015  {
    6016  return m_FreeSuballocationsBySize.back()->size;
    6017  }
    6018  else
    6019  {
    6020  return 0;
    6021  }
    6022 }
    6023 
    6024 bool VmaBlockMetadata::IsEmpty() const
    6025 {
    6026  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
    6027 }
    6028 
    6029 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
    6030 {
    6031  outInfo.blockCount = 1;
    6032 
    6033  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    6034  outInfo.allocationCount = rangeCount - m_FreeCount;
    6035  outInfo.unusedRangeCount = m_FreeCount;
    6036 
    6037  outInfo.unusedBytes = m_SumFreeSize;
    6038  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
    6039 
    6040  outInfo.allocationSizeMin = UINT64_MAX;
    6041  outInfo.allocationSizeMax = 0;
    6042  outInfo.unusedRangeSizeMin = UINT64_MAX;
    6043  outInfo.unusedRangeSizeMax = 0;
    6044 
    6045  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    6046  suballocItem != m_Suballocations.cend();
    6047  ++suballocItem)
    6048  {
    6049  const VmaSuballocation& suballoc = *suballocItem;
    6050  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
    6051  {
    6052  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
    6053  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
    6054  }
    6055  else
    6056  {
    6057  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
    6058  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
    6059  }
    6060  }
    6061 }
    6062 
    6063 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
    6064 {
    6065  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    6066 
    6067  inoutStats.size += m_Size;
    6068  inoutStats.unusedSize += m_SumFreeSize;
    6069  inoutStats.allocationCount += rangeCount - m_FreeCount;
    6070  inoutStats.unusedRangeCount += m_FreeCount;
    6071  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
    6072 }
    6073 
    6074 #if VMA_STATS_STRING_ENABLED
    6075 
    6076 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
    6077 {
    6078  json.BeginObject();
    6079 
    6080  json.WriteString("TotalBytes");
    6081  json.WriteNumber(m_Size);
    6082 
    6083  json.WriteString("UnusedBytes");
    6084  json.WriteNumber(m_SumFreeSize);
    6085 
    6086  json.WriteString("Allocations");
    6087  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
    6088 
    6089  json.WriteString("UnusedRanges");
    6090  json.WriteNumber(m_FreeCount);
    6091 
    6092  json.WriteString("Suballocations");
    6093  json.BeginArray();
    6094  size_t i = 0;
    6095  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    6096  suballocItem != m_Suballocations.cend();
    6097  ++suballocItem, ++i)
    6098  {
    6099  json.BeginObject(true);
    6100 
    6101  json.WriteString("Offset");
    6102  json.WriteNumber(suballocItem->offset);
    6103 
    6104  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6105  {
    6106  json.WriteString("Type");
    6107  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
    6108 
    6109  json.WriteString("Size");
    6110  json.WriteNumber(suballocItem->size);
    6111  }
    6112  else
    6113  {
    6114  suballocItem->hAllocation->PrintParameters(json);
    6115  }
    6116 
    6117  json.EndObject();
    6118  }
    6119  json.EndArray();
    6120 
    6121  json.EndObject();
    6122 }
    6123 
    6124 #endif // #if VMA_STATS_STRING_ENABLED
    6125 
    6126 /*
    6127 How many suitable free suballocations to analyze before choosing best one.
    6128 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
    6129  be chosen.
    6130 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
    6131  suballocations will be analized and best one will be chosen.
    6132 - Any other value is also acceptable.
    6133 */
    6134 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
    6135 
    6136 bool VmaBlockMetadata::CreateAllocationRequest(
    6137  uint32_t currentFrameIndex,
    6138  uint32_t frameInUseCount,
    6139  VkDeviceSize bufferImageGranularity,
    6140  VkDeviceSize allocSize,
    6141  VkDeviceSize allocAlignment,
    6142  VmaSuballocationType allocType,
    6143  bool canMakeOtherLost,
    6144  VmaAllocationRequest* pAllocationRequest)
    6145 {
    6146  VMA_ASSERT(allocSize > 0);
    6147  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6148  VMA_ASSERT(pAllocationRequest != VMA_NULL);
    6149  VMA_HEAVY_ASSERT(Validate());
    6150 
    6151  // There is not enough total free space in this block to fullfill the request: Early return.
    6152  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
    6153  {
    6154  return false;
    6155  }
    6156 
    6157  // New algorithm, efficiently searching freeSuballocationsBySize.
    6158  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
    6159  if(freeSuballocCount > 0)
    6160  {
    6161  if(VMA_BEST_FIT)
    6162  {
    6163  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
    6164  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6165  m_FreeSuballocationsBySize.data(),
    6166  m_FreeSuballocationsBySize.data() + freeSuballocCount,
    6167  allocSize + 2 * VMA_DEBUG_MARGIN,
    6168  VmaSuballocationItemSizeLess());
    6169  size_t index = it - m_FreeSuballocationsBySize.data();
    6170  for(; index < freeSuballocCount; ++index)
    6171  {
    6172  if(CheckAllocation(
    6173  currentFrameIndex,
    6174  frameInUseCount,
    6175  bufferImageGranularity,
    6176  allocSize,
    6177  allocAlignment,
    6178  allocType,
    6179  m_FreeSuballocationsBySize[index],
    6180  false, // canMakeOtherLost
    6181  &pAllocationRequest->offset,
    6182  &pAllocationRequest->itemsToMakeLostCount,
    6183  &pAllocationRequest->sumFreeSize,
    6184  &pAllocationRequest->sumItemSize))
    6185  {
    6186  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6187  return true;
    6188  }
    6189  }
    6190  }
    6191  else
    6192  {
    6193  // Search staring from biggest suballocations.
    6194  for(size_t index = freeSuballocCount; index--; )
    6195  {
    6196  if(CheckAllocation(
    6197  currentFrameIndex,
    6198  frameInUseCount,
    6199  bufferImageGranularity,
    6200  allocSize,
    6201  allocAlignment,
    6202  allocType,
    6203  m_FreeSuballocationsBySize[index],
    6204  false, // canMakeOtherLost
    6205  &pAllocationRequest->offset,
    6206  &pAllocationRequest->itemsToMakeLostCount,
    6207  &pAllocationRequest->sumFreeSize,
    6208  &pAllocationRequest->sumItemSize))
    6209  {
    6210  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6211  return true;
    6212  }
    6213  }
    6214  }
    6215  }
    6216 
    6217  if(canMakeOtherLost)
    6218  {
    6219  // Brute-force algorithm. TODO: Come up with something better.
    6220 
    6221  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
    6222  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
    6223 
    6224  VmaAllocationRequest tmpAllocRequest = {};
    6225  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
    6226  suballocIt != m_Suballocations.end();
    6227  ++suballocIt)
    6228  {
    6229  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
    6230  suballocIt->hAllocation->CanBecomeLost())
    6231  {
    6232  if(CheckAllocation(
    6233  currentFrameIndex,
    6234  frameInUseCount,
    6235  bufferImageGranularity,
    6236  allocSize,
    6237  allocAlignment,
    6238  allocType,
    6239  suballocIt,
    6240  canMakeOtherLost,
    6241  &tmpAllocRequest.offset,
    6242  &tmpAllocRequest.itemsToMakeLostCount,
    6243  &tmpAllocRequest.sumFreeSize,
    6244  &tmpAllocRequest.sumItemSize))
    6245  {
    6246  tmpAllocRequest.item = suballocIt;
    6247 
    6248  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
    6249  {
    6250  *pAllocationRequest = tmpAllocRequest;
    6251  }
    6252  }
    6253  }
    6254  }
    6255 
    6256  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
    6257  {
    6258  return true;
    6259  }
    6260  }
    6261 
    6262  return false;
    6263 }
    6264 
    6265 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
    6266  uint32_t currentFrameIndex,
    6267  uint32_t frameInUseCount,
    6268  VmaAllocationRequest* pAllocationRequest)
    6269 {
    6270  while(pAllocationRequest->itemsToMakeLostCount > 0)
    6271  {
    6272  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
    6273  {
    6274  ++pAllocationRequest->item;
    6275  }
    6276  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6277  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
    6278  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
    6279  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6280  {
    6281  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
    6282  --pAllocationRequest->itemsToMakeLostCount;
    6283  }
    6284  else
    6285  {
    6286  return false;
    6287  }
    6288  }
    6289 
    6290  VMA_HEAVY_ASSERT(Validate());
    6291  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6292  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6293 
    6294  return true;
    6295 }
    6296 
    6297 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    6298 {
    6299  uint32_t lostAllocationCount = 0;
    6300  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6301  it != m_Suballocations.end();
    6302  ++it)
    6303  {
    6304  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
    6305  it->hAllocation->CanBecomeLost() &&
    6306  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6307  {
    6308  it = FreeSuballocation(it);
    6309  ++lostAllocationCount;
    6310  }
    6311  }
    6312  return lostAllocationCount;
    6313 }
    6314 
    6315 VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData)
    6316 {
    6317  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6318  it != m_Suballocations.end();
    6319  ++it)
    6320  {
    6321  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6322  {
    6323  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
    6324  {
    6325  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
    6326  return VK_ERROR_VALIDATION_FAILED_EXT;
    6327  }
    6328  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
    6329  {
    6330  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
    6331  return VK_ERROR_VALIDATION_FAILED_EXT;
    6332  }
    6333  }
    6334  }
    6335 
    6336  return VK_SUCCESS;
    6337 }
    6338 
    6339 void VmaBlockMetadata::Alloc(
    6340  const VmaAllocationRequest& request,
    6341  VmaSuballocationType type,
    6342  VkDeviceSize allocSize,
    6343  VmaAllocation hAllocation)
    6344 {
    6345  VMA_ASSERT(request.item != m_Suballocations.end());
    6346  VmaSuballocation& suballoc = *request.item;
    6347  // Given suballocation is a free block.
    6348  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6349  // Given offset is inside this suballocation.
    6350  VMA_ASSERT(request.offset >= suballoc.offset);
    6351  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
    6352  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
    6353  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
    6354 
    6355  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
    6356  // it to become used.
    6357  UnregisterFreeSuballocation(request.item);
    6358 
    6359  suballoc.offset = request.offset;
    6360  suballoc.size = allocSize;
    6361  suballoc.type = type;
    6362  suballoc.hAllocation = hAllocation;
    6363 
    6364  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
    6365  if(paddingEnd)
    6366  {
    6367  VmaSuballocation paddingSuballoc = {};
    6368  paddingSuballoc.offset = request.offset + allocSize;
    6369  paddingSuballoc.size = paddingEnd;
    6370  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6371  VmaSuballocationList::iterator next = request.item;
    6372  ++next;
    6373  const VmaSuballocationList::iterator paddingEndItem =
    6374  m_Suballocations.insert(next, paddingSuballoc);
    6375  RegisterFreeSuballocation(paddingEndItem);
    6376  }
    6377 
    6378  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
    6379  if(paddingBegin)
    6380  {
    6381  VmaSuballocation paddingSuballoc = {};
    6382  paddingSuballoc.offset = request.offset - paddingBegin;
    6383  paddingSuballoc.size = paddingBegin;
    6384  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6385  const VmaSuballocationList::iterator paddingBeginItem =
    6386  m_Suballocations.insert(request.item, paddingSuballoc);
    6387  RegisterFreeSuballocation(paddingBeginItem);
    6388  }
    6389 
    6390  // Update totals.
    6391  m_FreeCount = m_FreeCount - 1;
    6392  if(paddingBegin > 0)
    6393  {
    6394  ++m_FreeCount;
    6395  }
    6396  if(paddingEnd > 0)
    6397  {
    6398  ++m_FreeCount;
    6399  }
    6400  m_SumFreeSize -= allocSize;
    6401 }
    6402 
    6403 void VmaBlockMetadata::Free(const VmaAllocation allocation)
    6404 {
    6405  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6406  suballocItem != m_Suballocations.end();
    6407  ++suballocItem)
    6408  {
    6409  VmaSuballocation& suballoc = *suballocItem;
    6410  if(suballoc.hAllocation == allocation)
    6411  {
    6412  FreeSuballocation(suballocItem);
    6413  VMA_HEAVY_ASSERT(Validate());
    6414  return;
    6415  }
    6416  }
    6417  VMA_ASSERT(0 && "Not found!");
    6418 }
    6419 
    6420 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
    6421 {
    6422  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6423  suballocItem != m_Suballocations.end();
    6424  ++suballocItem)
    6425  {
    6426  VmaSuballocation& suballoc = *suballocItem;
    6427  if(suballoc.offset == offset)
    6428  {
    6429  FreeSuballocation(suballocItem);
    6430  return;
    6431  }
    6432  }
    6433  VMA_ASSERT(0 && "Not found!");
    6434 }
    6435 
    6436 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
    6437 {
    6438  VkDeviceSize lastSize = 0;
    6439  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
    6440  {
    6441  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
    6442 
    6443  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6444  {
    6445  VMA_ASSERT(0);
    6446  return false;
    6447  }
    6448  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6449  {
    6450  VMA_ASSERT(0);
    6451  return false;
    6452  }
    6453  if(it->size < lastSize)
    6454  {
    6455  VMA_ASSERT(0);
    6456  return false;
    6457  }
    6458 
    6459  lastSize = it->size;
    6460  }
    6461  return true;
    6462 }
    6463 
    6464 bool VmaBlockMetadata::CheckAllocation(
    6465  uint32_t currentFrameIndex,
    6466  uint32_t frameInUseCount,
    6467  VkDeviceSize bufferImageGranularity,
    6468  VkDeviceSize allocSize,
    6469  VkDeviceSize allocAlignment,
    6470  VmaSuballocationType allocType,
    6471  VmaSuballocationList::const_iterator suballocItem,
    6472  bool canMakeOtherLost,
    6473  VkDeviceSize* pOffset,
    6474  size_t* itemsToMakeLostCount,
    6475  VkDeviceSize* pSumFreeSize,
    6476  VkDeviceSize* pSumItemSize) const
    6477 {
    6478  VMA_ASSERT(allocSize > 0);
    6479  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6480  VMA_ASSERT(suballocItem != m_Suballocations.cend());
    6481  VMA_ASSERT(pOffset != VMA_NULL);
    6482 
    6483  *itemsToMakeLostCount = 0;
    6484  *pSumFreeSize = 0;
    6485  *pSumItemSize = 0;
    6486 
    6487  if(canMakeOtherLost)
    6488  {
    6489  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6490  {
    6491  *pSumFreeSize = suballocItem->size;
    6492  }
    6493  else
    6494  {
    6495  if(suballocItem->hAllocation->CanBecomeLost() &&
    6496  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6497  {
    6498  ++*itemsToMakeLostCount;
    6499  *pSumItemSize = suballocItem->size;
    6500  }
    6501  else
    6502  {
    6503  return false;
    6504  }
    6505  }
    6506 
    6507  // Remaining size is too small for this request: Early return.
    6508  if(m_Size - suballocItem->offset < allocSize)
    6509  {
    6510  return false;
    6511  }
    6512 
    6513  // Start from offset equal to beginning of this suballocation.
    6514  *pOffset = suballocItem->offset;
    6515 
    6516  // Apply VMA_DEBUG_MARGIN at the beginning.
    6517  if(VMA_DEBUG_MARGIN > 0)
    6518  {
    6519  *pOffset += VMA_DEBUG_MARGIN;
    6520  }
    6521 
    6522  // Apply alignment.
    6523  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6524 
    6525  // Check previous suballocations for BufferImageGranularity conflicts.
    6526  // Make bigger alignment if necessary.
    6527  if(bufferImageGranularity > 1)
    6528  {
    6529  bool bufferImageGranularityConflict = false;
    6530  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6531  while(prevSuballocItem != m_Suballocations.cbegin())
    6532  {
    6533  --prevSuballocItem;
    6534  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6535  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6536  {
    6537  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6538  {
    6539  bufferImageGranularityConflict = true;
    6540  break;
    6541  }
    6542  }
    6543  else
    6544  // Already on previous page.
    6545  break;
    6546  }
    6547  if(bufferImageGranularityConflict)
    6548  {
    6549  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6550  }
    6551  }
    6552 
    6553  // Now that we have final *pOffset, check if we are past suballocItem.
    6554  // If yes, return false - this function should be called for another suballocItem as starting point.
    6555  if(*pOffset >= suballocItem->offset + suballocItem->size)
    6556  {
    6557  return false;
    6558  }
    6559 
    6560  // Calculate padding at the beginning based on current offset.
    6561  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
    6562 
    6563  // Calculate required margin at the end.
    6564  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6565 
    6566  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
    6567  // Another early return check.
    6568  if(suballocItem->offset + totalSize > m_Size)
    6569  {
    6570  return false;
    6571  }
    6572 
    6573  // Advance lastSuballocItem until desired size is reached.
    6574  // Update itemsToMakeLostCount.
    6575  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
    6576  if(totalSize > suballocItem->size)
    6577  {
    6578  VkDeviceSize remainingSize = totalSize - suballocItem->size;
    6579  while(remainingSize > 0)
    6580  {
    6581  ++lastSuballocItem;
    6582  if(lastSuballocItem == m_Suballocations.cend())
    6583  {
    6584  return false;
    6585  }
    6586  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6587  {
    6588  *pSumFreeSize += lastSuballocItem->size;
    6589  }
    6590  else
    6591  {
    6592  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
    6593  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
    6594  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6595  {
    6596  ++*itemsToMakeLostCount;
    6597  *pSumItemSize += lastSuballocItem->size;
    6598  }
    6599  else
    6600  {
    6601  return false;
    6602  }
    6603  }
    6604  remainingSize = (lastSuballocItem->size < remainingSize) ?
    6605  remainingSize - lastSuballocItem->size : 0;
    6606  }
    6607  }
    6608 
    6609  // Check next suballocations for BufferImageGranularity conflicts.
    6610  // If conflict exists, we must mark more allocations lost or fail.
    6611  if(bufferImageGranularity > 1)
    6612  {
    6613  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
    6614  ++nextSuballocItem;
    6615  while(nextSuballocItem != m_Suballocations.cend())
    6616  {
    6617  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6618  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6619  {
    6620  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6621  {
    6622  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
    6623  if(nextSuballoc.hAllocation->CanBecomeLost() &&
    6624  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6625  {
    6626  ++*itemsToMakeLostCount;
    6627  }
    6628  else
    6629  {
    6630  return false;
    6631  }
    6632  }
    6633  }
    6634  else
    6635  {
    6636  // Already on next page.
    6637  break;
    6638  }
    6639  ++nextSuballocItem;
    6640  }
    6641  }
    6642  }
    6643  else
    6644  {
    6645  const VmaSuballocation& suballoc = *suballocItem;
    6646  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6647 
    6648  *pSumFreeSize = suballoc.size;
    6649 
    6650  // Size of this suballocation is too small for this request: Early return.
    6651  if(suballoc.size < allocSize)
    6652  {
    6653  return false;
    6654  }
    6655 
    6656  // Start from offset equal to beginning of this suballocation.
    6657  *pOffset = suballoc.offset;
    6658 
    6659  // Apply VMA_DEBUG_MARGIN at the beginning.
    6660  if(VMA_DEBUG_MARGIN > 0)
    6661  {
    6662  *pOffset += VMA_DEBUG_MARGIN;
    6663  }
    6664 
    6665  // Apply alignment.
    6666  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6667 
    6668  // Check previous suballocations for BufferImageGranularity conflicts.
    6669  // Make bigger alignment if necessary.
    6670  if(bufferImageGranularity > 1)
    6671  {
    6672  bool bufferImageGranularityConflict = false;
    6673  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6674  while(prevSuballocItem != m_Suballocations.cbegin())
    6675  {
    6676  --prevSuballocItem;
    6677  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6678  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6679  {
    6680  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6681  {
    6682  bufferImageGranularityConflict = true;
    6683  break;
    6684  }
    6685  }
    6686  else
    6687  // Already on previous page.
    6688  break;
    6689  }
    6690  if(bufferImageGranularityConflict)
    6691  {
    6692  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6693  }
    6694  }
    6695 
    6696  // Calculate padding at the beginning based on current offset.
    6697  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
    6698 
    6699  // Calculate required margin at the end.
    6700  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6701 
    6702  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
    6703  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
    6704  {
    6705  return false;
    6706  }
    6707 
    6708  // Check next suballocations for BufferImageGranularity conflicts.
    6709  // If conflict exists, allocation cannot be made here.
    6710  if(bufferImageGranularity > 1)
    6711  {
    6712  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
    6713  ++nextSuballocItem;
    6714  while(nextSuballocItem != m_Suballocations.cend())
    6715  {
    6716  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6717  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6718  {
    6719  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6720  {
    6721  return false;
    6722  }
    6723  }
    6724  else
    6725  {
    6726  // Already on next page.
    6727  break;
    6728  }
    6729  ++nextSuballocItem;
    6730  }
    6731  }
    6732  }
    6733 
    6734  // All tests passed: Success. pOffset is already filled.
    6735  return true;
    6736 }
    6737 
    6738 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
    6739 {
    6740  VMA_ASSERT(item != m_Suballocations.end());
    6741  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6742 
    6743  VmaSuballocationList::iterator nextItem = item;
    6744  ++nextItem;
    6745  VMA_ASSERT(nextItem != m_Suballocations.end());
    6746  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
    6747 
    6748  item->size += nextItem->size;
    6749  --m_FreeCount;
    6750  m_Suballocations.erase(nextItem);
    6751 }
    6752 
    6753 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
    6754 {
    6755  // Change this suballocation to be marked as free.
    6756  VmaSuballocation& suballoc = *suballocItem;
    6757  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6758  suballoc.hAllocation = VK_NULL_HANDLE;
    6759 
    6760  // Update totals.
    6761  ++m_FreeCount;
    6762  m_SumFreeSize += suballoc.size;
    6763 
    6764  // Merge with previous and/or next suballocation if it's also free.
    6765  bool mergeWithNext = false;
    6766  bool mergeWithPrev = false;
    6767 
    6768  VmaSuballocationList::iterator nextItem = suballocItem;
    6769  ++nextItem;
    6770  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
    6771  {
    6772  mergeWithNext = true;
    6773  }
    6774 
    6775  VmaSuballocationList::iterator prevItem = suballocItem;
    6776  if(suballocItem != m_Suballocations.begin())
    6777  {
    6778  --prevItem;
    6779  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6780  {
    6781  mergeWithPrev = true;
    6782  }
    6783  }
    6784 
    6785  if(mergeWithNext)
    6786  {
    6787  UnregisterFreeSuballocation(nextItem);
    6788  MergeFreeWithNext(suballocItem);
    6789  }
    6790 
    6791  if(mergeWithPrev)
    6792  {
    6793  UnregisterFreeSuballocation(prevItem);
    6794  MergeFreeWithNext(prevItem);
    6795  RegisterFreeSuballocation(prevItem);
    6796  return prevItem;
    6797  }
    6798  else
    6799  {
    6800  RegisterFreeSuballocation(suballocItem);
    6801  return suballocItem;
    6802  }
    6803 }
    6804 
    6805 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
    6806 {
    6807  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6808  VMA_ASSERT(item->size > 0);
    6809 
    6810  // You may want to enable this validation at the beginning or at the end of
    6811  // this function, depending on what do you want to check.
    6812  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6813 
    6814  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6815  {
    6816  if(m_FreeSuballocationsBySize.empty())
    6817  {
    6818  m_FreeSuballocationsBySize.push_back(item);
    6819  }
    6820  else
    6821  {
    6822  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
    6823  }
    6824  }
    6825 
    6826  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6827 }
    6828 
    6829 
    6830 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
    6831 {
    6832  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6833  VMA_ASSERT(item->size > 0);
    6834 
    6835  // You may want to enable this validation at the beginning or at the end of
    6836  // this function, depending on what do you want to check.
    6837  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6838 
    6839  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6840  {
    6841  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6842  m_FreeSuballocationsBySize.data(),
    6843  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
    6844  item,
    6845  VmaSuballocationItemSizeLess());
    6846  for(size_t index = it - m_FreeSuballocationsBySize.data();
    6847  index < m_FreeSuballocationsBySize.size();
    6848  ++index)
    6849  {
    6850  if(m_FreeSuballocationsBySize[index] == item)
    6851  {
    6852  VmaVectorRemove(m_FreeSuballocationsBySize, index);
    6853  return;
    6854  }
    6855  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
    6856  }
    6857  VMA_ASSERT(0 && "Not found.");
    6858  }
    6859 
    6860  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6861 }
    6862 
    6864 // class VmaDeviceMemoryBlock
    6865 
    6866 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
    6867  m_Metadata(hAllocator),
    6868  m_MemoryTypeIndex(UINT32_MAX),
    6869  m_Id(0),
    6870  m_hMemory(VK_NULL_HANDLE),
    6871  m_MapCount(0),
    6872  m_pMappedData(VMA_NULL)
    6873 {
    6874 }
    6875 
    6876 void VmaDeviceMemoryBlock::Init(
    6877  uint32_t newMemoryTypeIndex,
    6878  VkDeviceMemory newMemory,
    6879  VkDeviceSize newSize,
    6880  uint32_t id)
    6881 {
    6882  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    6883 
    6884  m_MemoryTypeIndex = newMemoryTypeIndex;
    6885  m_Id = id;
    6886  m_hMemory = newMemory;
    6887 
    6888  m_Metadata.Init(newSize);
    6889 }
    6890 
    6891 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
    6892 {
    6893  // This is the most important assert in the entire library.
    6894  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
    6895  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
    6896 
    6897  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
    6898  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
    6899  m_hMemory = VK_NULL_HANDLE;
    6900 }
    6901 
    6902 bool VmaDeviceMemoryBlock::Validate() const
    6903 {
    6904  if((m_hMemory == VK_NULL_HANDLE) ||
    6905  (m_Metadata.GetSize() == 0))
    6906  {
    6907  return false;
    6908  }
    6909 
    6910  return m_Metadata.Validate();
    6911 }
    6912 
    6913 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
    6914 {
    6915  void* pData = nullptr;
    6916  VkResult res = Map(hAllocator, 1, &pData);
    6917  if(res != VK_SUCCESS)
    6918  {
    6919  return res;
    6920  }
    6921 
    6922  res = m_Metadata.CheckCorruption(pData);
    6923 
    6924  Unmap(hAllocator, 1);
    6925 
    6926  return res;
    6927 }
    6928 
    6929 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
    6930 {
    6931  if(count == 0)
    6932  {
    6933  return VK_SUCCESS;
    6934  }
    6935 
    6936  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6937  if(m_MapCount != 0)
    6938  {
    6939  m_MapCount += count;
    6940  VMA_ASSERT(m_pMappedData != VMA_NULL);
    6941  if(ppData != VMA_NULL)
    6942  {
    6943  *ppData = m_pMappedData;
    6944  }
    6945  return VK_SUCCESS;
    6946  }
    6947  else
    6948  {
    6949  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    6950  hAllocator->m_hDevice,
    6951  m_hMemory,
    6952  0, // offset
    6953  VK_WHOLE_SIZE,
    6954  0, // flags
    6955  &m_pMappedData);
    6956  if(result == VK_SUCCESS)
    6957  {
    6958  if(ppData != VMA_NULL)
    6959  {
    6960  *ppData = m_pMappedData;
    6961  }
    6962  m_MapCount = count;
    6963  }
    6964  return result;
    6965  }
    6966 }
    6967 
    6968 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
    6969 {
    6970  if(count == 0)
    6971  {
    6972  return;
    6973  }
    6974 
    6975  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6976  if(m_MapCount >= count)
    6977  {
    6978  m_MapCount -= count;
    6979  if(m_MapCount == 0)
    6980  {
    6981  m_pMappedData = VMA_NULL;
    6982  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
    6983  }
    6984  }
    6985  else
    6986  {
    6987  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
    6988  }
    6989 }
    6990 
    6991 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6992 {
    6993  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    6994  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    6995 
    6996  void* pData;
    6997  VkResult res = Map(hAllocator, 1, &pData);
    6998  if(res != VK_SUCCESS)
    6999  {
    7000  return res;
    7001  }
    7002 
    7003  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
    7004  VmaWriteMagicValue(pData, allocOffset + allocSize);
    7005 
    7006  Unmap(hAllocator, 1);
    7007 
    7008  return VK_SUCCESS;
    7009 }
    7010 
    7011 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    7012 {
    7013  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    7014  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    7015 
    7016  void* pData;
    7017  VkResult res = Map(hAllocator, 1, &pData);
    7018  if(res != VK_SUCCESS)
    7019  {
    7020  return res;
    7021  }
    7022 
    7023  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
    7024  {
    7025  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
    7026  }
    7027  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
    7028  {
    7029  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
    7030  }
    7031 
    7032  Unmap(hAllocator, 1);
    7033 
    7034  return VK_SUCCESS;
    7035 }
    7036 
    7037 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
    7038  const VmaAllocator hAllocator,
    7039  const VmaAllocation hAllocation,
    7040  VkBuffer hBuffer)
    7041 {
    7042  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    7043  hAllocation->GetBlock() == this);
    7044  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    7045  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    7046  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
    7047  hAllocator->m_hDevice,
    7048  hBuffer,
    7049  m_hMemory,
    7050  hAllocation->GetOffset());
    7051 }
    7052 
    7053 VkResult VmaDeviceMemoryBlock::BindImageMemory(
    7054  const VmaAllocator hAllocator,
    7055  const VmaAllocation hAllocation,
    7056  VkImage hImage)
    7057 {
    7058  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    7059  hAllocation->GetBlock() == this);
    7060  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    7061  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    7062  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
    7063  hAllocator->m_hDevice,
    7064  hImage,
    7065  m_hMemory,
    7066  hAllocation->GetOffset());
    7067 }
    7068 
    7069 static void InitStatInfo(VmaStatInfo& outInfo)
    7070 {
    7071  memset(&outInfo, 0, sizeof(outInfo));
    7072  outInfo.allocationSizeMin = UINT64_MAX;
    7073  outInfo.unusedRangeSizeMin = UINT64_MAX;
    7074 }
    7075 
    7076 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
    7077 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
    7078 {
    7079  inoutInfo.blockCount += srcInfo.blockCount;
    7080  inoutInfo.allocationCount += srcInfo.allocationCount;
    7081  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
    7082  inoutInfo.usedBytes += srcInfo.usedBytes;
    7083  inoutInfo.unusedBytes += srcInfo.unusedBytes;
    7084  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
    7085  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
    7086  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
    7087  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
    7088 }
    7089 
    7090 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
    7091 {
    7092  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
    7093  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
    7094  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
    7095  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
    7096 }
    7097 
    7098 VmaPool_T::VmaPool_T(
    7099  VmaAllocator hAllocator,
    7100  const VmaPoolCreateInfo& createInfo) :
    7101  m_BlockVector(
    7102  hAllocator,
    7103  createInfo.memoryTypeIndex,
    7104  createInfo.blockSize,
    7105  createInfo.minBlockCount,
    7106  createInfo.maxBlockCount,
    7107  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
    7108  createInfo.frameInUseCount,
    7109  true), // isCustomPool
    7110  m_Id(0)
    7111 {
    7112 }
    7113 
    7114 VmaPool_T::~VmaPool_T()
    7115 {
    7116 }
    7117 
    7118 #if VMA_STATS_STRING_ENABLED
    7119 
    7120 #endif // #if VMA_STATS_STRING_ENABLED
    7121 
    7122 VmaBlockVector::VmaBlockVector(
    7123  VmaAllocator hAllocator,
    7124  uint32_t memoryTypeIndex,
    7125  VkDeviceSize preferredBlockSize,
    7126  size_t minBlockCount,
    7127  size_t maxBlockCount,
    7128  VkDeviceSize bufferImageGranularity,
    7129  uint32_t frameInUseCount,
    7130  bool isCustomPool) :
    7131  m_hAllocator(hAllocator),
    7132  m_MemoryTypeIndex(memoryTypeIndex),
    7133  m_PreferredBlockSize(preferredBlockSize),
    7134  m_MinBlockCount(minBlockCount),
    7135  m_MaxBlockCount(maxBlockCount),
    7136  m_BufferImageGranularity(bufferImageGranularity),
    7137  m_FrameInUseCount(frameInUseCount),
    7138  m_IsCustomPool(isCustomPool),
    7139  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
    7140  m_HasEmptyBlock(false),
    7141  m_pDefragmentator(VMA_NULL),
    7142  m_NextBlockId(0)
    7143 {
    7144 }
    7145 
    7146 VmaBlockVector::~VmaBlockVector()
    7147 {
    7148  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
    7149 
    7150  for(size_t i = m_Blocks.size(); i--; )
    7151  {
    7152  m_Blocks[i]->Destroy(m_hAllocator);
    7153  vma_delete(m_hAllocator, m_Blocks[i]);
    7154  }
    7155 }
    7156 
    7157 VkResult VmaBlockVector::CreateMinBlocks()
    7158 {
    7159  for(size_t i = 0; i < m_MinBlockCount; ++i)
    7160  {
    7161  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
    7162  if(res != VK_SUCCESS)
    7163  {
    7164  return res;
    7165  }
    7166  }
    7167  return VK_SUCCESS;
    7168 }
    7169 
    7170 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
    7171 {
    7172  pStats->size = 0;
    7173  pStats->unusedSize = 0;
    7174  pStats->allocationCount = 0;
    7175  pStats->unusedRangeCount = 0;
    7176  pStats->unusedRangeSizeMax = 0;
    7177 
    7178  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7179 
    7180  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7181  {
    7182  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7183  VMA_ASSERT(pBlock);
    7184  VMA_HEAVY_ASSERT(pBlock->Validate());
    7185  pBlock->m_Metadata.AddPoolStats(*pStats);
    7186  }
    7187 }
    7188 
    7189 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
    7190 {
    7191  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    7192  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
    7193  (VMA_DEBUG_MARGIN > 0) &&
    7194  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
    7195 }
    7196 
    7197 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
    7198 
    7199 VkResult VmaBlockVector::Allocate(
    7200  VmaPool hCurrentPool,
    7201  uint32_t currentFrameIndex,
    7202  VkDeviceSize size,
    7203  VkDeviceSize alignment,
    7204  const VmaAllocationCreateInfo& createInfo,
    7205  VmaSuballocationType suballocType,
    7206  VmaAllocation* pAllocation)
    7207 {
    7208  // Early reject: requested allocation size is larger that maximum block size for this block vector.
    7209  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
    7210  {
    7211  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7212  }
    7213 
    7214  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    7215  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
    7216 
    7217  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7218 
    7219  // 1. Search existing allocations. Try to allocate without making other allocations lost.
    7220  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7221  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7222  {
    7223  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7224  VMA_ASSERT(pCurrBlock);
    7225  VmaAllocationRequest currRequest = {};
    7226  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7227  currentFrameIndex,
    7228  m_FrameInUseCount,
    7229  m_BufferImageGranularity,
    7230  size,
    7231  alignment,
    7232  suballocType,
    7233  false, // canMakeOtherLost
    7234  &currRequest))
    7235  {
    7236  // Allocate from pCurrBlock.
    7237  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
    7238 
    7239  if(mapped)
    7240  {
    7241  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
    7242  if(res != VK_SUCCESS)
    7243  {
    7244  return res;
    7245  }
    7246  }
    7247 
    7248  // We no longer have an empty Allocation.
    7249  if(pCurrBlock->m_Metadata.IsEmpty())
    7250  {
    7251  m_HasEmptyBlock = false;
    7252  }
    7253 
    7254  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7255  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation);
    7256  (*pAllocation)->InitBlockAllocation(
    7257  hCurrentPool,
    7258  pCurrBlock,
    7259  currRequest.offset,
    7260  alignment,
    7261  size,
    7262  suballocType,
    7263  mapped,
    7264  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7265  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
    7266  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7267  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7268  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7269  {
    7270  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7271  }
    7272  if(IsCorruptionDetectionEnabled())
    7273  {
    7274  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
    7275  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7276  }
    7277  return VK_SUCCESS;
    7278  }
    7279  }
    7280 
    7281  const bool canCreateNewBlock =
    7282  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
    7283  (m_Blocks.size() < m_MaxBlockCount);
    7284 
    7285  // 2. Try to create new block.
    7286  if(canCreateNewBlock)
    7287  {
    7288  // Calculate optimal size for new block.
    7289  VkDeviceSize newBlockSize = m_PreferredBlockSize;
    7290  uint32_t newBlockSizeShift = 0;
    7291  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
    7292 
    7293  // Allocating blocks of other sizes is allowed only in default pools.
    7294  // In custom pools block size is fixed.
    7295  if(m_IsCustomPool == false)
    7296  {
    7297  // Allocate 1/8, 1/4, 1/2 as first blocks.
    7298  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
    7299  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
    7300  {
    7301  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7302  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
    7303  {
    7304  newBlockSize = smallerNewBlockSize;
    7305  ++newBlockSizeShift;
    7306  }
    7307  else
    7308  {
    7309  break;
    7310  }
    7311  }
    7312  }
    7313 
    7314  size_t newBlockIndex = 0;
    7315  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
    7316  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
    7317  if(m_IsCustomPool == false)
    7318  {
    7319  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
    7320  {
    7321  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7322  if(smallerNewBlockSize >= size)
    7323  {
    7324  newBlockSize = smallerNewBlockSize;
    7325  ++newBlockSizeShift;
    7326  res = CreateBlock(newBlockSize, &newBlockIndex);
    7327  }
    7328  else
    7329  {
    7330  break;
    7331  }
    7332  }
    7333  }
    7334 
    7335  if(res == VK_SUCCESS)
    7336  {
    7337  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
    7338  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size);
    7339 
    7340  if(mapped)
    7341  {
    7342  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
    7343  if(res != VK_SUCCESS)
    7344  {
    7345  return res;
    7346  }
    7347  }
    7348 
    7349  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
    7350  VmaAllocationRequest allocRequest;
    7351  if(pBlock->m_Metadata.CreateAllocationRequest(
    7352  currentFrameIndex,
    7353  m_FrameInUseCount,
    7354  m_BufferImageGranularity,
    7355  size,
    7356  alignment,
    7357  suballocType,
    7358  false, // canMakeOtherLost
    7359  &allocRequest))
    7360  {
    7361  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7362  pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation);
    7363  (*pAllocation)->InitBlockAllocation(
    7364  hCurrentPool,
    7365  pBlock,
    7366  allocRequest.offset,
    7367  alignment,
    7368  size,
    7369  suballocType,
    7370  mapped,
    7371  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7372  VMA_HEAVY_ASSERT(pBlock->Validate());
    7373  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
    7374  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7375  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7376  {
    7377  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7378  }
    7379  if(IsCorruptionDetectionEnabled())
    7380  {
    7381  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
    7382  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7383  }
    7384  return VK_SUCCESS;
    7385  }
    7386  else
    7387  {
    7388  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
    7389  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7390  }
    7391  }
    7392  }
    7393 
    7394  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
    7395 
    7396  // 3. Try to allocate from existing blocks with making other allocations lost.
    7397  if(canMakeOtherLost)
    7398  {
    7399  uint32_t tryIndex = 0;
    7400  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
    7401  {
    7402  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
    7403  VmaAllocationRequest bestRequest = {};
    7404  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
    7405 
    7406  // 1. Search existing allocations.
    7407  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7408  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7409  {
    7410  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7411  VMA_ASSERT(pCurrBlock);
    7412  VmaAllocationRequest currRequest = {};
    7413  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7414  currentFrameIndex,
    7415  m_FrameInUseCount,
    7416  m_BufferImageGranularity,
    7417  size,
    7418  alignment,
    7419  suballocType,
    7420  canMakeOtherLost,
    7421  &currRequest))
    7422  {
    7423  const VkDeviceSize currRequestCost = currRequest.CalcCost();
    7424  if(pBestRequestBlock == VMA_NULL ||
    7425  currRequestCost < bestRequestCost)
    7426  {
    7427  pBestRequestBlock = pCurrBlock;
    7428  bestRequest = currRequest;
    7429  bestRequestCost = currRequestCost;
    7430 
    7431  if(bestRequestCost == 0)
    7432  {
    7433  break;
    7434  }
    7435  }
    7436  }
    7437  }
    7438 
    7439  if(pBestRequestBlock != VMA_NULL)
    7440  {
    7441  if(mapped)
    7442  {
    7443  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
    7444  if(res != VK_SUCCESS)
    7445  {
    7446  return res;
    7447  }
    7448  }
    7449 
    7450  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
    7451  currentFrameIndex,
    7452  m_FrameInUseCount,
    7453  &bestRequest))
    7454  {
    7455  // We no longer have an empty Allocation.
    7456  if(pBestRequestBlock->m_Metadata.IsEmpty())
    7457  {
    7458  m_HasEmptyBlock = false;
    7459  }
    7460  // Allocate from this pBlock.
    7461  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7462  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation);
    7463  (*pAllocation)->InitBlockAllocation(
    7464  hCurrentPool,
    7465  pBestRequestBlock,
    7466  bestRequest.offset,
    7467  alignment,
    7468  size,
    7469  suballocType,
    7470  mapped,
    7471  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7472  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
    7473  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7474  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7475  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7476  {
    7477  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7478  }
    7479  if(IsCorruptionDetectionEnabled())
    7480  {
    7481  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
    7482  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7483  }
    7484  return VK_SUCCESS;
    7485  }
    7486  // else: Some allocations must have been touched while we are here. Next try.
    7487  }
    7488  else
    7489  {
    7490  // Could not find place in any of the blocks - break outer loop.
    7491  break;
    7492  }
    7493  }
    7494  /* Maximum number of tries exceeded - a very unlike event when many other
    7495  threads are simultaneously touching allocations making it impossible to make
    7496  lost at the same time as we try to allocate. */
    7497  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
    7498  {
    7499  return VK_ERROR_TOO_MANY_OBJECTS;
    7500  }
    7501  }
    7502 
    7503  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7504 }
    7505 
    7506 void VmaBlockVector::Free(
    7507  VmaAllocation hAllocation)
    7508 {
    7509  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
    7510 
    7511  // Scope for lock.
    7512  {
    7513  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7514 
    7515  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    7516 
    7517  if(IsCorruptionDetectionEnabled())
    7518  {
    7519  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
    7520  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
    7521  }
    7522 
    7523  if(hAllocation->IsPersistentMap())
    7524  {
    7525  pBlock->Unmap(m_hAllocator, 1);
    7526  }
    7527 
    7528  pBlock->m_Metadata.Free(hAllocation);
    7529  VMA_HEAVY_ASSERT(pBlock->Validate());
    7530 
    7531  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
    7532 
    7533  // pBlock became empty after this deallocation.
    7534  if(pBlock->m_Metadata.IsEmpty())
    7535  {
    7536  // Already has empty Allocation. We don't want to have two, so delete this one.
    7537  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
    7538  {
    7539  pBlockToDelete = pBlock;
    7540  Remove(pBlock);
    7541  }
    7542  // We now have first empty block.
    7543  else
    7544  {
    7545  m_HasEmptyBlock = true;
    7546  }
    7547  }
    7548  // pBlock didn't become empty, but we have another empty block - find and free that one.
    7549  // (This is optional, heuristics.)
    7550  else if(m_HasEmptyBlock)
    7551  {
    7552  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
    7553  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
    7554  {
    7555  pBlockToDelete = pLastBlock;
    7556  m_Blocks.pop_back();
    7557  m_HasEmptyBlock = false;
    7558  }
    7559  }
    7560 
    7561  IncrementallySortBlocks();
    7562  }
    7563 
    7564  // Destruction of a free Allocation. Deferred until this point, outside of mutex
    7565  // lock, for performance reason.
    7566  if(pBlockToDelete != VMA_NULL)
    7567  {
    7568  VMA_DEBUG_LOG(" Deleted empty allocation");
    7569  pBlockToDelete->Destroy(m_hAllocator);
    7570  vma_delete(m_hAllocator, pBlockToDelete);
    7571  }
    7572 }
    7573 
    7574 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
    7575 {
    7576  VkDeviceSize result = 0;
    7577  for(size_t i = m_Blocks.size(); i--; )
    7578  {
    7579  result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize());
    7580  if(result >= m_PreferredBlockSize)
    7581  {
    7582  break;
    7583  }
    7584  }
    7585  return result;
    7586 }
    7587 
    7588 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
    7589 {
    7590  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7591  {
    7592  if(m_Blocks[blockIndex] == pBlock)
    7593  {
    7594  VmaVectorRemove(m_Blocks, blockIndex);
    7595  return;
    7596  }
    7597  }
    7598  VMA_ASSERT(0);
    7599 }
    7600 
    7601 void VmaBlockVector::IncrementallySortBlocks()
    7602 {
    7603  // Bubble sort only until first swap.
    7604  for(size_t i = 1; i < m_Blocks.size(); ++i)
    7605  {
    7606  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
    7607  {
    7608  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
    7609  return;
    7610  }
    7611  }
    7612 }
    7613 
    7614 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
    7615 {
    7616  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    7617  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
    7618  allocInfo.allocationSize = blockSize;
    7619  VkDeviceMemory mem = VK_NULL_HANDLE;
    7620  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
    7621  if(res < 0)
    7622  {
    7623  return res;
    7624  }
    7625 
    7626  // New VkDeviceMemory successfully created.
    7627 
    7628  // Create new Allocation for it.
    7629  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
    7630  pBlock->Init(
    7631  m_MemoryTypeIndex,
    7632  mem,
    7633  allocInfo.allocationSize,
    7634  m_NextBlockId++);
    7635 
    7636  m_Blocks.push_back(pBlock);
    7637  if(pNewBlockIndex != VMA_NULL)
    7638  {
    7639  *pNewBlockIndex = m_Blocks.size() - 1;
    7640  }
    7641 
    7642  return VK_SUCCESS;
    7643 }
    7644 
    7645 #if VMA_STATS_STRING_ENABLED
    7646 
    7647 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
    7648 {
    7649  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7650 
    7651  json.BeginObject();
    7652 
    7653  if(m_IsCustomPool)
    7654  {
    7655  json.WriteString("MemoryTypeIndex");
    7656  json.WriteNumber(m_MemoryTypeIndex);
    7657 
    7658  json.WriteString("BlockSize");
    7659  json.WriteNumber(m_PreferredBlockSize);
    7660 
    7661  json.WriteString("BlockCount");
    7662  json.BeginObject(true);
    7663  if(m_MinBlockCount > 0)
    7664  {
    7665  json.WriteString("Min");
    7666  json.WriteNumber((uint64_t)m_MinBlockCount);
    7667  }
    7668  if(m_MaxBlockCount < SIZE_MAX)
    7669  {
    7670  json.WriteString("Max");
    7671  json.WriteNumber((uint64_t)m_MaxBlockCount);
    7672  }
    7673  json.WriteString("Cur");
    7674  json.WriteNumber((uint64_t)m_Blocks.size());
    7675  json.EndObject();
    7676 
    7677  if(m_FrameInUseCount > 0)
    7678  {
    7679  json.WriteString("FrameInUseCount");
    7680  json.WriteNumber(m_FrameInUseCount);
    7681  }
    7682  }
    7683  else
    7684  {
    7685  json.WriteString("PreferredBlockSize");
    7686  json.WriteNumber(m_PreferredBlockSize);
    7687  }
    7688 
    7689  json.WriteString("Blocks");
    7690  json.BeginObject();
    7691  for(size_t i = 0; i < m_Blocks.size(); ++i)
    7692  {
    7693  json.BeginString();
    7694  json.ContinueString(m_Blocks[i]->GetId());
    7695  json.EndString();
    7696 
    7697  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
    7698  }
    7699  json.EndObject();
    7700 
    7701  json.EndObject();
    7702 }
    7703 
    7704 #endif // #if VMA_STATS_STRING_ENABLED
    7705 
    7706 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
    7707  VmaAllocator hAllocator,
    7708  uint32_t currentFrameIndex)
    7709 {
    7710  if(m_pDefragmentator == VMA_NULL)
    7711  {
    7712  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
    7713  hAllocator,
    7714  this,
    7715  currentFrameIndex);
    7716  }
    7717 
    7718  return m_pDefragmentator;
    7719 }
    7720 
    7721 VkResult VmaBlockVector::Defragment(
    7722  VmaDefragmentationStats* pDefragmentationStats,
    7723  VkDeviceSize& maxBytesToMove,
    7724  uint32_t& maxAllocationsToMove)
    7725 {
    7726  if(m_pDefragmentator == VMA_NULL)
    7727  {
    7728  return VK_SUCCESS;
    7729  }
    7730 
    7731  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7732 
    7733  // Defragment.
    7734  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
    7735 
    7736  // Accumulate statistics.
    7737  if(pDefragmentationStats != VMA_NULL)
    7738  {
    7739  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
    7740  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
    7741  pDefragmentationStats->bytesMoved += bytesMoved;
    7742  pDefragmentationStats->allocationsMoved += allocationsMoved;
    7743  VMA_ASSERT(bytesMoved <= maxBytesToMove);
    7744  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
    7745  maxBytesToMove -= bytesMoved;
    7746  maxAllocationsToMove -= allocationsMoved;
    7747  }
    7748 
    7749  // Free empty blocks.
    7750  m_HasEmptyBlock = false;
    7751  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
    7752  {
    7753  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
    7754  if(pBlock->m_Metadata.IsEmpty())
    7755  {
    7756  if(m_Blocks.size() > m_MinBlockCount)
    7757  {
    7758  if(pDefragmentationStats != VMA_NULL)
    7759  {
    7760  ++pDefragmentationStats->deviceMemoryBlocksFreed;
    7761  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
    7762  }
    7763 
    7764  VmaVectorRemove(m_Blocks, blockIndex);
    7765  pBlock->Destroy(m_hAllocator);
    7766  vma_delete(m_hAllocator, pBlock);
    7767  }
    7768  else
    7769  {
    7770  m_HasEmptyBlock = true;
    7771  }
    7772  }
    7773  }
    7774 
    7775  return result;
    7776 }
    7777 
    7778 void VmaBlockVector::DestroyDefragmentator()
    7779 {
    7780  if(m_pDefragmentator != VMA_NULL)
    7781  {
    7782  vma_delete(m_hAllocator, m_pDefragmentator);
    7783  m_pDefragmentator = VMA_NULL;
    7784  }
    7785 }
    7786 
    7787 void VmaBlockVector::MakePoolAllocationsLost(
    7788  uint32_t currentFrameIndex,
    7789  size_t* pLostAllocationCount)
    7790 {
    7791  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7792  size_t lostAllocationCount = 0;
    7793  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7794  {
    7795  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7796  VMA_ASSERT(pBlock);
    7797  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
    7798  }
    7799  if(pLostAllocationCount != VMA_NULL)
    7800  {
    7801  *pLostAllocationCount = lostAllocationCount;
    7802  }
    7803 }
    7804 
    7805 VkResult VmaBlockVector::CheckCorruption()
    7806 {
    7807  if(!IsCorruptionDetectionEnabled())
    7808  {
    7809  return VK_ERROR_FEATURE_NOT_PRESENT;
    7810  }
    7811 
    7812  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7813  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7814  {
    7815  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7816  VMA_ASSERT(pBlock);
    7817  VkResult res = pBlock->CheckCorruption(m_hAllocator);
    7818  if(res != VK_SUCCESS)
    7819  {
    7820  return res;
    7821  }
    7822  }
    7823  return VK_SUCCESS;
    7824 }
    7825 
    7826 void VmaBlockVector::AddStats(VmaStats* pStats)
    7827 {
    7828  const uint32_t memTypeIndex = m_MemoryTypeIndex;
    7829  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
    7830 
    7831  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7832 
    7833  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7834  {
    7835  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7836  VMA_ASSERT(pBlock);
    7837  VMA_HEAVY_ASSERT(pBlock->Validate());
    7838  VmaStatInfo allocationStatInfo;
    7839  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
    7840  VmaAddStatInfo(pStats->total, allocationStatInfo);
    7841  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    7842  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    7843  }
    7844 }
    7845 
    7847 // VmaDefragmentator members definition
    7848 
    7849 VmaDefragmentator::VmaDefragmentator(
    7850  VmaAllocator hAllocator,
    7851  VmaBlockVector* pBlockVector,
    7852  uint32_t currentFrameIndex) :
    7853  m_hAllocator(hAllocator),
    7854  m_pBlockVector(pBlockVector),
    7855  m_CurrentFrameIndex(currentFrameIndex),
    7856  m_BytesMoved(0),
    7857  m_AllocationsMoved(0),
    7858  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
    7859  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
    7860 {
    7861 }
    7862 
    7863 VmaDefragmentator::~VmaDefragmentator()
    7864 {
    7865  for(size_t i = m_Blocks.size(); i--; )
    7866  {
    7867  vma_delete(m_hAllocator, m_Blocks[i]);
    7868  }
    7869 }
    7870 
    7871 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
    7872 {
    7873  AllocationInfo allocInfo;
    7874  allocInfo.m_hAllocation = hAlloc;
    7875  allocInfo.m_pChanged = pChanged;
    7876  m_Allocations.push_back(allocInfo);
    7877 }
    7878 
    7879 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
    7880 {
    7881  // It has already been mapped for defragmentation.
    7882  if(m_pMappedDataForDefragmentation)
    7883  {
    7884  *ppMappedData = m_pMappedDataForDefragmentation;
    7885  return VK_SUCCESS;
    7886  }
    7887 
    7888  // It is originally mapped.
    7889  if(m_pBlock->GetMappedData())
    7890  {
    7891  *ppMappedData = m_pBlock->GetMappedData();
    7892  return VK_SUCCESS;
    7893  }
    7894 
    7895  // Map on first usage.
    7896  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
    7897  *ppMappedData = m_pMappedDataForDefragmentation;
    7898  return res;
    7899 }
    7900 
    7901 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
    7902 {
    7903  if(m_pMappedDataForDefragmentation != VMA_NULL)
    7904  {
    7905  m_pBlock->Unmap(hAllocator, 1);
    7906  }
    7907 }
    7908 
    7909 VkResult VmaDefragmentator::DefragmentRound(
    7910  VkDeviceSize maxBytesToMove,
    7911  uint32_t maxAllocationsToMove)
    7912 {
    7913  if(m_Blocks.empty())
    7914  {
    7915  return VK_SUCCESS;
    7916  }
    7917 
    7918  size_t srcBlockIndex = m_Blocks.size() - 1;
    7919  size_t srcAllocIndex = SIZE_MAX;
    7920  for(;;)
    7921  {
    7922  // 1. Find next allocation to move.
    7923  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
    7924  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
    7925  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
    7926  {
    7927  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
    7928  {
    7929  // Finished: no more allocations to process.
    7930  if(srcBlockIndex == 0)
    7931  {
    7932  return VK_SUCCESS;
    7933  }
    7934  else
    7935  {
    7936  --srcBlockIndex;
    7937  srcAllocIndex = SIZE_MAX;
    7938  }
    7939  }
    7940  else
    7941  {
    7942  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
    7943  }
    7944  }
    7945 
    7946  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
    7947  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
    7948 
    7949  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
    7950  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
    7951  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
    7952  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
    7953 
    7954  // 2. Try to find new place for this allocation in preceding or current block.
    7955  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
    7956  {
    7957  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
    7958  VmaAllocationRequest dstAllocRequest;
    7959  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
    7960  m_CurrentFrameIndex,
    7961  m_pBlockVector->GetFrameInUseCount(),
    7962  m_pBlockVector->GetBufferImageGranularity(),
    7963  size,
    7964  alignment,
    7965  suballocType,
    7966  false, // canMakeOtherLost
    7967  &dstAllocRequest) &&
    7968  MoveMakesSense(
    7969  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
    7970  {
    7971  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
    7972 
    7973  // Reached limit on number of allocations or bytes to move.
    7974  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
    7975  (m_BytesMoved + size > maxBytesToMove))
    7976  {
    7977  return VK_INCOMPLETE;
    7978  }
    7979 
    7980  void* pDstMappedData = VMA_NULL;
    7981  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
    7982  if(res != VK_SUCCESS)
    7983  {
    7984  return res;
    7985  }
    7986 
    7987  void* pSrcMappedData = VMA_NULL;
    7988  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
    7989  if(res != VK_SUCCESS)
    7990  {
    7991  return res;
    7992  }
    7993 
    7994  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
    7995  memcpy(
    7996  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
    7997  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
    7998  static_cast<size_t>(size));
    7999 
    8000  if(VMA_DEBUG_MARGIN > 0)
    8001  {
    8002  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
    8003  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
    8004  }
    8005 
    8006  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
    8007  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
    8008 
    8009  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
    8010 
    8011  if(allocInfo.m_pChanged != VMA_NULL)
    8012  {
    8013  *allocInfo.m_pChanged = VK_TRUE;
    8014  }
    8015 
    8016  ++m_AllocationsMoved;
    8017  m_BytesMoved += size;
    8018 
    8019  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
    8020 
    8021  break;
    8022  }
    8023  }
    8024 
    8025  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
    8026 
    8027  if(srcAllocIndex > 0)
    8028  {
    8029  --srcAllocIndex;
    8030  }
    8031  else
    8032  {
    8033  if(srcBlockIndex > 0)
    8034  {
    8035  --srcBlockIndex;
    8036  srcAllocIndex = SIZE_MAX;
    8037  }
    8038  else
    8039  {
    8040  return VK_SUCCESS;
    8041  }
    8042  }
    8043  }
    8044 }
    8045 
    8046 VkResult VmaDefragmentator::Defragment(
    8047  VkDeviceSize maxBytesToMove,
    8048  uint32_t maxAllocationsToMove)
    8049 {
    8050  if(m_Allocations.empty())
    8051  {
    8052  return VK_SUCCESS;
    8053  }
    8054 
    8055  // Create block info for each block.
    8056  const size_t blockCount = m_pBlockVector->m_Blocks.size();
    8057  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8058  {
    8059  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
    8060  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
    8061  m_Blocks.push_back(pBlockInfo);
    8062  }
    8063 
    8064  // Sort them by m_pBlock pointer value.
    8065  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
    8066 
    8067  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
    8068  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
    8069  {
    8070  AllocationInfo& allocInfo = m_Allocations[blockIndex];
    8071  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
    8072  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    8073  {
    8074  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
    8075  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
    8076  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
    8077  {
    8078  (*it)->m_Allocations.push_back(allocInfo);
    8079  }
    8080  else
    8081  {
    8082  VMA_ASSERT(0);
    8083  }
    8084  }
    8085  }
    8086  m_Allocations.clear();
    8087 
    8088  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8089  {
    8090  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
    8091  pBlockInfo->CalcHasNonMovableAllocations();
    8092  pBlockInfo->SortAllocationsBySizeDescecnding();
    8093  }
    8094 
    8095  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
    8096  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
    8097 
    8098  // Execute defragmentation rounds (the main part).
    8099  VkResult result = VK_SUCCESS;
    8100  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
    8101  {
    8102  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
    8103  }
    8104 
    8105  // Unmap blocks that were mapped for defragmentation.
    8106  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8107  {
    8108  m_Blocks[blockIndex]->Unmap(m_hAllocator);
    8109  }
    8110 
    8111  return result;
    8112 }
    8113 
    8114 bool VmaDefragmentator::MoveMakesSense(
    8115  size_t dstBlockIndex, VkDeviceSize dstOffset,
    8116  size_t srcBlockIndex, VkDeviceSize srcOffset)
    8117 {
    8118  if(dstBlockIndex < srcBlockIndex)
    8119  {
    8120  return true;
    8121  }
    8122  if(dstBlockIndex > srcBlockIndex)
    8123  {
    8124  return false;
    8125  }
    8126  if(dstOffset < srcOffset)
    8127  {
    8128  return true;
    8129  }
    8130  return false;
    8131 }
    8132 
    8134 // VmaRecorder
    8135 
    8136 #if VMA_RECORDING_ENABLED
    8137 
    8138 VmaRecorder::VmaRecorder() :
    8139  m_UseMutex(true),
    8140  m_Flags(0),
    8141  m_File(VMA_NULL),
    8142  m_Freq(INT64_MAX),
    8143  m_StartCounter(INT64_MAX)
    8144 {
    8145 }
    8146 
    8147 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
    8148 {
    8149  m_UseMutex = useMutex;
    8150  m_Flags = settings.flags;
    8151 
    8152  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
    8153  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
    8154 
    8155  // Open file for writing.
    8156  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
    8157  if(err != 0)
    8158  {
    8159  return VK_ERROR_INITIALIZATION_FAILED;
    8160  }
    8161 
    8162  // Write header.
    8163  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
    8164  fprintf(m_File, "%s\n", "1,2");
    8165 
    8166  return VK_SUCCESS;
    8167 }
    8168 
    8169 VmaRecorder::~VmaRecorder()
    8170 {
    8171  if(m_File != VMA_NULL)
    8172  {
    8173  fclose(m_File);
    8174  }
    8175 }
    8176 
    8177 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
    8178 {
    8179  CallParams callParams;
    8180  GetBasicParams(callParams);
    8181 
    8182  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8183  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8184  Flush();
    8185 }
    8186 
    8187 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
    8188 {
    8189  CallParams callParams;
    8190  GetBasicParams(callParams);
    8191 
    8192  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8193  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8194  Flush();
    8195 }
    8196 
    8197 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
    8198 {
    8199  CallParams callParams;
    8200  GetBasicParams(callParams);
    8201 
    8202  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8203  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
    8204  createInfo.memoryTypeIndex,
    8205  createInfo.flags,
    8206  createInfo.blockSize,
    8207  createInfo.minBlockCount,
    8208  createInfo.maxBlockCount,
    8209  createInfo.frameInUseCount,
    8210  pool);
    8211  Flush();
    8212 }
    8213 
    8214 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
    8215 {
    8216  CallParams callParams;
    8217  GetBasicParams(callParams);
    8218 
    8219  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8220  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
    8221  pool);
    8222  Flush();
    8223 }
    8224 
    8225 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
    8226  const VkMemoryRequirements& vkMemReq,
    8227  const VmaAllocationCreateInfo& createInfo,
    8228  VmaAllocation allocation)
    8229 {
    8230  CallParams callParams;
    8231  GetBasicParams(callParams);
    8232 
    8233  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8234  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8235  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8236  vkMemReq.size,
    8237  vkMemReq.alignment,
    8238  vkMemReq.memoryTypeBits,
    8239  createInfo.flags,
    8240  createInfo.usage,
    8241  createInfo.requiredFlags,
    8242  createInfo.preferredFlags,
    8243  createInfo.memoryTypeBits,
    8244  createInfo.pool,
    8245  allocation,
    8246  userDataStr.GetString());
    8247  Flush();
    8248 }
    8249 
    8250 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    8251  const VkMemoryRequirements& vkMemReq,
    8252  bool requiresDedicatedAllocation,
    8253  bool prefersDedicatedAllocation,
    8254  const VmaAllocationCreateInfo& createInfo,
    8255  VmaAllocation allocation)
    8256 {
    8257  CallParams callParams;
    8258  GetBasicParams(callParams);
    8259 
    8260  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8261  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8262  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8263  vkMemReq.size,
    8264  vkMemReq.alignment,
    8265  vkMemReq.memoryTypeBits,
    8266  requiresDedicatedAllocation ? 1 : 0,
    8267  prefersDedicatedAllocation ? 1 : 0,
    8268  createInfo.flags,
    8269  createInfo.usage,
    8270  createInfo.requiredFlags,
    8271  createInfo.preferredFlags,
    8272  createInfo.memoryTypeBits,
    8273  createInfo.pool,
    8274  allocation,
    8275  userDataStr.GetString());
    8276  Flush();
    8277 }
    8278 
    8279 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
    8280  const VkMemoryRequirements& vkMemReq,
    8281  bool requiresDedicatedAllocation,
    8282  bool prefersDedicatedAllocation,
    8283  const VmaAllocationCreateInfo& createInfo,
    8284  VmaAllocation allocation)
    8285 {
    8286  CallParams callParams;
    8287  GetBasicParams(callParams);
    8288 
    8289  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8290  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8291  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8292  vkMemReq.size,
    8293  vkMemReq.alignment,
    8294  vkMemReq.memoryTypeBits,
    8295  requiresDedicatedAllocation ? 1 : 0,
    8296  prefersDedicatedAllocation ? 1 : 0,
    8297  createInfo.flags,
    8298  createInfo.usage,
    8299  createInfo.requiredFlags,
    8300  createInfo.preferredFlags,
    8301  createInfo.memoryTypeBits,
    8302  createInfo.pool,
    8303  allocation,
    8304  userDataStr.GetString());
    8305  Flush();
    8306 }
    8307 
    8308 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
    8309  VmaAllocation allocation)
    8310 {
    8311  CallParams callParams;
    8312  GetBasicParams(callParams);
    8313 
    8314  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8315  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8316  allocation);
    8317  Flush();
    8318 }
    8319 
    8320 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
    8321  VmaAllocation allocation,
    8322  const void* pUserData)
    8323 {
    8324  CallParams callParams;
    8325  GetBasicParams(callParams);
    8326 
    8327  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8328  UserDataString userDataStr(
    8329  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
    8330  pUserData);
    8331  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8332  allocation,
    8333  userDataStr.GetString());
    8334  Flush();
    8335 }
    8336 
    8337 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
    8338  VmaAllocation allocation)
    8339 {
    8340  CallParams callParams;
    8341  GetBasicParams(callParams);
    8342 
    8343  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8344  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
    8345  allocation);
    8346  Flush();
    8347 }
    8348 
    8349 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
    8350  VmaAllocation allocation)
    8351 {
    8352  CallParams callParams;
    8353  GetBasicParams(callParams);
    8354 
    8355  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8356  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8357  allocation);
    8358  Flush();
    8359 }
    8360 
    8361 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
    8362  VmaAllocation allocation)
    8363 {
    8364  CallParams callParams;
    8365  GetBasicParams(callParams);
    8366 
    8367  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8368  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8369  allocation);
    8370  Flush();
    8371 }
    8372 
    8373 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
    8374  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8375 {
    8376  CallParams callParams;
    8377  GetBasicParams(callParams);
    8378 
    8379  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8380  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8381  allocation,
    8382  offset,
    8383  size);
    8384  Flush();
    8385 }
    8386 
    8387 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
    8388  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8389 {
    8390  CallParams callParams;
    8391  GetBasicParams(callParams);
    8392 
    8393  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8394  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8395  allocation,
    8396  offset,
    8397  size);
    8398  Flush();
    8399 }
    8400 
    8401 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
    8402  const VkBufferCreateInfo& bufCreateInfo,
    8403  const VmaAllocationCreateInfo& allocCreateInfo,
    8404  VmaAllocation allocation)
    8405 {
    8406  CallParams callParams;
    8407  GetBasicParams(callParams);
    8408 
    8409  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8410  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8411  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8412  bufCreateInfo.flags,
    8413  bufCreateInfo.size,
    8414  bufCreateInfo.usage,
    8415  bufCreateInfo.sharingMode,
    8416  allocCreateInfo.flags,
    8417  allocCreateInfo.usage,
    8418  allocCreateInfo.requiredFlags,
    8419  allocCreateInfo.preferredFlags,
    8420  allocCreateInfo.memoryTypeBits,
    8421  allocCreateInfo.pool,
    8422  allocation,
    8423  userDataStr.GetString());
    8424  Flush();
    8425 }
    8426 
    8427 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
    8428  const VkImageCreateInfo& imageCreateInfo,
    8429  const VmaAllocationCreateInfo& allocCreateInfo,
    8430  VmaAllocation allocation)
    8431 {
    8432  CallParams callParams;
    8433  GetBasicParams(callParams);
    8434 
    8435  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8436  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8437  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8438  imageCreateInfo.flags,
    8439  imageCreateInfo.imageType,
    8440  imageCreateInfo.format,
    8441  imageCreateInfo.extent.width,
    8442  imageCreateInfo.extent.height,
    8443  imageCreateInfo.extent.depth,
    8444  imageCreateInfo.mipLevels,
    8445  imageCreateInfo.arrayLayers,
    8446  imageCreateInfo.samples,
    8447  imageCreateInfo.tiling,
    8448  imageCreateInfo.usage,
    8449  imageCreateInfo.sharingMode,
    8450  imageCreateInfo.initialLayout,
    8451  allocCreateInfo.flags,
    8452  allocCreateInfo.usage,
    8453  allocCreateInfo.requiredFlags,
    8454  allocCreateInfo.preferredFlags,
    8455  allocCreateInfo.memoryTypeBits,
    8456  allocCreateInfo.pool,
    8457  allocation,
    8458  userDataStr.GetString());
    8459  Flush();
    8460 }
    8461 
    8462 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
    8463  VmaAllocation allocation)
    8464 {
    8465  CallParams callParams;
    8466  GetBasicParams(callParams);
    8467 
    8468  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8469  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
    8470  allocation);
    8471  Flush();
    8472 }
    8473 
    8474 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
    8475  VmaAllocation allocation)
    8476 {
    8477  CallParams callParams;
    8478  GetBasicParams(callParams);
    8479 
    8480  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8481  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
    8482  allocation);
    8483  Flush();
    8484 }
    8485 
    8486 void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex,
    8487  VmaAllocation allocation)
    8488 {
    8489  CallParams callParams;
    8490  GetBasicParams(callParams);
    8491 
    8492  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8493  fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
    8494  allocation);
    8495  Flush();
    8496 }
    8497 
    8498 void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex,
    8499  VmaAllocation allocation)
    8500 {
    8501  CallParams callParams;
    8502  GetBasicParams(callParams);
    8503 
    8504  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8505  fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex,
    8506  allocation);
    8507  Flush();
    8508 }
    8509 
    8510 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
    8511 {
    8512  if(pUserData != VMA_NULL)
    8513  {
    8514  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
    8515  {
    8516  m_Str = (const char*)pUserData;
    8517  }
    8518  else
    8519  {
    8520  sprintf_s(m_PtrStr, "%p", pUserData);
    8521  m_Str = m_PtrStr;
    8522  }
    8523  }
    8524  else
    8525  {
    8526  m_Str = "";
    8527  }
    8528 }
    8529 
    8530 void VmaRecorder::GetBasicParams(CallParams& outParams)
    8531 {
    8532  outParams.threadId = GetCurrentThreadId();
    8533 
    8534  LARGE_INTEGER counter;
    8535  QueryPerformanceCounter(&counter);
    8536  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
    8537 }
    8538 
    8539 void VmaRecorder::Flush()
    8540 {
    8541  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
    8542  {
    8543  fflush(m_File);
    8544  }
    8545 }
    8546 
    8547 #endif // #if VMA_RECORDING_ENABLED
    8548 
    8550 // VmaAllocator_T
    8551 
    8552 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
    8553  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
    8554  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
    8555  m_hDevice(pCreateInfo->device),
    8556  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
    8557  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
    8558  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
    8559  m_PreferredLargeHeapBlockSize(0),
    8560  m_PhysicalDevice(pCreateInfo->physicalDevice),
    8561  m_CurrentFrameIndex(0),
    8562  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
    8563  m_NextPoolId(0)
    8564 #if VMA_RECORDING_ENABLED
    8565  ,m_pRecorder(VMA_NULL)
    8566 #endif
    8567 {
    8568  if(VMA_DEBUG_DETECT_CORRUPTION)
    8569  {
    8570  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
    8571  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
    8572  }
    8573 
    8574  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
    8575 
    8576 #if !(VMA_DEDICATED_ALLOCATION)
    8577  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
    8578  {
    8579  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
    8580  }
    8581 #endif
    8582 
    8583  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
    8584  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
    8585  memset(&m_MemProps, 0, sizeof(m_MemProps));
    8586 
    8587  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
    8588  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
    8589 
    8590  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    8591  {
    8592  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
    8593  }
    8594 
    8595  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
    8596  {
    8597  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
    8598  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
    8599  }
    8600 
    8601  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
    8602 
    8603  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
    8604  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
    8605 
    8606  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
    8607  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
    8608 
    8609  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
    8610  {
    8611  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
    8612  {
    8613  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
    8614  if(limit != VK_WHOLE_SIZE)
    8615  {
    8616  m_HeapSizeLimit[heapIndex] = limit;
    8617  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
    8618  {
    8619  m_MemProps.memoryHeaps[heapIndex].size = limit;
    8620  }
    8621  }
    8622  }
    8623  }
    8624 
    8625  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    8626  {
    8627  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
    8628 
    8629  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
    8630  this,
    8631  memTypeIndex,
    8632  preferredBlockSize,
    8633  0,
    8634  SIZE_MAX,
    8635  GetBufferImageGranularity(),
    8636  pCreateInfo->frameInUseCount,
    8637  false); // isCustomPool
    8638  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
    8639  // becase minBlockCount is 0.
    8640  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
    8641 
    8642  }
    8643 }
    8644 
    8645 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
    8646 {
    8647  VkResult res = VK_SUCCESS;
    8648 
    8649  if(pCreateInfo->pRecordSettings != VMA_NULL &&
    8650  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
    8651  {
    8652 #if VMA_RECORDING_ENABLED
    8653  m_pRecorder = vma_new(this, VmaRecorder)();
    8654  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
    8655  if(res != VK_SUCCESS)
    8656  {
    8657  return res;
    8658  }
    8659  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
    8660 #else
    8661  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
    8662  return VK_ERROR_FEATURE_NOT_PRESENT;
    8663 #endif
    8664  }
    8665 
    8666  return res;
    8667 }
    8668 
    8669 VmaAllocator_T::~VmaAllocator_T()
    8670 {
    8671 #if VMA_RECORDING_ENABLED
    8672  if(m_pRecorder != VMA_NULL)
    8673  {
    8674  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
    8675  vma_delete(this, m_pRecorder);
    8676  }
    8677 #endif
    8678 
    8679  VMA_ASSERT(m_Pools.empty());
    8680 
    8681  for(size_t i = GetMemoryTypeCount(); i--; )
    8682  {
    8683  vma_delete(this, m_pDedicatedAllocations[i]);
    8684  vma_delete(this, m_pBlockVectors[i]);
    8685  }
    8686 }
    8687 
    8688 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
    8689 {
    8690 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8691  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
    8692  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
    8693  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    8694  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
    8695  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
    8696  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
    8697  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
    8698  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
    8699  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
    8700  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
    8701  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
    8702  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
    8703  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
    8704  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
    8705  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
    8706  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
    8707 #if VMA_DEDICATED_ALLOCATION
    8708  if(m_UseKhrDedicatedAllocation)
    8709  {
    8710  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
    8711  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
    8712  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
    8713  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
    8714  }
    8715 #endif // #if VMA_DEDICATED_ALLOCATION
    8716 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8717 
    8718 #define VMA_COPY_IF_NOT_NULL(funcName) \
    8719  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
    8720 
    8721  if(pVulkanFunctions != VMA_NULL)
    8722  {
    8723  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
    8724  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
    8725  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
    8726  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
    8727  VMA_COPY_IF_NOT_NULL(vkMapMemory);
    8728  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
    8729  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
    8730  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
    8731  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
    8732  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
    8733  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
    8734  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
    8735  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
    8736  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
    8737  VMA_COPY_IF_NOT_NULL(vkCreateImage);
    8738  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
    8739 #if VMA_DEDICATED_ALLOCATION
    8740  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
    8741  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
    8742 #endif
    8743  }
    8744 
    8745 #undef VMA_COPY_IF_NOT_NULL
    8746 
    8747  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
    8748  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
    8749  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
    8750  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
    8751  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
    8752  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
    8753  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
    8754  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
    8755  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
    8756  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
    8757  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
    8758  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
    8759  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
    8760  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
    8761  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
    8762  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
    8763  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
    8764  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
    8765 #if VMA_DEDICATED_ALLOCATION
    8766  if(m_UseKhrDedicatedAllocation)
    8767  {
    8768  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
    8769  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
    8770  }
    8771 #endif
    8772 }
    8773 
    8774 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
    8775 {
    8776  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    8777  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
    8778  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
    8779  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
    8780 }
    8781 
    8782 VkResult VmaAllocator_T::AllocateMemoryOfType(
    8783  VkDeviceSize size,
    8784  VkDeviceSize alignment,
    8785  bool dedicatedAllocation,
    8786  VkBuffer dedicatedBuffer,
    8787  VkImage dedicatedImage,
    8788  const VmaAllocationCreateInfo& createInfo,
    8789  uint32_t memTypeIndex,
    8790  VmaSuballocationType suballocType,
    8791  VmaAllocation* pAllocation)
    8792 {
    8793  VMA_ASSERT(pAllocation != VMA_NULL);
    8794  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
    8795 
    8796  VmaAllocationCreateInfo finalCreateInfo = createInfo;
    8797 
    8798  // If memory type is not HOST_VISIBLE, disable MAPPED.
    8799  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8800  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    8801  {
    8802  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
    8803  }
    8804 
    8805  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
    8806  VMA_ASSERT(blockVector);
    8807 
    8808  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
    8809  bool preferDedicatedMemory =
    8810  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
    8811  dedicatedAllocation ||
    8812  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
    8813  size > preferredBlockSize / 2;
    8814 
    8815  if(preferDedicatedMemory &&
    8816  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
    8817  finalCreateInfo.pool == VK_NULL_HANDLE)
    8818  {
    8819  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
    8820  }
    8821 
    8822  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
    8823  {
    8824  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8825  {
    8826  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8827  }
    8828  else
    8829  {
    8830  return AllocateDedicatedMemory(
    8831  size,
    8832  suballocType,
    8833  memTypeIndex,
    8834  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8835  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8836  finalCreateInfo.pUserData,
    8837  dedicatedBuffer,
    8838  dedicatedImage,
    8839  pAllocation);
    8840  }
    8841  }
    8842  else
    8843  {
    8844  VkResult res = blockVector->Allocate(
    8845  VK_NULL_HANDLE, // hCurrentPool
    8846  m_CurrentFrameIndex.load(),
    8847  size,
    8848  alignment,
    8849  finalCreateInfo,
    8850  suballocType,
    8851  pAllocation);
    8852  if(res == VK_SUCCESS)
    8853  {
    8854  return res;
    8855  }
    8856 
    8857  // 5. Try dedicated memory.
    8858  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8859  {
    8860  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8861  }
    8862  else
    8863  {
    8864  res = AllocateDedicatedMemory(
    8865  size,
    8866  suballocType,
    8867  memTypeIndex,
    8868  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8869  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8870  finalCreateInfo.pUserData,
    8871  dedicatedBuffer,
    8872  dedicatedImage,
    8873  pAllocation);
    8874  if(res == VK_SUCCESS)
    8875  {
    8876  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
    8877  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
    8878  return VK_SUCCESS;
    8879  }
    8880  else
    8881  {
    8882  // Everything failed: Return error code.
    8883  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8884  return res;
    8885  }
    8886  }
    8887  }
    8888 }
    8889 
    8890 VkResult VmaAllocator_T::AllocateDedicatedMemory(
    8891  VkDeviceSize size,
    8892  VmaSuballocationType suballocType,
    8893  uint32_t memTypeIndex,
    8894  bool map,
    8895  bool isUserDataString,
    8896  void* pUserData,
    8897  VkBuffer dedicatedBuffer,
    8898  VkImage dedicatedImage,
    8899  VmaAllocation* pAllocation)
    8900 {
    8901  VMA_ASSERT(pAllocation);
    8902 
    8903  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    8904  allocInfo.memoryTypeIndex = memTypeIndex;
    8905  allocInfo.allocationSize = size;
    8906 
    8907 #if VMA_DEDICATED_ALLOCATION
    8908  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
    8909  if(m_UseKhrDedicatedAllocation)
    8910  {
    8911  if(dedicatedBuffer != VK_NULL_HANDLE)
    8912  {
    8913  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
    8914  dedicatedAllocInfo.buffer = dedicatedBuffer;
    8915  allocInfo.pNext = &dedicatedAllocInfo;
    8916  }
    8917  else if(dedicatedImage != VK_NULL_HANDLE)
    8918  {
    8919  dedicatedAllocInfo.image = dedicatedImage;
    8920  allocInfo.pNext = &dedicatedAllocInfo;
    8921  }
    8922  }
    8923 #endif // #if VMA_DEDICATED_ALLOCATION
    8924 
    8925  // Allocate VkDeviceMemory.
    8926  VkDeviceMemory hMemory = VK_NULL_HANDLE;
    8927  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
    8928  if(res < 0)
    8929  {
    8930  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8931  return res;
    8932  }
    8933 
    8934  void* pMappedData = VMA_NULL;
    8935  if(map)
    8936  {
    8937  res = (*m_VulkanFunctions.vkMapMemory)(
    8938  m_hDevice,
    8939  hMemory,
    8940  0,
    8941  VK_WHOLE_SIZE,
    8942  0,
    8943  &pMappedData);
    8944  if(res < 0)
    8945  {
    8946  VMA_DEBUG_LOG(" vkMapMemory FAILED");
    8947  FreeVulkanMemory(memTypeIndex, size, hMemory);
    8948  return res;
    8949  }
    8950  }
    8951 
    8952  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
    8953  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
    8954  (*pAllocation)->SetUserData(this, pUserData);
    8955  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    8956  {
    8957  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    8958  }
    8959 
    8960  // Register it in m_pDedicatedAllocations.
    8961  {
    8962  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    8963  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    8964  VMA_ASSERT(pDedicatedAllocations);
    8965  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
    8966  }
    8967 
    8968  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
    8969 
    8970  return VK_SUCCESS;
    8971 }
    8972 
    8973 void VmaAllocator_T::GetBufferMemoryRequirements(
    8974  VkBuffer hBuffer,
    8975  VkMemoryRequirements& memReq,
    8976  bool& requiresDedicatedAllocation,
    8977  bool& prefersDedicatedAllocation) const
    8978 {
    8979 #if VMA_DEDICATED_ALLOCATION
    8980  if(m_UseKhrDedicatedAllocation)
    8981  {
    8982  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
    8983  memReqInfo.buffer = hBuffer;
    8984 
    8985  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    8986 
    8987  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    8988  memReq2.pNext = &memDedicatedReq;
    8989 
    8990  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    8991 
    8992  memReq = memReq2.memoryRequirements;
    8993  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    8994  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    8995  }
    8996  else
    8997 #endif // #if VMA_DEDICATED_ALLOCATION
    8998  {
    8999  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
    9000  requiresDedicatedAllocation = false;
    9001  prefersDedicatedAllocation = false;
    9002  }
    9003 }
    9004 
    9005 void VmaAllocator_T::GetImageMemoryRequirements(
    9006  VkImage hImage,
    9007  VkMemoryRequirements& memReq,
    9008  bool& requiresDedicatedAllocation,
    9009  bool& prefersDedicatedAllocation) const
    9010 {
    9011 #if VMA_DEDICATED_ALLOCATION
    9012  if(m_UseKhrDedicatedAllocation)
    9013  {
    9014  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
    9015  memReqInfo.image = hImage;
    9016 
    9017  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    9018 
    9019  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    9020  memReq2.pNext = &memDedicatedReq;
    9021 
    9022  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    9023 
    9024  memReq = memReq2.memoryRequirements;
    9025  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    9026  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    9027  }
    9028  else
    9029 #endif // #if VMA_DEDICATED_ALLOCATION
    9030  {
    9031  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
    9032  requiresDedicatedAllocation = false;
    9033  prefersDedicatedAllocation = false;
    9034  }
    9035 }
    9036 
    9037 VkResult VmaAllocator_T::AllocateMemory(
    9038  const VkMemoryRequirements& vkMemReq,
    9039  bool requiresDedicatedAllocation,
    9040  bool prefersDedicatedAllocation,
    9041  VkBuffer dedicatedBuffer,
    9042  VkImage dedicatedImage,
    9043  const VmaAllocationCreateInfo& createInfo,
    9044  VmaSuballocationType suballocType,
    9045  VmaAllocation* pAllocation)
    9046 {
    9047  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
    9048  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    9049  {
    9050  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
    9051  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9052  }
    9053  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    9054  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
    9055  {
    9056  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
    9057  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9058  }
    9059  if(requiresDedicatedAllocation)
    9060  {
    9061  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    9062  {
    9063  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
    9064  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9065  }
    9066  if(createInfo.pool != VK_NULL_HANDLE)
    9067  {
    9068  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
    9069  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9070  }
    9071  }
    9072  if((createInfo.pool != VK_NULL_HANDLE) &&
    9073  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
    9074  {
    9075  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
    9076  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9077  }
    9078 
    9079  if(createInfo.pool != VK_NULL_HANDLE)
    9080  {
    9081  const VkDeviceSize alignmentForPool = VMA_MAX(
    9082  vkMemReq.alignment,
    9083  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
    9084  return createInfo.pool->m_BlockVector.Allocate(
    9085  createInfo.pool,
    9086  m_CurrentFrameIndex.load(),
    9087  vkMemReq.size,
    9088  alignmentForPool,
    9089  createInfo,
    9090  suballocType,
    9091  pAllocation);
    9092  }
    9093  else
    9094  {
    9095  // Bit mask of memory Vulkan types acceptable for this allocation.
    9096  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
    9097  uint32_t memTypeIndex = UINT32_MAX;
    9098  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9099  if(res == VK_SUCCESS)
    9100  {
    9101  VkDeviceSize alignmentForMemType = VMA_MAX(
    9102  vkMemReq.alignment,
    9103  GetMemoryTypeMinAlignment(memTypeIndex));
    9104 
    9105  res = AllocateMemoryOfType(
    9106  vkMemReq.size,
    9107  alignmentForMemType,
    9108  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9109  dedicatedBuffer,
    9110  dedicatedImage,
    9111  createInfo,
    9112  memTypeIndex,
    9113  suballocType,
    9114  pAllocation);
    9115  // Succeeded on first try.
    9116  if(res == VK_SUCCESS)
    9117  {
    9118  return res;
    9119  }
    9120  // Allocation from this memory type failed. Try other compatible memory types.
    9121  else
    9122  {
    9123  for(;;)
    9124  {
    9125  // Remove old memTypeIndex from list of possibilities.
    9126  memoryTypeBits &= ~(1u << memTypeIndex);
    9127  // Find alternative memTypeIndex.
    9128  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9129  if(res == VK_SUCCESS)
    9130  {
    9131  alignmentForMemType = VMA_MAX(
    9132  vkMemReq.alignment,
    9133  GetMemoryTypeMinAlignment(memTypeIndex));
    9134 
    9135  res = AllocateMemoryOfType(
    9136  vkMemReq.size,
    9137  alignmentForMemType,
    9138  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9139  dedicatedBuffer,
    9140  dedicatedImage,
    9141  createInfo,
    9142  memTypeIndex,
    9143  suballocType,
    9144  pAllocation);
    9145  // Allocation from this alternative memory type succeeded.
    9146  if(res == VK_SUCCESS)
    9147  {
    9148  return res;
    9149  }
    9150  // else: Allocation from this memory type failed. Try next one - next loop iteration.
    9151  }
    9152  // No other matching memory type index could be found.
    9153  else
    9154  {
    9155  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
    9156  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9157  }
    9158  }
    9159  }
    9160  }
    9161  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
    9162  else
    9163  return res;
    9164  }
    9165 }
    9166 
    9167 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
    9168 {
    9169  VMA_ASSERT(allocation);
    9170 
    9171  if(allocation->CanBecomeLost() == false ||
    9172  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    9173  {
    9174  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    9175  {
    9176  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
    9177  }
    9178 
    9179  switch(allocation->GetType())
    9180  {
    9181  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9182  {
    9183  VmaBlockVector* pBlockVector = VMA_NULL;
    9184  VmaPool hPool = allocation->GetPool();
    9185  if(hPool != VK_NULL_HANDLE)
    9186  {
    9187  pBlockVector = &hPool->m_BlockVector;
    9188  }
    9189  else
    9190  {
    9191  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9192  pBlockVector = m_pBlockVectors[memTypeIndex];
    9193  }
    9194  pBlockVector->Free(allocation);
    9195  }
    9196  break;
    9197  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9198  FreeDedicatedMemory(allocation);
    9199  break;
    9200  default:
    9201  VMA_ASSERT(0);
    9202  }
    9203  }
    9204 
    9205  allocation->SetUserData(this, VMA_NULL);
    9206  vma_delete(this, allocation);
    9207 }
    9208 
    9209 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
    9210 {
    9211  // Initialize.
    9212  InitStatInfo(pStats->total);
    9213  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
    9214  InitStatInfo(pStats->memoryType[i]);
    9215  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    9216  InitStatInfo(pStats->memoryHeap[i]);
    9217 
    9218  // Process default pools.
    9219  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9220  {
    9221  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9222  VMA_ASSERT(pBlockVector);
    9223  pBlockVector->AddStats(pStats);
    9224  }
    9225 
    9226  // Process custom pools.
    9227  {
    9228  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9229  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9230  {
    9231  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
    9232  }
    9233  }
    9234 
    9235  // Process dedicated allocations.
    9236  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9237  {
    9238  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    9239  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9240  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9241  VMA_ASSERT(pDedicatedAllocVector);
    9242  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
    9243  {
    9244  VmaStatInfo allocationStatInfo;
    9245  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
    9246  VmaAddStatInfo(pStats->total, allocationStatInfo);
    9247  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    9248  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    9249  }
    9250  }
    9251 
    9252  // Postprocess.
    9253  VmaPostprocessCalcStatInfo(pStats->total);
    9254  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
    9255  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
    9256  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
    9257  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
    9258 }
    9259 
    9260 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
    9261 
    9262 VkResult VmaAllocator_T::Defragment(
    9263  VmaAllocation* pAllocations,
    9264  size_t allocationCount,
    9265  VkBool32* pAllocationsChanged,
    9266  const VmaDefragmentationInfo* pDefragmentationInfo,
    9267  VmaDefragmentationStats* pDefragmentationStats)
    9268 {
    9269  if(pAllocationsChanged != VMA_NULL)
    9270  {
    9271  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
    9272  }
    9273  if(pDefragmentationStats != VMA_NULL)
    9274  {
    9275  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
    9276  }
    9277 
    9278  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
    9279 
    9280  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
    9281 
    9282  const size_t poolCount = m_Pools.size();
    9283 
    9284  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
    9285  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
    9286  {
    9287  VmaAllocation hAlloc = pAllocations[allocIndex];
    9288  VMA_ASSERT(hAlloc);
    9289  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
    9290  // DedicatedAlloc cannot be defragmented.
    9291  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
    9292  // Only HOST_VISIBLE memory types can be defragmented.
    9293  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
    9294  // Lost allocation cannot be defragmented.
    9295  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
    9296  {
    9297  VmaBlockVector* pAllocBlockVector = VMA_NULL;
    9298 
    9299  const VmaPool hAllocPool = hAlloc->GetPool();
    9300  // This allocation belongs to custom pool.
    9301  if(hAllocPool != VK_NULL_HANDLE)
    9302  {
    9303  pAllocBlockVector = &hAllocPool->GetBlockVector();
    9304  }
    9305  // This allocation belongs to general pool.
    9306  else
    9307  {
    9308  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
    9309  }
    9310 
    9311  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
    9312 
    9313  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
    9314  &pAllocationsChanged[allocIndex] : VMA_NULL;
    9315  pDefragmentator->AddAllocation(hAlloc, pChanged);
    9316  }
    9317  }
    9318 
    9319  VkResult result = VK_SUCCESS;
    9320 
    9321  // ======== Main processing.
    9322 
    9323  VkDeviceSize maxBytesToMove = SIZE_MAX;
    9324  uint32_t maxAllocationsToMove = UINT32_MAX;
    9325  if(pDefragmentationInfo != VMA_NULL)
    9326  {
    9327  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
    9328  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
    9329  }
    9330 
    9331  // Process standard memory.
    9332  for(uint32_t memTypeIndex = 0;
    9333  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
    9334  ++memTypeIndex)
    9335  {
    9336  // Only HOST_VISIBLE memory types can be defragmented.
    9337  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9338  {
    9339  result = m_pBlockVectors[memTypeIndex]->Defragment(
    9340  pDefragmentationStats,
    9341  maxBytesToMove,
    9342  maxAllocationsToMove);
    9343  }
    9344  }
    9345 
    9346  // Process custom pools.
    9347  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
    9348  {
    9349  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
    9350  pDefragmentationStats,
    9351  maxBytesToMove,
    9352  maxAllocationsToMove);
    9353  }
    9354 
    9355  // ======== Destroy defragmentators.
    9356 
    9357  // Process custom pools.
    9358  for(size_t poolIndex = poolCount; poolIndex--; )
    9359  {
    9360  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
    9361  }
    9362 
    9363  // Process standard memory.
    9364  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
    9365  {
    9366  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9367  {
    9368  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
    9369  }
    9370  }
    9371 
    9372  return result;
    9373 }
    9374 
    9375 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
    9376 {
    9377  if(hAllocation->CanBecomeLost())
    9378  {
    9379  /*
    9380  Warning: This is a carefully designed algorithm.
    9381  Do not modify unless you really know what you're doing :)
    9382  */
    9383  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9384  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9385  for(;;)
    9386  {
    9387  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9388  {
    9389  pAllocationInfo->memoryType = UINT32_MAX;
    9390  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
    9391  pAllocationInfo->offset = 0;
    9392  pAllocationInfo->size = hAllocation->GetSize();
    9393  pAllocationInfo->pMappedData = VMA_NULL;
    9394  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9395  return;
    9396  }
    9397  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9398  {
    9399  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9400  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9401  pAllocationInfo->offset = hAllocation->GetOffset();
    9402  pAllocationInfo->size = hAllocation->GetSize();
    9403  pAllocationInfo->pMappedData = VMA_NULL;
    9404  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9405  return;
    9406  }
    9407  else // Last use time earlier than current time.
    9408  {
    9409  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9410  {
    9411  localLastUseFrameIndex = localCurrFrameIndex;
    9412  }
    9413  }
    9414  }
    9415  }
    9416  else
    9417  {
    9418 #if VMA_STATS_STRING_ENABLED
    9419  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9420  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9421  for(;;)
    9422  {
    9423  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9424  if(localLastUseFrameIndex == localCurrFrameIndex)
    9425  {
    9426  break;
    9427  }
    9428  else // Last use time earlier than current time.
    9429  {
    9430  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9431  {
    9432  localLastUseFrameIndex = localCurrFrameIndex;
    9433  }
    9434  }
    9435  }
    9436 #endif
    9437 
    9438  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9439  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9440  pAllocationInfo->offset = hAllocation->GetOffset();
    9441  pAllocationInfo->size = hAllocation->GetSize();
    9442  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
    9443  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9444  }
    9445 }
    9446 
    9447 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
    9448 {
    9449  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
    9450  if(hAllocation->CanBecomeLost())
    9451  {
    9452  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9453  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9454  for(;;)
    9455  {
    9456  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9457  {
    9458  return false;
    9459  }
    9460  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9461  {
    9462  return true;
    9463  }
    9464  else // Last use time earlier than current time.
    9465  {
    9466  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9467  {
    9468  localLastUseFrameIndex = localCurrFrameIndex;
    9469  }
    9470  }
    9471  }
    9472  }
    9473  else
    9474  {
    9475 #if VMA_STATS_STRING_ENABLED
    9476  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9477  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9478  for(;;)
    9479  {
    9480  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9481  if(localLastUseFrameIndex == localCurrFrameIndex)
    9482  {
    9483  break;
    9484  }
    9485  else // Last use time earlier than current time.
    9486  {
    9487  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9488  {
    9489  localLastUseFrameIndex = localCurrFrameIndex;
    9490  }
    9491  }
    9492  }
    9493 #endif
    9494 
    9495  return true;
    9496  }
    9497 }
    9498 
    9499 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
    9500 {
    9501  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
    9502 
    9503  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
    9504 
    9505  if(newCreateInfo.maxBlockCount == 0)
    9506  {
    9507  newCreateInfo.maxBlockCount = SIZE_MAX;
    9508  }
    9509  if(newCreateInfo.blockSize == 0)
    9510  {
    9511  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
    9512  }
    9513 
    9514  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
    9515 
    9516  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
    9517  if(res != VK_SUCCESS)
    9518  {
    9519  vma_delete(this, *pPool);
    9520  *pPool = VMA_NULL;
    9521  return res;
    9522  }
    9523 
    9524  // Add to m_Pools.
    9525  {
    9526  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9527  (*pPool)->SetId(m_NextPoolId++);
    9528  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
    9529  }
    9530 
    9531  return VK_SUCCESS;
    9532 }
    9533 
    9534 void VmaAllocator_T::DestroyPool(VmaPool pool)
    9535 {
    9536  // Remove from m_Pools.
    9537  {
    9538  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9539  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
    9540  VMA_ASSERT(success && "Pool not found in Allocator.");
    9541  }
    9542 
    9543  vma_delete(this, pool);
    9544 }
    9545 
    9546 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
    9547 {
    9548  pool->m_BlockVector.GetPoolStats(pPoolStats);
    9549 }
    9550 
    9551 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
    9552 {
    9553  m_CurrentFrameIndex.store(frameIndex);
    9554 }
    9555 
    9556 void VmaAllocator_T::MakePoolAllocationsLost(
    9557  VmaPool hPool,
    9558  size_t* pLostAllocationCount)
    9559 {
    9560  hPool->m_BlockVector.MakePoolAllocationsLost(
    9561  m_CurrentFrameIndex.load(),
    9562  pLostAllocationCount);
    9563 }
    9564 
    9565 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
    9566 {
    9567  return hPool->m_BlockVector.CheckCorruption();
    9568 }
    9569 
    9570 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
    9571 {
    9572  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
    9573 
    9574  // Process default pools.
    9575  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9576  {
    9577  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
    9578  {
    9579  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9580  VMA_ASSERT(pBlockVector);
    9581  VkResult localRes = pBlockVector->CheckCorruption();
    9582  switch(localRes)
    9583  {
    9584  case VK_ERROR_FEATURE_NOT_PRESENT:
    9585  break;
    9586  case VK_SUCCESS:
    9587  finalRes = VK_SUCCESS;
    9588  break;
    9589  default:
    9590  return localRes;
    9591  }
    9592  }
    9593  }
    9594 
    9595  // Process custom pools.
    9596  {
    9597  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9598  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9599  {
    9600  if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0)
    9601  {
    9602  VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption();
    9603  switch(localRes)
    9604  {
    9605  case VK_ERROR_FEATURE_NOT_PRESENT:
    9606  break;
    9607  case VK_SUCCESS:
    9608  finalRes = VK_SUCCESS;
    9609  break;
    9610  default:
    9611  return localRes;
    9612  }
    9613  }
    9614  }
    9615  }
    9616 
    9617  return finalRes;
    9618 }
    9619 
    9620 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
    9621 {
    9622  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
    9623  (*pAllocation)->InitLost();
    9624 }
    9625 
    9626 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
    9627 {
    9628  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
    9629 
    9630  VkResult res;
    9631  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9632  {
    9633  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9634  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
    9635  {
    9636  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9637  if(res == VK_SUCCESS)
    9638  {
    9639  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
    9640  }
    9641  }
    9642  else
    9643  {
    9644  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9645  }
    9646  }
    9647  else
    9648  {
    9649  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9650  }
    9651 
    9652  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
    9653  {
    9654  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
    9655  }
    9656 
    9657  return res;
    9658 }
    9659 
    9660 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
    9661 {
    9662  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
    9663  {
    9664  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
    9665  }
    9666 
    9667  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
    9668 
    9669  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
    9670  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9671  {
    9672  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9673  m_HeapSizeLimit[heapIndex] += size;
    9674  }
    9675 }
    9676 
    9677 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
    9678 {
    9679  if(hAllocation->CanBecomeLost())
    9680  {
    9681  return VK_ERROR_MEMORY_MAP_FAILED;
    9682  }
    9683 
    9684  switch(hAllocation->GetType())
    9685  {
    9686  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9687  {
    9688  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9689  char *pBytes = VMA_NULL;
    9690  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
    9691  if(res == VK_SUCCESS)
    9692  {
    9693  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
    9694  hAllocation->BlockAllocMap();
    9695  }
    9696  return res;
    9697  }
    9698  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9699  return hAllocation->DedicatedAllocMap(this, ppData);
    9700  default:
    9701  VMA_ASSERT(0);
    9702  return VK_ERROR_MEMORY_MAP_FAILED;
    9703  }
    9704 }
    9705 
    9706 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
    9707 {
    9708  switch(hAllocation->GetType())
    9709  {
    9710  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9711  {
    9712  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9713  hAllocation->BlockAllocUnmap();
    9714  pBlock->Unmap(this, 1);
    9715  }
    9716  break;
    9717  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9718  hAllocation->DedicatedAllocUnmap(this);
    9719  break;
    9720  default:
    9721  VMA_ASSERT(0);
    9722  }
    9723 }
    9724 
    9725 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
    9726 {
    9727  VkResult res = VK_SUCCESS;
    9728  switch(hAllocation->GetType())
    9729  {
    9730  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9731  res = GetVulkanFunctions().vkBindBufferMemory(
    9732  m_hDevice,
    9733  hBuffer,
    9734  hAllocation->GetMemory(),
    9735  0); //memoryOffset
    9736  break;
    9737  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9738  {
    9739  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9740  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
    9741  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
    9742  break;
    9743  }
    9744  default:
    9745  VMA_ASSERT(0);
    9746  }
    9747  return res;
    9748 }
    9749 
    9750 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
    9751 {
    9752  VkResult res = VK_SUCCESS;
    9753  switch(hAllocation->GetType())
    9754  {
    9755  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9756  res = GetVulkanFunctions().vkBindImageMemory(
    9757  m_hDevice,
    9758  hImage,
    9759  hAllocation->GetMemory(),
    9760  0); //memoryOffset
    9761  break;
    9762  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9763  {
    9764  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9765  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
    9766  res = pBlock->BindImageMemory(this, hAllocation, hImage);
    9767  break;
    9768  }
    9769  default:
    9770  VMA_ASSERT(0);
    9771  }
    9772  return res;
    9773 }
    9774 
    9775 void VmaAllocator_T::FlushOrInvalidateAllocation(
    9776  VmaAllocation hAllocation,
    9777  VkDeviceSize offset, VkDeviceSize size,
    9778  VMA_CACHE_OPERATION op)
    9779 {
    9780  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
    9781  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
    9782  {
    9783  const VkDeviceSize allocationSize = hAllocation->GetSize();
    9784  VMA_ASSERT(offset <= allocationSize);
    9785 
    9786  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
    9787 
    9788  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
    9789  memRange.memory = hAllocation->GetMemory();
    9790 
    9791  switch(hAllocation->GetType())
    9792  {
    9793  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9794  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9795  if(size == VK_WHOLE_SIZE)
    9796  {
    9797  memRange.size = allocationSize - memRange.offset;
    9798  }
    9799  else
    9800  {
    9801  VMA_ASSERT(offset + size <= allocationSize);
    9802  memRange.size = VMA_MIN(
    9803  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
    9804  allocationSize - memRange.offset);
    9805  }
    9806  break;
    9807 
    9808  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9809  {
    9810  // 1. Still within this allocation.
    9811  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9812  if(size == VK_WHOLE_SIZE)
    9813  {
    9814  size = allocationSize - offset;
    9815  }
    9816  else
    9817  {
    9818  VMA_ASSERT(offset + size <= allocationSize);
    9819  }
    9820  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
    9821 
    9822  // 2. Adjust to whole block.
    9823  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
    9824  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
    9825  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize();
    9826  memRange.offset += allocationOffset;
    9827  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
    9828 
    9829  break;
    9830  }
    9831 
    9832  default:
    9833  VMA_ASSERT(0);
    9834  }
    9835 
    9836  switch(op)
    9837  {
    9838  case VMA_CACHE_FLUSH:
    9839  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9840  break;
    9841  case VMA_CACHE_INVALIDATE:
    9842  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9843  break;
    9844  default:
    9845  VMA_ASSERT(0);
    9846  }
    9847  }
    9848  // else: Just ignore this call.
    9849 }
    9850 
    9851 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
    9852 {
    9853  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
    9854 
    9855  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9856  {
    9857  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9858  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    9859  VMA_ASSERT(pDedicatedAllocations);
    9860  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
    9861  VMA_ASSERT(success);
    9862  }
    9863 
    9864  VkDeviceMemory hMemory = allocation->GetMemory();
    9865 
    9866  if(allocation->GetMappedData() != VMA_NULL)
    9867  {
    9868  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
    9869  }
    9870 
    9871  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
    9872 
    9873  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
    9874 }
    9875 
    9876 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
    9877 {
    9878  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
    9879  !hAllocation->CanBecomeLost() &&
    9880  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9881  {
    9882  void* pData = VMA_NULL;
    9883  VkResult res = Map(hAllocation, &pData);
    9884  if(res == VK_SUCCESS)
    9885  {
    9886  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
    9887  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
    9888  Unmap(hAllocation);
    9889  }
    9890  else
    9891  {
    9892  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
    9893  }
    9894  }
    9895 }
    9896 
    9897 #if VMA_STATS_STRING_ENABLED
    9898 
    9899 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
    9900 {
    9901  bool dedicatedAllocationsStarted = false;
    9902  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9903  {
    9904  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9905  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9906  VMA_ASSERT(pDedicatedAllocVector);
    9907  if(pDedicatedAllocVector->empty() == false)
    9908  {
    9909  if(dedicatedAllocationsStarted == false)
    9910  {
    9911  dedicatedAllocationsStarted = true;
    9912  json.WriteString("DedicatedAllocations");
    9913  json.BeginObject();
    9914  }
    9915 
    9916  json.BeginString("Type ");
    9917  json.ContinueString(memTypeIndex);
    9918  json.EndString();
    9919 
    9920  json.BeginArray();
    9921 
    9922  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
    9923  {
    9924  json.BeginObject(true);
    9925  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
    9926  hAlloc->PrintParameters(json);
    9927  json.EndObject();
    9928  }
    9929 
    9930  json.EndArray();
    9931  }
    9932  }
    9933  if(dedicatedAllocationsStarted)
    9934  {
    9935  json.EndObject();
    9936  }
    9937 
    9938  {
    9939  bool allocationsStarted = false;
    9940  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9941  {
    9942  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
    9943  {
    9944  if(allocationsStarted == false)
    9945  {
    9946  allocationsStarted = true;
    9947  json.WriteString("DefaultPools");
    9948  json.BeginObject();
    9949  }
    9950 
    9951  json.BeginString("Type ");
    9952  json.ContinueString(memTypeIndex);
    9953  json.EndString();
    9954 
    9955  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
    9956  }
    9957  }
    9958  if(allocationsStarted)
    9959  {
    9960  json.EndObject();
    9961  }
    9962  }
    9963 
    9964  {
    9965  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9966  const size_t poolCount = m_Pools.size();
    9967  if(poolCount > 0)
    9968  {
    9969  json.WriteString("Pools");
    9970  json.BeginObject();
    9971  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
    9972  {
    9973  json.BeginString();
    9974  json.ContinueString(m_Pools[poolIndex]->GetId());
    9975  json.EndString();
    9976 
    9977  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
    9978  }
    9979  json.EndObject();
    9980  }
    9981  }
    9982 }
    9983 
    9984 #endif // #if VMA_STATS_STRING_ENABLED
    9985 
    9987 // Public interface
    9988 
    9989 VkResult vmaCreateAllocator(
    9990  const VmaAllocatorCreateInfo* pCreateInfo,
    9991  VmaAllocator* pAllocator)
    9992 {
    9993  VMA_ASSERT(pCreateInfo && pAllocator);
    9994  VMA_DEBUG_LOG("vmaCreateAllocator");
    9995  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
    9996  return (*pAllocator)->Init(pCreateInfo);
    9997 }
    9998 
    9999 void vmaDestroyAllocator(
    10000  VmaAllocator allocator)
    10001 {
    10002  if(allocator != VK_NULL_HANDLE)
    10003  {
    10004  VMA_DEBUG_LOG("vmaDestroyAllocator");
    10005  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
    10006  vma_delete(&allocationCallbacks, allocator);
    10007  }
    10008 }
    10009 
    10010 void vmaGetPhysicalDeviceProperties(
    10011  VmaAllocator allocator,
    10012  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
    10013 {
    10014  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
    10015  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
    10016 }
    10017 
    10018 void vmaGetMemoryProperties(
    10019  VmaAllocator allocator,
    10020  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
    10021 {
    10022  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
    10023  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
    10024 }
    10025 
    10026 void vmaGetMemoryTypeProperties(
    10027  VmaAllocator allocator,
    10028  uint32_t memoryTypeIndex,
    10029  VkMemoryPropertyFlags* pFlags)
    10030 {
    10031  VMA_ASSERT(allocator && pFlags);
    10032  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
    10033  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
    10034 }
    10035 
    10036 void vmaSetCurrentFrameIndex(
    10037  VmaAllocator allocator,
    10038  uint32_t frameIndex)
    10039 {
    10040  VMA_ASSERT(allocator);
    10041  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
    10042 
    10043  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10044 
    10045  allocator->SetCurrentFrameIndex(frameIndex);
    10046 }
    10047 
    10048 void vmaCalculateStats(
    10049  VmaAllocator allocator,
    10050  VmaStats* pStats)
    10051 {
    10052  VMA_ASSERT(allocator && pStats);
    10053  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10054  allocator->CalculateStats(pStats);
    10055 }
    10056 
    10057 #if VMA_STATS_STRING_ENABLED
    10058 
    10059 void vmaBuildStatsString(
    10060  VmaAllocator allocator,
    10061  char** ppStatsString,
    10062  VkBool32 detailedMap)
    10063 {
    10064  VMA_ASSERT(allocator && ppStatsString);
    10065  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10066 
    10067  VmaStringBuilder sb(allocator);
    10068  {
    10069  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
    10070  json.BeginObject();
    10071 
    10072  VmaStats stats;
    10073  allocator->CalculateStats(&stats);
    10074 
    10075  json.WriteString("Total");
    10076  VmaPrintStatInfo(json, stats.total);
    10077 
    10078  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
    10079  {
    10080  json.BeginString("Heap ");
    10081  json.ContinueString(heapIndex);
    10082  json.EndString();
    10083  json.BeginObject();
    10084 
    10085  json.WriteString("Size");
    10086  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
    10087 
    10088  json.WriteString("Flags");
    10089  json.BeginArray(true);
    10090  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
    10091  {
    10092  json.WriteString("DEVICE_LOCAL");
    10093  }
    10094  json.EndArray();
    10095 
    10096  if(stats.memoryHeap[heapIndex].blockCount > 0)
    10097  {
    10098  json.WriteString("Stats");
    10099  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
    10100  }
    10101 
    10102  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
    10103  {
    10104  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
    10105  {
    10106  json.BeginString("Type ");
    10107  json.ContinueString(typeIndex);
    10108  json.EndString();
    10109 
    10110  json.BeginObject();
    10111 
    10112  json.WriteString("Flags");
    10113  json.BeginArray(true);
    10114  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
    10115  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
    10116  {
    10117  json.WriteString("DEVICE_LOCAL");
    10118  }
    10119  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    10120  {
    10121  json.WriteString("HOST_VISIBLE");
    10122  }
    10123  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
    10124  {
    10125  json.WriteString("HOST_COHERENT");
    10126  }
    10127  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
    10128  {
    10129  json.WriteString("HOST_CACHED");
    10130  }
    10131  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
    10132  {
    10133  json.WriteString("LAZILY_ALLOCATED");
    10134  }
    10135  json.EndArray();
    10136 
    10137  if(stats.memoryType[typeIndex].blockCount > 0)
    10138  {
    10139  json.WriteString("Stats");
    10140  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
    10141  }
    10142 
    10143  json.EndObject();
    10144  }
    10145  }
    10146 
    10147  json.EndObject();
    10148  }
    10149  if(detailedMap == VK_TRUE)
    10150  {
    10151  allocator->PrintDetailedMap(json);
    10152  }
    10153 
    10154  json.EndObject();
    10155  }
    10156 
    10157  const size_t len = sb.GetLength();
    10158  char* const pChars = vma_new_array(allocator, char, len + 1);
    10159  if(len > 0)
    10160  {
    10161  memcpy(pChars, sb.GetData(), len);
    10162  }
    10163  pChars[len] = '\0';
    10164  *ppStatsString = pChars;
    10165 }
    10166 
    10167 void vmaFreeStatsString(
    10168  VmaAllocator allocator,
    10169  char* pStatsString)
    10170 {
    10171  if(pStatsString != VMA_NULL)
    10172  {
    10173  VMA_ASSERT(allocator);
    10174  size_t len = strlen(pStatsString);
    10175  vma_delete_array(allocator, pStatsString, len + 1);
    10176  }
    10177 }
    10178 
    10179 #endif // #if VMA_STATS_STRING_ENABLED
    10180 
    10181 /*
    10182 This function is not protected by any mutex because it just reads immutable data.
    10183 */
    10184 VkResult vmaFindMemoryTypeIndex(
    10185  VmaAllocator allocator,
    10186  uint32_t memoryTypeBits,
    10187  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10188  uint32_t* pMemoryTypeIndex)
    10189 {
    10190  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10191  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10192  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10193 
    10194  if(pAllocationCreateInfo->memoryTypeBits != 0)
    10195  {
    10196  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
    10197  }
    10198 
    10199  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
    10200  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
    10201 
    10202  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    10203  if(mapped)
    10204  {
    10205  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10206  }
    10207 
    10208  // Convert usage to requiredFlags and preferredFlags.
    10209  switch(pAllocationCreateInfo->usage)
    10210  {
    10211  case VMA_MEMORY_USAGE_UNKNOWN:
    10212  break;
    10213  case VMA_MEMORY_USAGE_GPU_ONLY:
    10214  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10215  {
    10216  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10217  }
    10218  break;
    10219  case VMA_MEMORY_USAGE_CPU_ONLY:
    10220  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    10221  break;
    10222  case VMA_MEMORY_USAGE_CPU_TO_GPU:
    10223  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10224  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10225  {
    10226  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10227  }
    10228  break;
    10229  case VMA_MEMORY_USAGE_GPU_TO_CPU:
    10230  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10231  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
    10232  break;
    10233  default:
    10234  break;
    10235  }
    10236 
    10237  *pMemoryTypeIndex = UINT32_MAX;
    10238  uint32_t minCost = UINT32_MAX;
    10239  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
    10240  memTypeIndex < allocator->GetMemoryTypeCount();
    10241  ++memTypeIndex, memTypeBit <<= 1)
    10242  {
    10243  // This memory type is acceptable according to memoryTypeBits bitmask.
    10244  if((memTypeBit & memoryTypeBits) != 0)
    10245  {
    10246  const VkMemoryPropertyFlags currFlags =
    10247  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
    10248  // This memory type contains requiredFlags.
    10249  if((requiredFlags & ~currFlags) == 0)
    10250  {
    10251  // Calculate cost as number of bits from preferredFlags not present in this memory type.
    10252  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
    10253  // Remember memory type with lowest cost.
    10254  if(currCost < minCost)
    10255  {
    10256  *pMemoryTypeIndex = memTypeIndex;
    10257  if(currCost == 0)
    10258  {
    10259  return VK_SUCCESS;
    10260  }
    10261  minCost = currCost;
    10262  }
    10263  }
    10264  }
    10265  }
    10266  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
    10267 }
    10268 
    10269 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    10270  VmaAllocator allocator,
    10271  const VkBufferCreateInfo* pBufferCreateInfo,
    10272  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10273  uint32_t* pMemoryTypeIndex)
    10274 {
    10275  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10276  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
    10277  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10278  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10279 
    10280  const VkDevice hDev = allocator->m_hDevice;
    10281  VkBuffer hBuffer = VK_NULL_HANDLE;
    10282  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
    10283  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
    10284  if(res == VK_SUCCESS)
    10285  {
    10286  VkMemoryRequirements memReq = {};
    10287  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
    10288  hDev, hBuffer, &memReq);
    10289 
    10290  res = vmaFindMemoryTypeIndex(
    10291  allocator,
    10292  memReq.memoryTypeBits,
    10293  pAllocationCreateInfo,
    10294  pMemoryTypeIndex);
    10295 
    10296  allocator->GetVulkanFunctions().vkDestroyBuffer(
    10297  hDev, hBuffer, allocator->GetAllocationCallbacks());
    10298  }
    10299  return res;
    10300 }
    10301 
    10302 VkResult vmaFindMemoryTypeIndexForImageInfo(
    10303  VmaAllocator allocator,
    10304  const VkImageCreateInfo* pImageCreateInfo,
    10305  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10306  uint32_t* pMemoryTypeIndex)
    10307 {
    10308  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10309  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
    10310  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10311  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10312 
    10313  const VkDevice hDev = allocator->m_hDevice;
    10314  VkImage hImage = VK_NULL_HANDLE;
    10315  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
    10316  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
    10317  if(res == VK_SUCCESS)
    10318  {
    10319  VkMemoryRequirements memReq = {};
    10320  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
    10321  hDev, hImage, &memReq);
    10322 
    10323  res = vmaFindMemoryTypeIndex(
    10324  allocator,
    10325  memReq.memoryTypeBits,
    10326  pAllocationCreateInfo,
    10327  pMemoryTypeIndex);
    10328 
    10329  allocator->GetVulkanFunctions().vkDestroyImage(
    10330  hDev, hImage, allocator->GetAllocationCallbacks());
    10331  }
    10332  return res;
    10333 }
    10334 
    10335 VkResult vmaCreatePool(
    10336  VmaAllocator allocator,
    10337  const VmaPoolCreateInfo* pCreateInfo,
    10338  VmaPool* pPool)
    10339 {
    10340  VMA_ASSERT(allocator && pCreateInfo && pPool);
    10341 
    10342  VMA_DEBUG_LOG("vmaCreatePool");
    10343 
    10344  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10345 
    10346  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
    10347 
    10348 #if VMA_RECORDING_ENABLED
    10349  if(allocator->GetRecorder() != VMA_NULL)
    10350  {
    10351  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
    10352  }
    10353 #endif
    10354 
    10355  return res;
    10356 }
    10357 
    10358 void vmaDestroyPool(
    10359  VmaAllocator allocator,
    10360  VmaPool pool)
    10361 {
    10362  VMA_ASSERT(allocator);
    10363 
    10364  if(pool == VK_NULL_HANDLE)
    10365  {
    10366  return;
    10367  }
    10368 
    10369  VMA_DEBUG_LOG("vmaDestroyPool");
    10370 
    10371  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10372 
    10373 #if VMA_RECORDING_ENABLED
    10374  if(allocator->GetRecorder() != VMA_NULL)
    10375  {
    10376  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
    10377  }
    10378 #endif
    10379 
    10380  allocator->DestroyPool(pool);
    10381 }
    10382 
    10383 void vmaGetPoolStats(
    10384  VmaAllocator allocator,
    10385  VmaPool pool,
    10386  VmaPoolStats* pPoolStats)
    10387 {
    10388  VMA_ASSERT(allocator && pool && pPoolStats);
    10389 
    10390  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10391 
    10392  allocator->GetPoolStats(pool, pPoolStats);
    10393 }
    10394 
    10395 void vmaMakePoolAllocationsLost(
    10396  VmaAllocator allocator,
    10397  VmaPool pool,
    10398  size_t* pLostAllocationCount)
    10399 {
    10400  VMA_ASSERT(allocator && pool);
    10401 
    10402  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10403 
    10404  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
    10405 }
    10406 
    10407 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
    10408 {
    10409  VMA_ASSERT(allocator && pool);
    10410 
    10411  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10412 
    10413  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
    10414 
    10415  return allocator->CheckPoolCorruption(pool);
    10416 }
    10417 
    10418 VkResult vmaAllocateMemory(
    10419  VmaAllocator allocator,
    10420  const VkMemoryRequirements* pVkMemoryRequirements,
    10421  const VmaAllocationCreateInfo* pCreateInfo,
    10422  VmaAllocation* pAllocation,
    10423  VmaAllocationInfo* pAllocationInfo)
    10424 {
    10425  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
    10426 
    10427  VMA_DEBUG_LOG("vmaAllocateMemory");
    10428 
    10429  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10430 
    10431  VkResult result = allocator->AllocateMemory(
    10432  *pVkMemoryRequirements,
    10433  false, // requiresDedicatedAllocation
    10434  false, // prefersDedicatedAllocation
    10435  VK_NULL_HANDLE, // dedicatedBuffer
    10436  VK_NULL_HANDLE, // dedicatedImage
    10437  *pCreateInfo,
    10438  VMA_SUBALLOCATION_TYPE_UNKNOWN,
    10439  pAllocation);
    10440 
    10441 #if VMA_RECORDING_ENABLED
    10442  if(allocator->GetRecorder() != VMA_NULL)
    10443  {
    10444  allocator->GetRecorder()->RecordAllocateMemory(
    10445  allocator->GetCurrentFrameIndex(),
    10446  *pVkMemoryRequirements,
    10447  *pCreateInfo,
    10448  *pAllocation);
    10449  }
    10450 #endif
    10451 
    10452  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
    10453  {
    10454  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10455  }
    10456 
    10457  return result;
    10458 }
    10459 
    10460 VkResult vmaAllocateMemoryForBuffer(
    10461  VmaAllocator allocator,
    10462  VkBuffer buffer,
    10463  const VmaAllocationCreateInfo* pCreateInfo,
    10464  VmaAllocation* pAllocation,
    10465  VmaAllocationInfo* pAllocationInfo)
    10466 {
    10467  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10468 
    10469  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
    10470 
    10471  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10472 
    10473  VkMemoryRequirements vkMemReq = {};
    10474  bool requiresDedicatedAllocation = false;
    10475  bool prefersDedicatedAllocation = false;
    10476  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
    10477  requiresDedicatedAllocation,
    10478  prefersDedicatedAllocation);
    10479 
    10480  VkResult result = allocator->AllocateMemory(
    10481  vkMemReq,
    10482  requiresDedicatedAllocation,
    10483  prefersDedicatedAllocation,
    10484  buffer, // dedicatedBuffer
    10485  VK_NULL_HANDLE, // dedicatedImage
    10486  *pCreateInfo,
    10487  VMA_SUBALLOCATION_TYPE_BUFFER,
    10488  pAllocation);
    10489 
    10490 #if VMA_RECORDING_ENABLED
    10491  if(allocator->GetRecorder() != VMA_NULL)
    10492  {
    10493  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
    10494  allocator->GetCurrentFrameIndex(),
    10495  vkMemReq,
    10496  requiresDedicatedAllocation,
    10497  prefersDedicatedAllocation,
    10498  *pCreateInfo,
    10499  *pAllocation);
    10500  }
    10501 #endif
    10502 
    10503  if(pAllocationInfo && result == VK_SUCCESS)
    10504  {
    10505  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10506  }
    10507 
    10508  return result;
    10509 }
    10510 
    10511 VkResult vmaAllocateMemoryForImage(
    10512  VmaAllocator allocator,
    10513  VkImage image,
    10514  const VmaAllocationCreateInfo* pCreateInfo,
    10515  VmaAllocation* pAllocation,
    10516  VmaAllocationInfo* pAllocationInfo)
    10517 {
    10518  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10519 
    10520  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
    10521 
    10522  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10523 
    10524  VkMemoryRequirements vkMemReq = {};
    10525  bool requiresDedicatedAllocation = false;
    10526  bool prefersDedicatedAllocation = false;
    10527  allocator->GetImageMemoryRequirements(image, vkMemReq,
    10528  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10529 
    10530  VkResult result = allocator->AllocateMemory(
    10531  vkMemReq,
    10532  requiresDedicatedAllocation,
    10533  prefersDedicatedAllocation,
    10534  VK_NULL_HANDLE, // dedicatedBuffer
    10535  image, // dedicatedImage
    10536  *pCreateInfo,
    10537  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
    10538  pAllocation);
    10539 
    10540 #if VMA_RECORDING_ENABLED
    10541  if(allocator->GetRecorder() != VMA_NULL)
    10542  {
    10543  allocator->GetRecorder()->RecordAllocateMemoryForImage(
    10544  allocator->GetCurrentFrameIndex(),
    10545  vkMemReq,
    10546  requiresDedicatedAllocation,
    10547  prefersDedicatedAllocation,
    10548  *pCreateInfo,
    10549  *pAllocation);
    10550  }
    10551 #endif
    10552 
    10553  if(pAllocationInfo && result == VK_SUCCESS)
    10554  {
    10555  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10556  }
    10557 
    10558  return result;
    10559 }
    10560 
    10561 void vmaFreeMemory(
    10562  VmaAllocator allocator,
    10563  VmaAllocation allocation)
    10564 {
    10565  VMA_ASSERT(allocator);
    10566 
    10567  if(allocation == VK_NULL_HANDLE)
    10568  {
    10569  return;
    10570  }
    10571 
    10572  VMA_DEBUG_LOG("vmaFreeMemory");
    10573 
    10574  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10575 
    10576 #if VMA_RECORDING_ENABLED
    10577  if(allocator->GetRecorder() != VMA_NULL)
    10578  {
    10579  allocator->GetRecorder()->RecordFreeMemory(
    10580  allocator->GetCurrentFrameIndex(),
    10581  allocation);
    10582  }
    10583 #endif
    10584 
    10585  allocator->FreeMemory(allocation);
    10586 }
    10587 
    10588 void vmaGetAllocationInfo(
    10589  VmaAllocator allocator,
    10590  VmaAllocation allocation,
    10591  VmaAllocationInfo* pAllocationInfo)
    10592 {
    10593  VMA_ASSERT(allocator && allocation && pAllocationInfo);
    10594 
    10595  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10596 
    10597 #if VMA_RECORDING_ENABLED
    10598  if(allocator->GetRecorder() != VMA_NULL)
    10599  {
    10600  allocator->GetRecorder()->RecordGetAllocationInfo(
    10601  allocator->GetCurrentFrameIndex(),
    10602  allocation);
    10603  }
    10604 #endif
    10605 
    10606  allocator->GetAllocationInfo(allocation, pAllocationInfo);
    10607 }
    10608 
    10609 VkBool32 vmaTouchAllocation(
    10610  VmaAllocator allocator,
    10611  VmaAllocation allocation)
    10612 {
    10613  VMA_ASSERT(allocator && allocation);
    10614 
    10615  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10616 
    10617 #if VMA_RECORDING_ENABLED
    10618  if(allocator->GetRecorder() != VMA_NULL)
    10619  {
    10620  allocator->GetRecorder()->RecordTouchAllocation(
    10621  allocator->GetCurrentFrameIndex(),
    10622  allocation);
    10623  }
    10624 #endif
    10625 
    10626  return allocator->TouchAllocation(allocation);
    10627 }
    10628 
    10629 void vmaSetAllocationUserData(
    10630  VmaAllocator allocator,
    10631  VmaAllocation allocation,
    10632  void* pUserData)
    10633 {
    10634  VMA_ASSERT(allocator && allocation);
    10635 
    10636  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10637 
    10638  allocation->SetUserData(allocator, pUserData);
    10639 
    10640 #if VMA_RECORDING_ENABLED
    10641  if(allocator->GetRecorder() != VMA_NULL)
    10642  {
    10643  allocator->GetRecorder()->RecordSetAllocationUserData(
    10644  allocator->GetCurrentFrameIndex(),
    10645  allocation,
    10646  pUserData);
    10647  }
    10648 #endif
    10649 }
    10650 
    10651 void vmaCreateLostAllocation(
    10652  VmaAllocator allocator,
    10653  VmaAllocation* pAllocation)
    10654 {
    10655  VMA_ASSERT(allocator && pAllocation);
    10656 
    10657  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
    10658 
    10659  allocator->CreateLostAllocation(pAllocation);
    10660 
    10661 #if VMA_RECORDING_ENABLED
    10662  if(allocator->GetRecorder() != VMA_NULL)
    10663  {
    10664  allocator->GetRecorder()->RecordCreateLostAllocation(
    10665  allocator->GetCurrentFrameIndex(),
    10666  *pAllocation);
    10667  }
    10668 #endif
    10669 }
    10670 
    10671 VkResult vmaMapMemory(
    10672  VmaAllocator allocator,
    10673  VmaAllocation allocation,
    10674  void** ppData)
    10675 {
    10676  VMA_ASSERT(allocator && allocation && ppData);
    10677 
    10678  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10679 
    10680  VkResult res = allocator->Map(allocation, ppData);
    10681 
    10682 #if VMA_RECORDING_ENABLED
    10683  if(allocator->GetRecorder() != VMA_NULL)
    10684  {
    10685  allocator->GetRecorder()->RecordMapMemory(
    10686  allocator->GetCurrentFrameIndex(),
    10687  allocation);
    10688  }
    10689 #endif
    10690 
    10691  return res;
    10692 }
    10693 
    10694 void vmaUnmapMemory(
    10695  VmaAllocator allocator,
    10696  VmaAllocation allocation)
    10697 {
    10698  VMA_ASSERT(allocator && allocation);
    10699 
    10700  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10701 
    10702 #if VMA_RECORDING_ENABLED
    10703  if(allocator->GetRecorder() != VMA_NULL)
    10704  {
    10705  allocator->GetRecorder()->RecordUnmapMemory(
    10706  allocator->GetCurrentFrameIndex(),
    10707  allocation);
    10708  }
    10709 #endif
    10710 
    10711  allocator->Unmap(allocation);
    10712 }
    10713 
    10714 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10715 {
    10716  VMA_ASSERT(allocator && allocation);
    10717 
    10718  VMA_DEBUG_LOG("vmaFlushAllocation");
    10719 
    10720  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10721 
    10722  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
    10723 
    10724 #if VMA_RECORDING_ENABLED
    10725  if(allocator->GetRecorder() != VMA_NULL)
    10726  {
    10727  allocator->GetRecorder()->RecordFlushAllocation(
    10728  allocator->GetCurrentFrameIndex(),
    10729  allocation, offset, size);
    10730  }
    10731 #endif
    10732 }
    10733 
    10734 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10735 {
    10736  VMA_ASSERT(allocator && allocation);
    10737 
    10738  VMA_DEBUG_LOG("vmaInvalidateAllocation");
    10739 
    10740  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10741 
    10742  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
    10743 
    10744 #if VMA_RECORDING_ENABLED
    10745  if(allocator->GetRecorder() != VMA_NULL)
    10746  {
    10747  allocator->GetRecorder()->RecordInvalidateAllocation(
    10748  allocator->GetCurrentFrameIndex(),
    10749  allocation, offset, size);
    10750  }
    10751 #endif
    10752 }
    10753 
    10754 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
    10755 {
    10756  VMA_ASSERT(allocator);
    10757 
    10758  VMA_DEBUG_LOG("vmaCheckCorruption");
    10759 
    10760  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10761 
    10762  return allocator->CheckCorruption(memoryTypeBits);
    10763 }
    10764 
    10765 VkResult vmaDefragment(
    10766  VmaAllocator allocator,
    10767  VmaAllocation* pAllocations,
    10768  size_t allocationCount,
    10769  VkBool32* pAllocationsChanged,
    10770  const VmaDefragmentationInfo *pDefragmentationInfo,
    10771  VmaDefragmentationStats* pDefragmentationStats)
    10772 {
    10773  VMA_ASSERT(allocator && pAllocations);
    10774 
    10775  VMA_DEBUG_LOG("vmaDefragment");
    10776 
    10777  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10778 
    10779  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
    10780 }
    10781 
    10782 VkResult vmaBindBufferMemory(
    10783  VmaAllocator allocator,
    10784  VmaAllocation allocation,
    10785  VkBuffer buffer)
    10786 {
    10787  VMA_ASSERT(allocator && allocation && buffer);
    10788 
    10789  VMA_DEBUG_LOG("vmaBindBufferMemory");
    10790 
    10791  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10792 
    10793  return allocator->BindBufferMemory(allocation, buffer);
    10794 }
    10795 
    10796 VkResult vmaBindImageMemory(
    10797  VmaAllocator allocator,
    10798  VmaAllocation allocation,
    10799  VkImage image)
    10800 {
    10801  VMA_ASSERT(allocator && allocation && image);
    10802 
    10803  VMA_DEBUG_LOG("vmaBindImageMemory");
    10804 
    10805  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10806 
    10807  return allocator->BindImageMemory(allocation, image);
    10808 }
    10809 
    10810 VkResult vmaCreateBuffer(
    10811  VmaAllocator allocator,
    10812  const VkBufferCreateInfo* pBufferCreateInfo,
    10813  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10814  VkBuffer* pBuffer,
    10815  VmaAllocation* pAllocation,
    10816  VmaAllocationInfo* pAllocationInfo)
    10817 {
    10818  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
    10819 
    10820  VMA_DEBUG_LOG("vmaCreateBuffer");
    10821 
    10822  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10823 
    10824  *pBuffer = VK_NULL_HANDLE;
    10825  *pAllocation = VK_NULL_HANDLE;
    10826 
    10827  // 1. Create VkBuffer.
    10828  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
    10829  allocator->m_hDevice,
    10830  pBufferCreateInfo,
    10831  allocator->GetAllocationCallbacks(),
    10832  pBuffer);
    10833  if(res >= 0)
    10834  {
    10835  // 2. vkGetBufferMemoryRequirements.
    10836  VkMemoryRequirements vkMemReq = {};
    10837  bool requiresDedicatedAllocation = false;
    10838  bool prefersDedicatedAllocation = false;
    10839  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
    10840  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10841 
    10842  // Make sure alignment requirements for specific buffer usages reported
    10843  // in Physical Device Properties are included in alignment reported by memory requirements.
    10844  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
    10845  {
    10846  VMA_ASSERT(vkMemReq.alignment %
    10847  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
    10848  }
    10849  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
    10850  {
    10851  VMA_ASSERT(vkMemReq.alignment %
    10852  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
    10853  }
    10854  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
    10855  {
    10856  VMA_ASSERT(vkMemReq.alignment %
    10857  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
    10858  }
    10859 
    10860  // 3. Allocate memory using allocator.
    10861  res = allocator->AllocateMemory(
    10862  vkMemReq,
    10863  requiresDedicatedAllocation,
    10864  prefersDedicatedAllocation,
    10865  *pBuffer, // dedicatedBuffer
    10866  VK_NULL_HANDLE, // dedicatedImage
    10867  *pAllocationCreateInfo,
    10868  VMA_SUBALLOCATION_TYPE_BUFFER,
    10869  pAllocation);
    10870 
    10871 #if VMA_RECORDING_ENABLED
    10872  if(allocator->GetRecorder() != VMA_NULL)
    10873  {
    10874  allocator->GetRecorder()->RecordCreateBuffer(
    10875  allocator->GetCurrentFrameIndex(),
    10876  *pBufferCreateInfo,
    10877  *pAllocationCreateInfo,
    10878  *pAllocation);
    10879  }
    10880 #endif
    10881 
    10882  if(res >= 0)
    10883  {
    10884  // 3. Bind buffer with memory.
    10885  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
    10886  if(res >= 0)
    10887  {
    10888  // All steps succeeded.
    10889  #if VMA_STATS_STRING_ENABLED
    10890  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
    10891  #endif
    10892  if(pAllocationInfo != VMA_NULL)
    10893  {
    10894  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10895  }
    10896 
    10897  return VK_SUCCESS;
    10898  }
    10899  allocator->FreeMemory(*pAllocation);
    10900  *pAllocation = VK_NULL_HANDLE;
    10901  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10902  *pBuffer = VK_NULL_HANDLE;
    10903  return res;
    10904  }
    10905  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10906  *pBuffer = VK_NULL_HANDLE;
    10907  return res;
    10908  }
    10909  return res;
    10910 }
    10911 
    10912 void vmaDestroyBuffer(
    10913  VmaAllocator allocator,
    10914  VkBuffer buffer,
    10915  VmaAllocation allocation)
    10916 {
    10917  VMA_ASSERT(allocator);
    10918 
    10919  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    10920  {
    10921  return;
    10922  }
    10923 
    10924  VMA_DEBUG_LOG("vmaDestroyBuffer");
    10925 
    10926  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10927 
    10928 #if VMA_RECORDING_ENABLED
    10929  if(allocator->GetRecorder() != VMA_NULL)
    10930  {
    10931  allocator->GetRecorder()->RecordDestroyBuffer(
    10932  allocator->GetCurrentFrameIndex(),
    10933  allocation);
    10934  }
    10935 #endif
    10936 
    10937  if(buffer != VK_NULL_HANDLE)
    10938  {
    10939  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
    10940  }
    10941 
    10942  if(allocation != VK_NULL_HANDLE)
    10943  {
    10944  allocator->FreeMemory(allocation);
    10945  }
    10946 }
    10947 
    10948 VkResult vmaCreateImage(
    10949  VmaAllocator allocator,
    10950  const VkImageCreateInfo* pImageCreateInfo,
    10951  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10952  VkImage* pImage,
    10953  VmaAllocation* pAllocation,
    10954  VmaAllocationInfo* pAllocationInfo)
    10955 {
    10956  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
    10957 
    10958  VMA_DEBUG_LOG("vmaCreateImage");
    10959 
    10960  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10961 
    10962  *pImage = VK_NULL_HANDLE;
    10963  *pAllocation = VK_NULL_HANDLE;
    10964 
    10965  // 1. Create VkImage.
    10966  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
    10967  allocator->m_hDevice,
    10968  pImageCreateInfo,
    10969  allocator->GetAllocationCallbacks(),
    10970  pImage);
    10971  if(res >= 0)
    10972  {
    10973  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
    10974  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
    10975  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
    10976 
    10977  // 2. Allocate memory using allocator.
    10978  VkMemoryRequirements vkMemReq = {};
    10979  bool requiresDedicatedAllocation = false;
    10980  bool prefersDedicatedAllocation = false;
    10981  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
    10982  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10983 
    10984  res = allocator->AllocateMemory(
    10985  vkMemReq,
    10986  requiresDedicatedAllocation,
    10987  prefersDedicatedAllocation,
    10988  VK_NULL_HANDLE, // dedicatedBuffer
    10989  *pImage, // dedicatedImage
    10990  *pAllocationCreateInfo,
    10991  suballocType,
    10992  pAllocation);
    10993 
    10994 #if VMA_RECORDING_ENABLED
    10995  if(allocator->GetRecorder() != VMA_NULL)
    10996  {
    10997  allocator->GetRecorder()->RecordCreateImage(
    10998  allocator->GetCurrentFrameIndex(),
    10999  *pImageCreateInfo,
    11000  *pAllocationCreateInfo,
    11001  *pAllocation);
    11002  }
    11003 #endif
    11004 
    11005  if(res >= 0)
    11006  {
    11007  // 3. Bind image with memory.
    11008  res = allocator->BindImageMemory(*pAllocation, *pImage);
    11009  if(res >= 0)
    11010  {
    11011  // All steps succeeded.
    11012  #if VMA_STATS_STRING_ENABLED
    11013  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
    11014  #endif
    11015  if(pAllocationInfo != VMA_NULL)
    11016  {
    11017  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    11018  }
    11019 
    11020  return VK_SUCCESS;
    11021  }
    11022  allocator->FreeMemory(*pAllocation);
    11023  *pAllocation = VK_NULL_HANDLE;
    11024  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    11025  *pImage = VK_NULL_HANDLE;
    11026  return res;
    11027  }
    11028  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    11029  *pImage = VK_NULL_HANDLE;
    11030  return res;
    11031  }
    11032  return res;
    11033 }
    11034 
    11035 void vmaDestroyImage(
    11036  VmaAllocator allocator,
    11037  VkImage image,
    11038  VmaAllocation allocation)
    11039 {
    11040  VMA_ASSERT(allocator);
    11041 
    11042  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    11043  {
    11044  return;
    11045  }
    11046 
    11047  VMA_DEBUG_LOG("vmaDestroyImage");
    11048 
    11049  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    11050 
    11051 #if VMA_RECORDING_ENABLED
    11052  if(allocator->GetRecorder() != VMA_NULL)
    11053  {
    11054  allocator->GetRecorder()->RecordDestroyImage(
    11055  allocator->GetCurrentFrameIndex(),
    11056  allocation);
    11057  }
    11058 #endif
    11059 
    11060  if(image != VK_NULL_HANDLE)
    11061  {
    11062  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
    11063  }
    11064  if(allocation != VK_NULL_HANDLE)
    11065  {
    11066  allocator->FreeMemory(allocation);
    11067  }
    11068 }
    11069 
    11070 #endif // #ifdef VMA_IMPLEMENTATION
    VmaVulkanFunctions::vkGetPhysicalDeviceProperties
    PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
    Definition: vk_mem_alloc.h:1345
    +Go to the documentation of this file.
    1 //
    2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
    3 //
    4 // Permission is hereby granted, free of charge, to any person obtaining a copy
    5 // of this software and associated documentation files (the "Software"), to deal
    6 // in the Software without restriction, including without limitation the rights
    7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    8 // copies of the Software, and to permit persons to whom the Software is
    9 // furnished to do so, subject to the following conditions:
    10 //
    11 // The above copyright notice and this permission notice shall be included in
    12 // all copies or substantial portions of the Software.
    13 //
    14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    20 // THE SOFTWARE.
    21 //
    22 
    23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
    24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
    25 
    26 #ifdef __cplusplus
    27 extern "C" {
    28 #endif
    29 
    1258 #include <vulkan/vulkan.h>
    1259 
    1260 #if !defined(VMA_DEDICATED_ALLOCATION)
    1261  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
    1262  #define VMA_DEDICATED_ALLOCATION 1
    1263  #else
    1264  #define VMA_DEDICATED_ALLOCATION 0
    1265  #endif
    1266 #endif
    1267 
    1277 VK_DEFINE_HANDLE(VmaAllocator)
    1278 
    1279 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
    1281  VmaAllocator allocator,
    1282  uint32_t memoryType,
    1283  VkDeviceMemory memory,
    1284  VkDeviceSize size);
    1286 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
    1287  VmaAllocator allocator,
    1288  uint32_t memoryType,
    1289  VkDeviceMemory memory,
    1290  VkDeviceSize size);
    1291 
    1299 typedef struct VmaDeviceMemoryCallbacks {
    1301  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
    1303  PFN_vmaFreeDeviceMemoryFunction pfnFree;
    1304 } VmaDeviceMemoryCallbacks;
    1305 
    1307 typedef enum VmaAllocatorCreateFlagBits {
    1312  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
    1334  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
    1335 
    1336  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1337 } VmaAllocatorCreateFlagBits;
    1338 typedef VkFlags VmaAllocatorCreateFlags;
    1339 
    1344 typedef struct VmaVulkanFunctions {
    1345  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
    1346  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
    1347  PFN_vkAllocateMemory vkAllocateMemory;
    1348  PFN_vkFreeMemory vkFreeMemory;
    1349  PFN_vkMapMemory vkMapMemory;
    1350  PFN_vkUnmapMemory vkUnmapMemory;
    1351  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
    1352  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
    1353  PFN_vkBindBufferMemory vkBindBufferMemory;
    1354  PFN_vkBindImageMemory vkBindImageMemory;
    1355  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
    1356  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
    1357  PFN_vkCreateBuffer vkCreateBuffer;
    1358  PFN_vkDestroyBuffer vkDestroyBuffer;
    1359  PFN_vkCreateImage vkCreateImage;
    1360  PFN_vkDestroyImage vkDestroyImage;
    1361 #if VMA_DEDICATED_ALLOCATION
    1362  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
    1363  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
    1364 #endif
    1365 } VmaVulkanFunctions;
    1366 
    1368 typedef enum VmaRecordFlagBits {
    1374  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
    1375 
    1376  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1377 } VmaRecordFlagBits;
    1378 typedef VkFlags VmaRecordFlags;
    1379 
    1380 /*
    1381 Define this macro to 0/1 to disable/enable support for recording functionality,
    1382 available through VmaAllocatorCreateInfo::pRecordSettings.
    1383 */
    1384 #ifndef VMA_RECORDING_ENABLED
    1385  #ifdef _WIN32
    1386  #define VMA_RECORDING_ENABLED 1
    1387  #else
    1388  #define VMA_RECORDING_ENABLED 0
    1389  #endif
    1390 #endif
    1391 
    1393 typedef struct VmaRecordSettings
    1394 {
    1396  VmaRecordFlags flags;
    1404  const char* pFilePath;
    1405 } VmaRecordSettings;
    1406 
    1408 typedef struct VmaAllocatorCreateInfo
    1409 {
    1411  VmaAllocatorCreateFlags flags;
    1413 
    1414  VkPhysicalDevice physicalDevice;
    1416 
    1417  VkDevice device;
    1419 
    1420  VkDeviceSize preferredLargeHeapBlockSize;
    1422 
    1423  const VkAllocationCallbacks* pAllocationCallbacks;
    1425 
    1426  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
    1440  uint32_t frameInUseCount;
    1464  const VkDeviceSize* pHeapSizeLimit;
    1476  const VmaVulkanFunctions* pVulkanFunctions;
    1483  const VmaRecordSettings* pRecordSettings;
    1484 } VmaAllocatorCreateInfo;
    1485 
    1487 VkResult vmaCreateAllocator(
    1488  const VmaAllocatorCreateInfo* pCreateInfo,
    1489  VmaAllocator* pAllocator);
    1490 
    1492 void vmaDestroyAllocator(
    1493  VmaAllocator allocator);
    1494 
    1499 void vmaGetPhysicalDeviceProperties(
    1500  VmaAllocator allocator,
    1501  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
    1502 
    1507 void vmaGetMemoryProperties(
    1508  VmaAllocator allocator,
    1509  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
    1510 
    1517 void vmaGetMemoryTypeProperties(
    1518  VmaAllocator allocator,
    1519  uint32_t memoryTypeIndex,
    1520  VkMemoryPropertyFlags* pFlags);
    1521 
    1530 void vmaSetCurrentFrameIndex(
    1531  VmaAllocator allocator,
    1532  uint32_t frameIndex);
    1533 
    1536 typedef struct VmaStatInfo
    1537 {
    1539  uint32_t blockCount;
    1541  uint32_t allocationCount;
    1543  uint32_t unusedRangeCount;
    1545  VkDeviceSize usedBytes;
    1547  VkDeviceSize unusedBytes;
    1548  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
    1549  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
    1550 } VmaStatInfo;
    1551 
    1553 typedef struct VmaStats
    1554 {
    1555  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
    1556  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
    1557  VmaStatInfo total;
    1558 } VmaStats;
    1559 
    1561 void vmaCalculateStats(
    1562  VmaAllocator allocator,
    1563  VmaStats* pStats);
    1564 
    1565 #define VMA_STATS_STRING_ENABLED 1
    1566 
    1567 #if VMA_STATS_STRING_ENABLED
    1568 
    1570 
    1572 void vmaBuildStatsString(
    1573  VmaAllocator allocator,
    1574  char** ppStatsString,
    1575  VkBool32 detailedMap);
    1576 
    1577 void vmaFreeStatsString(
    1578  VmaAllocator allocator,
    1579  char* pStatsString);
    1580 
    1581 #endif // #if VMA_STATS_STRING_ENABLED
    1582 
    1591 VK_DEFINE_HANDLE(VmaPool)
    1592 
    1593 typedef enum VmaMemoryUsage
    1594 {
    1598  VMA_MEMORY_USAGE_UNKNOWN = 0,
    1615  VMA_MEMORY_USAGE_GPU_ONLY = 1,
    1625  VMA_MEMORY_USAGE_CPU_ONLY = 2,
    1632  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
    1641  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
    1642  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
    1643 } VmaMemoryUsage;
    1644 
    1646 typedef enum VmaAllocationCreateFlagBits {
    1658  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
    1659 
    1669  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
    1682  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
    1695  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
    1702  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
    1708  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
    1709 
    1710  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1711 } VmaAllocationCreateFlagBits;
    1712 typedef VkFlags VmaAllocationCreateFlags;
    1713 
    1714 typedef struct VmaAllocationCreateInfo
    1715 {
    1717  VmaAllocationCreateFlags flags;
    1723  VmaMemoryUsage usage;
    1728  VkMemoryPropertyFlags requiredFlags;
    1733  VkMemoryPropertyFlags preferredFlags;
    1741  uint32_t memoryTypeBits;
    1747  VmaPool pool;
    1754  void* pUserData;
    1755 } VmaAllocationCreateInfo;
    1756 
    1773 VkResult vmaFindMemoryTypeIndex(
    1774  VmaAllocator allocator,
    1775  uint32_t memoryTypeBits,
    1776  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1777  uint32_t* pMemoryTypeIndex);
    1778 
    1791 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    1792  VmaAllocator allocator,
    1793  const VkBufferCreateInfo* pBufferCreateInfo,
    1794  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1795  uint32_t* pMemoryTypeIndex);
    1796 
    1809 VkResult vmaFindMemoryTypeIndexForImageInfo(
    1810  VmaAllocator allocator,
    1811  const VkImageCreateInfo* pImageCreateInfo,
    1812  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    1813  uint32_t* pMemoryTypeIndex);
    1814 
    1816 typedef enum VmaPoolCreateFlagBits {
    1834  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
    1835 
    1836  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
    1837 } VmaPoolCreateFlagBits;
    1838 typedef VkFlags VmaPoolCreateFlags;
    1839 
    1842 typedef struct VmaPoolCreateInfo {
    1845  uint32_t memoryTypeIndex;
    1848  VmaPoolCreateFlags flags;
    1853  VkDeviceSize blockSize;
    1858  size_t minBlockCount;
    1866  size_t maxBlockCount;
    1880  uint32_t frameInUseCount;
    1881 } VmaPoolCreateInfo;
    1882 
    1885 typedef struct VmaPoolStats {
    1888  VkDeviceSize size;
    1891  VkDeviceSize unusedSize;
    1894  size_t allocationCount;
    1897  size_t unusedRangeCount;
    1904  VkDeviceSize unusedRangeSizeMax;
    1905 } VmaPoolStats;
    1906 
    1913 VkResult vmaCreatePool(
    1914  VmaAllocator allocator,
    1915  const VmaPoolCreateInfo* pCreateInfo,
    1916  VmaPool* pPool);
    1917 
    1920 void vmaDestroyPool(
    1921  VmaAllocator allocator,
    1922  VmaPool pool);
    1923 
    1930 void vmaGetPoolStats(
    1931  VmaAllocator allocator,
    1932  VmaPool pool,
    1933  VmaPoolStats* pPoolStats);
    1934 
    1941 void vmaMakePoolAllocationsLost(
    1942  VmaAllocator allocator,
    1943  VmaPool pool,
    1944  size_t* pLostAllocationCount);
    1945 
    1960 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
    1961 
    1986 VK_DEFINE_HANDLE(VmaAllocation)
    1987 
    1988 
    1990 typedef struct VmaAllocationInfo {
    1995  uint32_t memoryType;
    2004  VkDeviceMemory deviceMemory;
    2009  VkDeviceSize offset;
    2014  VkDeviceSize size;
    2023  void* pMappedData;
    2028  void* pUserData;
    2029 } VmaAllocationInfo;
    2030 
    2041 VkResult vmaAllocateMemory(
    2042  VmaAllocator allocator,
    2043  const VkMemoryRequirements* pVkMemoryRequirements,
    2044  const VmaAllocationCreateInfo* pCreateInfo,
    2045  VmaAllocation* pAllocation,
    2046  VmaAllocationInfo* pAllocationInfo);
    2047 
    2054 VkResult vmaAllocateMemoryForBuffer(
    2055  VmaAllocator allocator,
    2056  VkBuffer buffer,
    2057  const VmaAllocationCreateInfo* pCreateInfo,
    2058  VmaAllocation* pAllocation,
    2059  VmaAllocationInfo* pAllocationInfo);
    2060 
    2062 VkResult vmaAllocateMemoryForImage(
    2063  VmaAllocator allocator,
    2064  VkImage image,
    2065  const VmaAllocationCreateInfo* pCreateInfo,
    2066  VmaAllocation* pAllocation,
    2067  VmaAllocationInfo* pAllocationInfo);
    2068 
    2070 void vmaFreeMemory(
    2071  VmaAllocator allocator,
    2072  VmaAllocation allocation);
    2073 
    2090 void vmaGetAllocationInfo(
    2091  VmaAllocator allocator,
    2092  VmaAllocation allocation,
    2093  VmaAllocationInfo* pAllocationInfo);
    2094 
    2109 VkBool32 vmaTouchAllocation(
    2110  VmaAllocator allocator,
    2111  VmaAllocation allocation);
    2112 
    2126 void vmaSetAllocationUserData(
    2127  VmaAllocator allocator,
    2128  VmaAllocation allocation,
    2129  void* pUserData);
    2130 
    2141 void vmaCreateLostAllocation(
    2142  VmaAllocator allocator,
    2143  VmaAllocation* pAllocation);
    2144 
    2179 VkResult vmaMapMemory(
    2180  VmaAllocator allocator,
    2181  VmaAllocation allocation,
    2182  void** ppData);
    2183 
    2188 void vmaUnmapMemory(
    2189  VmaAllocator allocator,
    2190  VmaAllocation allocation);
    2191 
    2204 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2205 
    2218 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    2219 
    2236 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
    2237 
    2239 typedef struct VmaDefragmentationInfo {
    2244  VkDeviceSize maxBytesToMove;
    2249  uint32_t maxAllocationsToMove;
    2250 } VmaDefragmentationInfo;
    2251 
    2253 typedef struct VmaDefragmentationStats {
    2255  VkDeviceSize bytesMoved;
    2257  VkDeviceSize bytesFreed;
    2259  uint32_t allocationsMoved;
    2261  uint32_t deviceMemoryBlocksFreed;
    2262 } VmaDefragmentationStats;
    2263 
    2346 VkResult vmaDefragment(
    2347  VmaAllocator allocator,
    2348  VmaAllocation* pAllocations,
    2349  size_t allocationCount,
    2350  VkBool32* pAllocationsChanged,
    2351  const VmaDefragmentationInfo *pDefragmentationInfo,
    2352  VmaDefragmentationStats* pDefragmentationStats);
    2353 
    2366 VkResult vmaBindBufferMemory(
    2367  VmaAllocator allocator,
    2368  VmaAllocation allocation,
    2369  VkBuffer buffer);
    2370 
    2383 VkResult vmaBindImageMemory(
    2384  VmaAllocator allocator,
    2385  VmaAllocation allocation,
    2386  VkImage image);
    2387 
    2414 VkResult vmaCreateBuffer(
    2415  VmaAllocator allocator,
    2416  const VkBufferCreateInfo* pBufferCreateInfo,
    2417  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2418  VkBuffer* pBuffer,
    2419  VmaAllocation* pAllocation,
    2420  VmaAllocationInfo* pAllocationInfo);
    2421 
    2433 void vmaDestroyBuffer(
    2434  VmaAllocator allocator,
    2435  VkBuffer buffer,
    2436  VmaAllocation allocation);
    2437 
    2439 VkResult vmaCreateImage(
    2440  VmaAllocator allocator,
    2441  const VkImageCreateInfo* pImageCreateInfo,
    2442  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    2443  VkImage* pImage,
    2444  VmaAllocation* pAllocation,
    2445  VmaAllocationInfo* pAllocationInfo);
    2446 
    2458 void vmaDestroyImage(
    2459  VmaAllocator allocator,
    2460  VkImage image,
    2461  VmaAllocation allocation);
    2462 
    2463 #ifdef __cplusplus
    2464 }
    2465 #endif
    2466 
    2467 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
    2468 
    2469 // For Visual Studio IntelliSense.
    2470 #if defined(__cplusplus) && defined(__INTELLISENSE__)
    2471 #define VMA_IMPLEMENTATION
    2472 #endif
    2473 
    2474 #ifdef VMA_IMPLEMENTATION
    2475 #undef VMA_IMPLEMENTATION
    2476 
    2477 #include <cstdint>
    2478 #include <cstdlib>
    2479 #include <cstring>
    2480 
    2481 /*******************************************************************************
    2482 CONFIGURATION SECTION
    2483 
    2484 Define some of these macros before each #include of this header or change them
    2485 here if you need other then default behavior depending on your environment.
    2486 */
    2487 
    2488 /*
    2489 Define this macro to 1 to make the library fetch pointers to Vulkan functions
    2490 internally, like:
    2491 
    2492  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    2493 
    2494 Define to 0 if you are going to provide you own pointers to Vulkan functions via
    2495 VmaAllocatorCreateInfo::pVulkanFunctions.
    2496 */
    2497 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
    2498 #define VMA_STATIC_VULKAN_FUNCTIONS 1
    2499 #endif
    2500 
    2501 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
    2502 //#define VMA_USE_STL_CONTAINERS 1
    2503 
    2504 /* Set this macro to 1 to make the library including and using STL containers:
    2505 std::pair, std::vector, std::list, std::unordered_map.
    2506 
    2507 Set it to 0 or undefined to make the library using its own implementation of
    2508 the containers.
    2509 */
    2510 #if VMA_USE_STL_CONTAINERS
    2511  #define VMA_USE_STL_VECTOR 1
    2512  #define VMA_USE_STL_UNORDERED_MAP 1
    2513  #define VMA_USE_STL_LIST 1
    2514 #endif
    2515 
    2516 #if VMA_USE_STL_VECTOR
    2517  #include <vector>
    2518 #endif
    2519 
    2520 #if VMA_USE_STL_UNORDERED_MAP
    2521  #include <unordered_map>
    2522 #endif
    2523 
    2524 #if VMA_USE_STL_LIST
    2525  #include <list>
    2526 #endif
    2527 
    2528 /*
    2529 Following headers are used in this CONFIGURATION section only, so feel free to
    2530 remove them if not needed.
    2531 */
    2532 #include <cassert> // for assert
    2533 #include <algorithm> // for min, max
    2534 #include <mutex> // for std::mutex
    2535 #include <atomic> // for std::atomic
    2536 
    2537 #ifndef VMA_NULL
    2538  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
    2539  #define VMA_NULL nullptr
    2540 #endif
    2541 
    2542 #if defined(__APPLE__) || defined(__ANDROID__)
    2543 #include <cstdlib>
    2544 void *aligned_alloc(size_t alignment, size_t size)
    2545 {
    2546  // alignment must be >= sizeof(void*)
    2547  if(alignment < sizeof(void*))
    2548  {
    2549  alignment = sizeof(void*);
    2550  }
    2551 
    2552  void *pointer;
    2553  if(posix_memalign(&pointer, alignment, size) == 0)
    2554  return pointer;
    2555  return VMA_NULL;
    2556 }
    2557 #endif
    2558 
    2559 // If your compiler is not compatible with C++11 and definition of
    2560 // aligned_alloc() function is missing, uncommeting following line may help:
    2561 
    2562 //#include <malloc.h>
    2563 
    2564 // Normal assert to check for programmer's errors, especially in Debug configuration.
    2565 #ifndef VMA_ASSERT
    2566  #ifdef _DEBUG
    2567  #define VMA_ASSERT(expr) assert(expr)
    2568  #else
    2569  #define VMA_ASSERT(expr)
    2570  #endif
    2571 #endif
    2572 
    2573 // Assert that will be called very often, like inside data structures e.g. operator[].
    2574 // Making it non-empty can make program slow.
    2575 #ifndef VMA_HEAVY_ASSERT
    2576  #ifdef _DEBUG
    2577  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
    2578  #else
    2579  #define VMA_HEAVY_ASSERT(expr)
    2580  #endif
    2581 #endif
    2582 
    2583 #ifndef VMA_ALIGN_OF
    2584  #define VMA_ALIGN_OF(type) (__alignof(type))
    2585 #endif
    2586 
    2587 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
    2588  #if defined(_WIN32)
    2589  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
    2590  #else
    2591  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
    2592  #endif
    2593 #endif
    2594 
    2595 #ifndef VMA_SYSTEM_FREE
    2596  #if defined(_WIN32)
    2597  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
    2598  #else
    2599  #define VMA_SYSTEM_FREE(ptr) free(ptr)
    2600  #endif
    2601 #endif
    2602 
    2603 #ifndef VMA_MIN
    2604  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
    2605 #endif
    2606 
    2607 #ifndef VMA_MAX
    2608  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
    2609 #endif
    2610 
    2611 #ifndef VMA_SWAP
    2612  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
    2613 #endif
    2614 
    2615 #ifndef VMA_SORT
    2616  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
    2617 #endif
    2618 
    2619 #ifndef VMA_DEBUG_LOG
    2620  #define VMA_DEBUG_LOG(format, ...)
    2621  /*
    2622  #define VMA_DEBUG_LOG(format, ...) do { \
    2623  printf(format, __VA_ARGS__); \
    2624  printf("\n"); \
    2625  } while(false)
    2626  */
    2627 #endif
    2628 
    2629 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
    2630 #if VMA_STATS_STRING_ENABLED
    2631  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
    2632  {
    2633  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
    2634  }
    2635  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
    2636  {
    2637  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
    2638  }
    2639  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
    2640  {
    2641  snprintf(outStr, strLen, "%p", ptr);
    2642  }
    2643 #endif
    2644 
    2645 #ifndef VMA_MUTEX
    2646  class VmaMutex
    2647  {
    2648  public:
    2649  VmaMutex() { }
    2650  ~VmaMutex() { }
    2651  void Lock() { m_Mutex.lock(); }
    2652  void Unlock() { m_Mutex.unlock(); }
    2653  private:
    2654  std::mutex m_Mutex;
    2655  };
    2656  #define VMA_MUTEX VmaMutex
    2657 #endif
    2658 
    2659 /*
    2660 If providing your own implementation, you need to implement a subset of std::atomic:
    2661 
    2662 - Constructor(uint32_t desired)
    2663 - uint32_t load() const
    2664 - void store(uint32_t desired)
    2665 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
    2666 */
    2667 #ifndef VMA_ATOMIC_UINT32
    2668  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
    2669 #endif
    2670 
    2671 #ifndef VMA_BEST_FIT
    2672 
    2684  #define VMA_BEST_FIT (1)
    2685 #endif
    2686 
    2687 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
    2688 
    2692  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
    2693 #endif
    2694 
    2695 #ifndef VMA_DEBUG_ALIGNMENT
    2696 
    2700  #define VMA_DEBUG_ALIGNMENT (1)
    2701 #endif
    2702 
    2703 #ifndef VMA_DEBUG_MARGIN
    2704 
    2708  #define VMA_DEBUG_MARGIN (0)
    2709 #endif
    2710 
    2711 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
    2712 
    2716  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
    2717 #endif
    2718 
    2719 #ifndef VMA_DEBUG_DETECT_CORRUPTION
    2720 
    2725  #define VMA_DEBUG_DETECT_CORRUPTION (0)
    2726 #endif
    2727 
    2728 #ifndef VMA_DEBUG_GLOBAL_MUTEX
    2729 
    2733  #define VMA_DEBUG_GLOBAL_MUTEX (0)
    2734 #endif
    2735 
    2736 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
    2737 
    2741  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
    2742 #endif
    2743 
    2744 #ifndef VMA_SMALL_HEAP_MAX_SIZE
    2745  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
    2747 #endif
    2748 
    2749 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
    2750  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
    2752 #endif
    2753 
    2754 #ifndef VMA_CLASS_NO_COPY
    2755  #define VMA_CLASS_NO_COPY(className) \
    2756  private: \
    2757  className(const className&) = delete; \
    2758  className& operator=(const className&) = delete;
    2759 #endif
    2760 
    2761 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
    2762 
    2763 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
    2764 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
    2765 
    2766 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
    2767 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
    2768 
    2769 /*******************************************************************************
    2770 END OF CONFIGURATION
    2771 */
    2772 
    2773 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
    2774  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
    2775 
    2776 // Returns number of bits set to 1 in (v).
    2777 static inline uint32_t VmaCountBitsSet(uint32_t v)
    2778 {
    2779  uint32_t c = v - ((v >> 1) & 0x55555555);
    2780  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
    2781  c = ((c >> 4) + c) & 0x0F0F0F0F;
    2782  c = ((c >> 8) + c) & 0x00FF00FF;
    2783  c = ((c >> 16) + c) & 0x0000FFFF;
    2784  return c;
    2785 }
    2786 
    2787 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
    2788 // Use types like uint32_t, uint64_t as T.
    2789 template <typename T>
    2790 static inline T VmaAlignUp(T val, T align)
    2791 {
    2792  return (val + align - 1) / align * align;
    2793 }
    2794 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
    2795 // Use types like uint32_t, uint64_t as T.
    2796 template <typename T>
    2797 static inline T VmaAlignDown(T val, T align)
    2798 {
    2799  return val / align * align;
    2800 }
    2801 
    2802 // Division with mathematical rounding to nearest number.
    2803 template <typename T>
    2804 inline T VmaRoundDiv(T x, T y)
    2805 {
    2806  return (x + (y / (T)2)) / y;
    2807 }
    2808 
    2809 static inline bool VmaStrIsEmpty(const char* pStr)
    2810 {
    2811  return pStr == VMA_NULL || *pStr == '\0';
    2812 }
    2813 
    2814 #ifndef VMA_SORT
    2815 
    2816 template<typename Iterator, typename Compare>
    2817 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
    2818 {
    2819  Iterator centerValue = end; --centerValue;
    2820  Iterator insertIndex = beg;
    2821  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
    2822  {
    2823  if(cmp(*memTypeIndex, *centerValue))
    2824  {
    2825  if(insertIndex != memTypeIndex)
    2826  {
    2827  VMA_SWAP(*memTypeIndex, *insertIndex);
    2828  }
    2829  ++insertIndex;
    2830  }
    2831  }
    2832  if(insertIndex != centerValue)
    2833  {
    2834  VMA_SWAP(*insertIndex, *centerValue);
    2835  }
    2836  return insertIndex;
    2837 }
    2838 
    2839 template<typename Iterator, typename Compare>
    2840 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
    2841 {
    2842  if(beg < end)
    2843  {
    2844  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
    2845  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
    2846  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
    2847  }
    2848 }
    2849 
    2850 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
    2851 
    2852 #endif // #ifndef VMA_SORT
    2853 
    2854 /*
    2855 Returns true if two memory blocks occupy overlapping pages.
    2856 ResourceA must be in less memory offset than ResourceB.
    2857 
    2858 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
    2859 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
    2860 */
    2861 static inline bool VmaBlocksOnSamePage(
    2862  VkDeviceSize resourceAOffset,
    2863  VkDeviceSize resourceASize,
    2864  VkDeviceSize resourceBOffset,
    2865  VkDeviceSize pageSize)
    2866 {
    2867  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
    2868  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
    2869  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
    2870  VkDeviceSize resourceBStart = resourceBOffset;
    2871  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
    2872  return resourceAEndPage == resourceBStartPage;
    2873 }
    2874 
    2875 enum VmaSuballocationType
    2876 {
    2877  VMA_SUBALLOCATION_TYPE_FREE = 0,
    2878  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
    2879  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
    2880  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
    2881  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
    2882  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
    2883  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
    2884 };
    2885 
    2886 /*
    2887 Returns true if given suballocation types could conflict and must respect
    2888 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
    2889 or linear image and another one is optimal image. If type is unknown, behave
    2890 conservatively.
    2891 */
    2892 static inline bool VmaIsBufferImageGranularityConflict(
    2893  VmaSuballocationType suballocType1,
    2894  VmaSuballocationType suballocType2)
    2895 {
    2896  if(suballocType1 > suballocType2)
    2897  {
    2898  VMA_SWAP(suballocType1, suballocType2);
    2899  }
    2900 
    2901  switch(suballocType1)
    2902  {
    2903  case VMA_SUBALLOCATION_TYPE_FREE:
    2904  return false;
    2905  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
    2906  return true;
    2907  case VMA_SUBALLOCATION_TYPE_BUFFER:
    2908  return
    2909  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2910  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2911  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
    2912  return
    2913  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
    2914  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
    2915  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2916  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
    2917  return
    2918  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
    2919  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
    2920  return false;
    2921  default:
    2922  VMA_ASSERT(0);
    2923  return true;
    2924  }
    2925 }
    2926 
    2927 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
    2928 {
    2929  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
    2930  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2931  for(size_t i = 0; i < numberCount; ++i, ++pDst)
    2932  {
    2933  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
    2934  }
    2935 }
    2936 
    2937 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
    2938 {
    2939  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
    2940  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
    2941  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
    2942  {
    2943  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
    2944  {
    2945  return false;
    2946  }
    2947  }
    2948  return true;
    2949 }
    2950 
    2951 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
    2952 struct VmaMutexLock
    2953 {
    2954  VMA_CLASS_NO_COPY(VmaMutexLock)
    2955 public:
    2956  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
    2957  m_pMutex(useMutex ? &mutex : VMA_NULL)
    2958  {
    2959  if(m_pMutex)
    2960  {
    2961  m_pMutex->Lock();
    2962  }
    2963  }
    2964 
    2965  ~VmaMutexLock()
    2966  {
    2967  if(m_pMutex)
    2968  {
    2969  m_pMutex->Unlock();
    2970  }
    2971  }
    2972 
    2973 private:
    2974  VMA_MUTEX* m_pMutex;
    2975 };
    2976 
    2977 #if VMA_DEBUG_GLOBAL_MUTEX
    2978  static VMA_MUTEX gDebugGlobalMutex;
    2979  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
    2980 #else
    2981  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
    2982 #endif
    2983 
    2984 // Minimum size of a free suballocation to register it in the free suballocation collection.
    2985 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
    2986 
    2987 /*
    2988 Performs binary search and returns iterator to first element that is greater or
    2989 equal to (key), according to comparison (cmp).
    2990 
    2991 Cmp should return true if first argument is less than second argument.
    2992 
    2993 Returned value is the found element, if present in the collection or place where
    2994 new element with value (key) should be inserted.
    2995 */
    2996 template <typename IterT, typename KeyT, typename CmpT>
    2997 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
    2998 {
    2999  size_t down = 0, up = (end - beg);
    3000  while(down < up)
    3001  {
    3002  const size_t mid = (down + up) / 2;
    3003  if(cmp(*(beg+mid), key))
    3004  {
    3005  down = mid + 1;
    3006  }
    3007  else
    3008  {
    3009  up = mid;
    3010  }
    3011  }
    3012  return beg + down;
    3013 }
    3014 
    3016 // Memory allocation
    3017 
    3018 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
    3019 {
    3020  if((pAllocationCallbacks != VMA_NULL) &&
    3021  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
    3022  {
    3023  return (*pAllocationCallbacks->pfnAllocation)(
    3024  pAllocationCallbacks->pUserData,
    3025  size,
    3026  alignment,
    3027  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    3028  }
    3029  else
    3030  {
    3031  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
    3032  }
    3033 }
    3034 
    3035 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
    3036 {
    3037  if((pAllocationCallbacks != VMA_NULL) &&
    3038  (pAllocationCallbacks->pfnFree != VMA_NULL))
    3039  {
    3040  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
    3041  }
    3042  else
    3043  {
    3044  VMA_SYSTEM_FREE(ptr);
    3045  }
    3046 }
    3047 
    3048 template<typename T>
    3049 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
    3050 {
    3051  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
    3052 }
    3053 
    3054 template<typename T>
    3055 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
    3056 {
    3057  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
    3058 }
    3059 
    3060 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
    3061 
    3062 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
    3063 
    3064 template<typename T>
    3065 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
    3066 {
    3067  ptr->~T();
    3068  VmaFree(pAllocationCallbacks, ptr);
    3069 }
    3070 
    3071 template<typename T>
    3072 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
    3073 {
    3074  if(ptr != VMA_NULL)
    3075  {
    3076  for(size_t i = count; i--; )
    3077  {
    3078  ptr[i].~T();
    3079  }
    3080  VmaFree(pAllocationCallbacks, ptr);
    3081  }
    3082 }
    3083 
    3084 // STL-compatible allocator.
    3085 template<typename T>
    3086 class VmaStlAllocator
    3087 {
    3088 public:
    3089  const VkAllocationCallbacks* const m_pCallbacks;
    3090  typedef T value_type;
    3091 
    3092  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
    3093  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
    3094 
    3095  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
    3096  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
    3097 
    3098  template<typename U>
    3099  bool operator==(const VmaStlAllocator<U>& rhs) const
    3100  {
    3101  return m_pCallbacks == rhs.m_pCallbacks;
    3102  }
    3103  template<typename U>
    3104  bool operator!=(const VmaStlAllocator<U>& rhs) const
    3105  {
    3106  return m_pCallbacks != rhs.m_pCallbacks;
    3107  }
    3108 
    3109  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
    3110 };
    3111 
    3112 #if VMA_USE_STL_VECTOR
    3113 
    3114 #define VmaVector std::vector
    3115 
    3116 template<typename T, typename allocatorT>
    3117 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
    3118 {
    3119  vec.insert(vec.begin() + index, item);
    3120 }
    3121 
    3122 template<typename T, typename allocatorT>
    3123 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
    3124 {
    3125  vec.erase(vec.begin() + index);
    3126 }
    3127 
    3128 #else // #if VMA_USE_STL_VECTOR
    3129 
    3130 /* Class with interface compatible with subset of std::vector.
    3131 T must be POD because constructors and destructors are not called and memcpy is
    3132 used for these objects. */
    3133 template<typename T, typename AllocatorT>
    3134 class VmaVector
    3135 {
    3136 public:
    3137  typedef T value_type;
    3138 
    3139  VmaVector(const AllocatorT& allocator) :
    3140  m_Allocator(allocator),
    3141  m_pArray(VMA_NULL),
    3142  m_Count(0),
    3143  m_Capacity(0)
    3144  {
    3145  }
    3146 
    3147  VmaVector(size_t count, const AllocatorT& allocator) :
    3148  m_Allocator(allocator),
    3149  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
    3150  m_Count(count),
    3151  m_Capacity(count)
    3152  {
    3153  }
    3154 
    3155  VmaVector(const VmaVector<T, AllocatorT>& src) :
    3156  m_Allocator(src.m_Allocator),
    3157  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
    3158  m_Count(src.m_Count),
    3159  m_Capacity(src.m_Count)
    3160  {
    3161  if(m_Count != 0)
    3162  {
    3163  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
    3164  }
    3165  }
    3166 
    3167  ~VmaVector()
    3168  {
    3169  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3170  }
    3171 
    3172  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
    3173  {
    3174  if(&rhs != this)
    3175  {
    3176  resize(rhs.m_Count);
    3177  if(m_Count != 0)
    3178  {
    3179  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
    3180  }
    3181  }
    3182  return *this;
    3183  }
    3184 
    3185  bool empty() const { return m_Count == 0; }
    3186  size_t size() const { return m_Count; }
    3187  T* data() { return m_pArray; }
    3188  const T* data() const { return m_pArray; }
    3189 
    3190  T& operator[](size_t index)
    3191  {
    3192  VMA_HEAVY_ASSERT(index < m_Count);
    3193  return m_pArray[index];
    3194  }
    3195  const T& operator[](size_t index) const
    3196  {
    3197  VMA_HEAVY_ASSERT(index < m_Count);
    3198  return m_pArray[index];
    3199  }
    3200 
    3201  T& front()
    3202  {
    3203  VMA_HEAVY_ASSERT(m_Count > 0);
    3204  return m_pArray[0];
    3205  }
    3206  const T& front() const
    3207  {
    3208  VMA_HEAVY_ASSERT(m_Count > 0);
    3209  return m_pArray[0];
    3210  }
    3211  T& back()
    3212  {
    3213  VMA_HEAVY_ASSERT(m_Count > 0);
    3214  return m_pArray[m_Count - 1];
    3215  }
    3216  const T& back() const
    3217  {
    3218  VMA_HEAVY_ASSERT(m_Count > 0);
    3219  return m_pArray[m_Count - 1];
    3220  }
    3221 
    3222  void reserve(size_t newCapacity, bool freeMemory = false)
    3223  {
    3224  newCapacity = VMA_MAX(newCapacity, m_Count);
    3225 
    3226  if((newCapacity < m_Capacity) && !freeMemory)
    3227  {
    3228  newCapacity = m_Capacity;
    3229  }
    3230 
    3231  if(newCapacity != m_Capacity)
    3232  {
    3233  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
    3234  if(m_Count != 0)
    3235  {
    3236  memcpy(newArray, m_pArray, m_Count * sizeof(T));
    3237  }
    3238  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3239  m_Capacity = newCapacity;
    3240  m_pArray = newArray;
    3241  }
    3242  }
    3243 
    3244  void resize(size_t newCount, bool freeMemory = false)
    3245  {
    3246  size_t newCapacity = m_Capacity;
    3247  if(newCount > m_Capacity)
    3248  {
    3249  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
    3250  }
    3251  else if(freeMemory)
    3252  {
    3253  newCapacity = newCount;
    3254  }
    3255 
    3256  if(newCapacity != m_Capacity)
    3257  {
    3258  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
    3259  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
    3260  if(elementsToCopy != 0)
    3261  {
    3262  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
    3263  }
    3264  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
    3265  m_Capacity = newCapacity;
    3266  m_pArray = newArray;
    3267  }
    3268 
    3269  m_Count = newCount;
    3270  }
    3271 
    3272  void clear(bool freeMemory = false)
    3273  {
    3274  resize(0, freeMemory);
    3275  }
    3276 
    3277  void insert(size_t index, const T& src)
    3278  {
    3279  VMA_HEAVY_ASSERT(index <= m_Count);
    3280  const size_t oldCount = size();
    3281  resize(oldCount + 1);
    3282  if(index < oldCount)
    3283  {
    3284  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
    3285  }
    3286  m_pArray[index] = src;
    3287  }
    3288 
    3289  void remove(size_t index)
    3290  {
    3291  VMA_HEAVY_ASSERT(index < m_Count);
    3292  const size_t oldCount = size();
    3293  if(index < oldCount - 1)
    3294  {
    3295  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
    3296  }
    3297  resize(oldCount - 1);
    3298  }
    3299 
    3300  void push_back(const T& src)
    3301  {
    3302  const size_t newIndex = size();
    3303  resize(newIndex + 1);
    3304  m_pArray[newIndex] = src;
    3305  }
    3306 
    3307  void pop_back()
    3308  {
    3309  VMA_HEAVY_ASSERT(m_Count > 0);
    3310  resize(size() - 1);
    3311  }
    3312 
    3313  void push_front(const T& src)
    3314  {
    3315  insert(0, src);
    3316  }
    3317 
    3318  void pop_front()
    3319  {
    3320  VMA_HEAVY_ASSERT(m_Count > 0);
    3321  remove(0);
    3322  }
    3323 
    3324  typedef T* iterator;
    3325 
    3326  iterator begin() { return m_pArray; }
    3327  iterator end() { return m_pArray + m_Count; }
    3328 
    3329 private:
    3330  AllocatorT m_Allocator;
    3331  T* m_pArray;
    3332  size_t m_Count;
    3333  size_t m_Capacity;
    3334 };
    3335 
    3336 template<typename T, typename allocatorT>
    3337 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
    3338 {
    3339  vec.insert(index, item);
    3340 }
    3341 
    3342 template<typename T, typename allocatorT>
    3343 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
    3344 {
    3345  vec.remove(index);
    3346 }
    3347 
    3348 #endif // #if VMA_USE_STL_VECTOR
    3349 
    3350 template<typename CmpLess, typename VectorT>
    3351 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
    3352 {
    3353  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    3354  vector.data(),
    3355  vector.data() + vector.size(),
    3356  value,
    3357  CmpLess()) - vector.data();
    3358  VmaVectorInsert(vector, indexToInsert, value);
    3359  return indexToInsert;
    3360 }
    3361 
    3362 template<typename CmpLess, typename VectorT>
    3363 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
    3364 {
    3365  CmpLess comparator;
    3366  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3367  vector.begin(),
    3368  vector.end(),
    3369  value,
    3370  comparator);
    3371  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
    3372  {
    3373  size_t indexToRemove = it - vector.begin();
    3374  VmaVectorRemove(vector, indexToRemove);
    3375  return true;
    3376  }
    3377  return false;
    3378 }
    3379 
    3380 template<typename CmpLess, typename VectorT>
    3381 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
    3382 {
    3383  CmpLess comparator;
    3384  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
    3385  vector.data(),
    3386  vector.data() + vector.size(),
    3387  value,
    3388  comparator);
    3389  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
    3390  {
    3391  return it - vector.begin();
    3392  }
    3393  else
    3394  {
    3395  return vector.size();
    3396  }
    3397 }
    3398 
    3400 // class VmaPoolAllocator
    3401 
    3402 /*
    3403 Allocator for objects of type T using a list of arrays (pools) to speed up
    3404 allocation. Number of elements that can be allocated is not bounded because
    3405 allocator can create multiple blocks.
    3406 */
    3407 template<typename T>
    3408 class VmaPoolAllocator
    3409 {
    3410  VMA_CLASS_NO_COPY(VmaPoolAllocator)
    3411 public:
    3412  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
    3413  ~VmaPoolAllocator();
    3414  void Clear();
    3415  T* Alloc();
    3416  void Free(T* ptr);
    3417 
    3418 private:
    3419  union Item
    3420  {
    3421  uint32_t NextFreeIndex;
    3422  T Value;
    3423  };
    3424 
    3425  struct ItemBlock
    3426  {
    3427  Item* pItems;
    3428  uint32_t FirstFreeIndex;
    3429  };
    3430 
    3431  const VkAllocationCallbacks* m_pAllocationCallbacks;
    3432  size_t m_ItemsPerBlock;
    3433  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
    3434 
    3435  ItemBlock& CreateNewBlock();
    3436 };
    3437 
    3438 template<typename T>
    3439 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
    3440  m_pAllocationCallbacks(pAllocationCallbacks),
    3441  m_ItemsPerBlock(itemsPerBlock),
    3442  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
    3443 {
    3444  VMA_ASSERT(itemsPerBlock > 0);
    3445 }
    3446 
    3447 template<typename T>
    3448 VmaPoolAllocator<T>::~VmaPoolAllocator()
    3449 {
    3450  Clear();
    3451 }
    3452 
    3453 template<typename T>
    3454 void VmaPoolAllocator<T>::Clear()
    3455 {
    3456  for(size_t i = m_ItemBlocks.size(); i--; )
    3457  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
    3458  m_ItemBlocks.clear();
    3459 }
    3460 
    3461 template<typename T>
    3462 T* VmaPoolAllocator<T>::Alloc()
    3463 {
    3464  for(size_t i = m_ItemBlocks.size(); i--; )
    3465  {
    3466  ItemBlock& block = m_ItemBlocks[i];
    3467  // This block has some free items: Use first one.
    3468  if(block.FirstFreeIndex != UINT32_MAX)
    3469  {
    3470  Item* const pItem = &block.pItems[block.FirstFreeIndex];
    3471  block.FirstFreeIndex = pItem->NextFreeIndex;
    3472  return &pItem->Value;
    3473  }
    3474  }
    3475 
    3476  // No block has free item: Create new one and use it.
    3477  ItemBlock& newBlock = CreateNewBlock();
    3478  Item* const pItem = &newBlock.pItems[0];
    3479  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
    3480  return &pItem->Value;
    3481 }
    3482 
    3483 template<typename T>
    3484 void VmaPoolAllocator<T>::Free(T* ptr)
    3485 {
    3486  // Search all memory blocks to find ptr.
    3487  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
    3488  {
    3489  ItemBlock& block = m_ItemBlocks[i];
    3490 
    3491  // Casting to union.
    3492  Item* pItemPtr;
    3493  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
    3494 
    3495  // Check if pItemPtr is in address range of this block.
    3496  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
    3497  {
    3498  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
    3499  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
    3500  block.FirstFreeIndex = index;
    3501  return;
    3502  }
    3503  }
    3504  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
    3505 }
    3506 
    3507 template<typename T>
    3508 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
    3509 {
    3510  ItemBlock newBlock = {
    3511  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
    3512 
    3513  m_ItemBlocks.push_back(newBlock);
    3514 
    3515  // Setup singly-linked list of all free items in this block.
    3516  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
    3517  newBlock.pItems[i].NextFreeIndex = i + 1;
    3518  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
    3519  return m_ItemBlocks.back();
    3520 }
    3521 
    3523 // class VmaRawList, VmaList
    3524 
    3525 #if VMA_USE_STL_LIST
    3526 
    3527 #define VmaList std::list
    3528 
    3529 #else // #if VMA_USE_STL_LIST
    3530 
    3531 template<typename T>
    3532 struct VmaListItem
    3533 {
    3534  VmaListItem* pPrev;
    3535  VmaListItem* pNext;
    3536  T Value;
    3537 };
    3538 
    3539 // Doubly linked list.
    3540 template<typename T>
    3541 class VmaRawList
    3542 {
    3543  VMA_CLASS_NO_COPY(VmaRawList)
    3544 public:
    3545  typedef VmaListItem<T> ItemType;
    3546 
    3547  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
    3548  ~VmaRawList();
    3549  void Clear();
    3550 
    3551  size_t GetCount() const { return m_Count; }
    3552  bool IsEmpty() const { return m_Count == 0; }
    3553 
    3554  ItemType* Front() { return m_pFront; }
    3555  const ItemType* Front() const { return m_pFront; }
    3556  ItemType* Back() { return m_pBack; }
    3557  const ItemType* Back() const { return m_pBack; }
    3558 
    3559  ItemType* PushBack();
    3560  ItemType* PushFront();
    3561  ItemType* PushBack(const T& value);
    3562  ItemType* PushFront(const T& value);
    3563  void PopBack();
    3564  void PopFront();
    3565 
    3566  // Item can be null - it means PushBack.
    3567  ItemType* InsertBefore(ItemType* pItem);
    3568  // Item can be null - it means PushFront.
    3569  ItemType* InsertAfter(ItemType* pItem);
    3570 
    3571  ItemType* InsertBefore(ItemType* pItem, const T& value);
    3572  ItemType* InsertAfter(ItemType* pItem, const T& value);
    3573 
    3574  void Remove(ItemType* pItem);
    3575 
    3576 private:
    3577  const VkAllocationCallbacks* const m_pAllocationCallbacks;
    3578  VmaPoolAllocator<ItemType> m_ItemAllocator;
    3579  ItemType* m_pFront;
    3580  ItemType* m_pBack;
    3581  size_t m_Count;
    3582 };
    3583 
    3584 template<typename T>
    3585 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
    3586  m_pAllocationCallbacks(pAllocationCallbacks),
    3587  m_ItemAllocator(pAllocationCallbacks, 128),
    3588  m_pFront(VMA_NULL),
    3589  m_pBack(VMA_NULL),
    3590  m_Count(0)
    3591 {
    3592 }
    3593 
    3594 template<typename T>
    3595 VmaRawList<T>::~VmaRawList()
    3596 {
    3597  // Intentionally not calling Clear, because that would be unnecessary
    3598  // computations to return all items to m_ItemAllocator as free.
    3599 }
    3600 
    3601 template<typename T>
    3602 void VmaRawList<T>::Clear()
    3603 {
    3604  if(IsEmpty() == false)
    3605  {
    3606  ItemType* pItem = m_pBack;
    3607  while(pItem != VMA_NULL)
    3608  {
    3609  ItemType* const pPrevItem = pItem->pPrev;
    3610  m_ItemAllocator.Free(pItem);
    3611  pItem = pPrevItem;
    3612  }
    3613  m_pFront = VMA_NULL;
    3614  m_pBack = VMA_NULL;
    3615  m_Count = 0;
    3616  }
    3617 }
    3618 
    3619 template<typename T>
    3620 VmaListItem<T>* VmaRawList<T>::PushBack()
    3621 {
    3622  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3623  pNewItem->pNext = VMA_NULL;
    3624  if(IsEmpty())
    3625  {
    3626  pNewItem->pPrev = VMA_NULL;
    3627  m_pFront = pNewItem;
    3628  m_pBack = pNewItem;
    3629  m_Count = 1;
    3630  }
    3631  else
    3632  {
    3633  pNewItem->pPrev = m_pBack;
    3634  m_pBack->pNext = pNewItem;
    3635  m_pBack = pNewItem;
    3636  ++m_Count;
    3637  }
    3638  return pNewItem;
    3639 }
    3640 
    3641 template<typename T>
    3642 VmaListItem<T>* VmaRawList<T>::PushFront()
    3643 {
    3644  ItemType* const pNewItem = m_ItemAllocator.Alloc();
    3645  pNewItem->pPrev = VMA_NULL;
    3646  if(IsEmpty())
    3647  {
    3648  pNewItem->pNext = VMA_NULL;
    3649  m_pFront = pNewItem;
    3650  m_pBack = pNewItem;
    3651  m_Count = 1;
    3652  }
    3653  else
    3654  {
    3655  pNewItem->pNext = m_pFront;
    3656  m_pFront->pPrev = pNewItem;
    3657  m_pFront = pNewItem;
    3658  ++m_Count;
    3659  }
    3660  return pNewItem;
    3661 }
    3662 
    3663 template<typename T>
    3664 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
    3665 {
    3666  ItemType* const pNewItem = PushBack();
    3667  pNewItem->Value = value;
    3668  return pNewItem;
    3669 }
    3670 
    3671 template<typename T>
    3672 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
    3673 {
    3674  ItemType* const pNewItem = PushFront();
    3675  pNewItem->Value = value;
    3676  return pNewItem;
    3677 }
    3678 
    3679 template<typename T>
    3680 void VmaRawList<T>::PopBack()
    3681 {
    3682  VMA_HEAVY_ASSERT(m_Count > 0);
    3683  ItemType* const pBackItem = m_pBack;
    3684  ItemType* const pPrevItem = pBackItem->pPrev;
    3685  if(pPrevItem != VMA_NULL)
    3686  {
    3687  pPrevItem->pNext = VMA_NULL;
    3688  }
    3689  m_pBack = pPrevItem;
    3690  m_ItemAllocator.Free(pBackItem);
    3691  --m_Count;
    3692 }
    3693 
    3694 template<typename T>
    3695 void VmaRawList<T>::PopFront()
    3696 {
    3697  VMA_HEAVY_ASSERT(m_Count > 0);
    3698  ItemType* const pFrontItem = m_pFront;
    3699  ItemType* const pNextItem = pFrontItem->pNext;
    3700  if(pNextItem != VMA_NULL)
    3701  {
    3702  pNextItem->pPrev = VMA_NULL;
    3703  }
    3704  m_pFront = pNextItem;
    3705  m_ItemAllocator.Free(pFrontItem);
    3706  --m_Count;
    3707 }
    3708 
    3709 template<typename T>
    3710 void VmaRawList<T>::Remove(ItemType* pItem)
    3711 {
    3712  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
    3713  VMA_HEAVY_ASSERT(m_Count > 0);
    3714 
    3715  if(pItem->pPrev != VMA_NULL)
    3716  {
    3717  pItem->pPrev->pNext = pItem->pNext;
    3718  }
    3719  else
    3720  {
    3721  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3722  m_pFront = pItem->pNext;
    3723  }
    3724 
    3725  if(pItem->pNext != VMA_NULL)
    3726  {
    3727  pItem->pNext->pPrev = pItem->pPrev;
    3728  }
    3729  else
    3730  {
    3731  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3732  m_pBack = pItem->pPrev;
    3733  }
    3734 
    3735  m_ItemAllocator.Free(pItem);
    3736  --m_Count;
    3737 }
    3738 
    3739 template<typename T>
    3740 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
    3741 {
    3742  if(pItem != VMA_NULL)
    3743  {
    3744  ItemType* const prevItem = pItem->pPrev;
    3745  ItemType* const newItem = m_ItemAllocator.Alloc();
    3746  newItem->pPrev = prevItem;
    3747  newItem->pNext = pItem;
    3748  pItem->pPrev = newItem;
    3749  if(prevItem != VMA_NULL)
    3750  {
    3751  prevItem->pNext = newItem;
    3752  }
    3753  else
    3754  {
    3755  VMA_HEAVY_ASSERT(m_pFront == pItem);
    3756  m_pFront = newItem;
    3757  }
    3758  ++m_Count;
    3759  return newItem;
    3760  }
    3761  else
    3762  return PushBack();
    3763 }
    3764 
    3765 template<typename T>
    3766 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
    3767 {
    3768  if(pItem != VMA_NULL)
    3769  {
    3770  ItemType* const nextItem = pItem->pNext;
    3771  ItemType* const newItem = m_ItemAllocator.Alloc();
    3772  newItem->pNext = nextItem;
    3773  newItem->pPrev = pItem;
    3774  pItem->pNext = newItem;
    3775  if(nextItem != VMA_NULL)
    3776  {
    3777  nextItem->pPrev = newItem;
    3778  }
    3779  else
    3780  {
    3781  VMA_HEAVY_ASSERT(m_pBack == pItem);
    3782  m_pBack = newItem;
    3783  }
    3784  ++m_Count;
    3785  return newItem;
    3786  }
    3787  else
    3788  return PushFront();
    3789 }
    3790 
    3791 template<typename T>
    3792 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
    3793 {
    3794  ItemType* const newItem = InsertBefore(pItem);
    3795  newItem->Value = value;
    3796  return newItem;
    3797 }
    3798 
    3799 template<typename T>
    3800 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
    3801 {
    3802  ItemType* const newItem = InsertAfter(pItem);
    3803  newItem->Value = value;
    3804  return newItem;
    3805 }
    3806 
    3807 template<typename T, typename AllocatorT>
    3808 class VmaList
    3809 {
    3810  VMA_CLASS_NO_COPY(VmaList)
    3811 public:
    3812  class iterator
    3813  {
    3814  public:
    3815  iterator() :
    3816  m_pList(VMA_NULL),
    3817  m_pItem(VMA_NULL)
    3818  {
    3819  }
    3820 
    3821  T& operator*() const
    3822  {
    3823  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3824  return m_pItem->Value;
    3825  }
    3826  T* operator->() const
    3827  {
    3828  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3829  return &m_pItem->Value;
    3830  }
    3831 
    3832  iterator& operator++()
    3833  {
    3834  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3835  m_pItem = m_pItem->pNext;
    3836  return *this;
    3837  }
    3838  iterator& operator--()
    3839  {
    3840  if(m_pItem != VMA_NULL)
    3841  {
    3842  m_pItem = m_pItem->pPrev;
    3843  }
    3844  else
    3845  {
    3846  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3847  m_pItem = m_pList->Back();
    3848  }
    3849  return *this;
    3850  }
    3851 
    3852  iterator operator++(int)
    3853  {
    3854  iterator result = *this;
    3855  ++*this;
    3856  return result;
    3857  }
    3858  iterator operator--(int)
    3859  {
    3860  iterator result = *this;
    3861  --*this;
    3862  return result;
    3863  }
    3864 
    3865  bool operator==(const iterator& rhs) const
    3866  {
    3867  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3868  return m_pItem == rhs.m_pItem;
    3869  }
    3870  bool operator!=(const iterator& rhs) const
    3871  {
    3872  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3873  return m_pItem != rhs.m_pItem;
    3874  }
    3875 
    3876  private:
    3877  VmaRawList<T>* m_pList;
    3878  VmaListItem<T>* m_pItem;
    3879 
    3880  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
    3881  m_pList(pList),
    3882  m_pItem(pItem)
    3883  {
    3884  }
    3885 
    3886  friend class VmaList<T, AllocatorT>;
    3887  };
    3888 
    3889  class const_iterator
    3890  {
    3891  public:
    3892  const_iterator() :
    3893  m_pList(VMA_NULL),
    3894  m_pItem(VMA_NULL)
    3895  {
    3896  }
    3897 
    3898  const_iterator(const iterator& src) :
    3899  m_pList(src.m_pList),
    3900  m_pItem(src.m_pItem)
    3901  {
    3902  }
    3903 
    3904  const T& operator*() const
    3905  {
    3906  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3907  return m_pItem->Value;
    3908  }
    3909  const T* operator->() const
    3910  {
    3911  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3912  return &m_pItem->Value;
    3913  }
    3914 
    3915  const_iterator& operator++()
    3916  {
    3917  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
    3918  m_pItem = m_pItem->pNext;
    3919  return *this;
    3920  }
    3921  const_iterator& operator--()
    3922  {
    3923  if(m_pItem != VMA_NULL)
    3924  {
    3925  m_pItem = m_pItem->pPrev;
    3926  }
    3927  else
    3928  {
    3929  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
    3930  m_pItem = m_pList->Back();
    3931  }
    3932  return *this;
    3933  }
    3934 
    3935  const_iterator operator++(int)
    3936  {
    3937  const_iterator result = *this;
    3938  ++*this;
    3939  return result;
    3940  }
    3941  const_iterator operator--(int)
    3942  {
    3943  const_iterator result = *this;
    3944  --*this;
    3945  return result;
    3946  }
    3947 
    3948  bool operator==(const const_iterator& rhs) const
    3949  {
    3950  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3951  return m_pItem == rhs.m_pItem;
    3952  }
    3953  bool operator!=(const const_iterator& rhs) const
    3954  {
    3955  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
    3956  return m_pItem != rhs.m_pItem;
    3957  }
    3958 
    3959  private:
    3960  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
    3961  m_pList(pList),
    3962  m_pItem(pItem)
    3963  {
    3964  }
    3965 
    3966  const VmaRawList<T>* m_pList;
    3967  const VmaListItem<T>* m_pItem;
    3968 
    3969  friend class VmaList<T, AllocatorT>;
    3970  };
    3971 
    3972  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
    3973 
    3974  bool empty() const { return m_RawList.IsEmpty(); }
    3975  size_t size() const { return m_RawList.GetCount(); }
    3976 
    3977  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
    3978  iterator end() { return iterator(&m_RawList, VMA_NULL); }
    3979 
    3980  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
    3981  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
    3982 
    3983  void clear() { m_RawList.Clear(); }
    3984  void push_back(const T& value) { m_RawList.PushBack(value); }
    3985  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
    3986  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
    3987 
    3988 private:
    3989  VmaRawList<T> m_RawList;
    3990 };
    3991 
    3992 #endif // #if VMA_USE_STL_LIST
    3993 
    3995 // class VmaMap
    3996 
    3997 // Unused in this version.
    3998 #if 0
    3999 
    4000 #if VMA_USE_STL_UNORDERED_MAP
    4001 
    4002 #define VmaPair std::pair
    4003 
    4004 #define VMA_MAP_TYPE(KeyT, ValueT) \
    4005  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
    4006 
    4007 #else // #if VMA_USE_STL_UNORDERED_MAP
    4008 
    4009 template<typename T1, typename T2>
    4010 struct VmaPair
    4011 {
    4012  T1 first;
    4013  T2 second;
    4014 
    4015  VmaPair() : first(), second() { }
    4016  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
    4017 };
    4018 
    4019 /* Class compatible with subset of interface of std::unordered_map.
    4020 KeyT, ValueT must be POD because they will be stored in VmaVector.
    4021 */
    4022 template<typename KeyT, typename ValueT>
    4023 class VmaMap
    4024 {
    4025 public:
    4026  typedef VmaPair<KeyT, ValueT> PairType;
    4027  typedef PairType* iterator;
    4028 
    4029  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
    4030 
    4031  iterator begin() { return m_Vector.begin(); }
    4032  iterator end() { return m_Vector.end(); }
    4033 
    4034  void insert(const PairType& pair);
    4035  iterator find(const KeyT& key);
    4036  void erase(iterator it);
    4037 
    4038 private:
    4039  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
    4040 };
    4041 
    4042 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
    4043 
    4044 template<typename FirstT, typename SecondT>
    4045 struct VmaPairFirstLess
    4046 {
    4047  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
    4048  {
    4049  return lhs.first < rhs.first;
    4050  }
    4051  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
    4052  {
    4053  return lhs.first < rhsFirst;
    4054  }
    4055 };
    4056 
    4057 template<typename KeyT, typename ValueT>
    4058 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
    4059 {
    4060  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
    4061  m_Vector.data(),
    4062  m_Vector.data() + m_Vector.size(),
    4063  pair,
    4064  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
    4065  VmaVectorInsert(m_Vector, indexToInsert, pair);
    4066 }
    4067 
    4068 template<typename KeyT, typename ValueT>
    4069 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
    4070 {
    4071  PairType* it = VmaBinaryFindFirstNotLess(
    4072  m_Vector.data(),
    4073  m_Vector.data() + m_Vector.size(),
    4074  key,
    4075  VmaPairFirstLess<KeyT, ValueT>());
    4076  if((it != m_Vector.end()) && (it->first == key))
    4077  {
    4078  return it;
    4079  }
    4080  else
    4081  {
    4082  return m_Vector.end();
    4083  }
    4084 }
    4085 
    4086 template<typename KeyT, typename ValueT>
    4087 void VmaMap<KeyT, ValueT>::erase(iterator it)
    4088 {
    4089  VmaVectorRemove(m_Vector, it - m_Vector.begin());
    4090 }
    4091 
    4092 #endif // #if VMA_USE_STL_UNORDERED_MAP
    4093 
    4094 #endif // #if 0
    4095 
    4097 
    4098 class VmaDeviceMemoryBlock;
    4099 
    4100 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
    4101 
    4102 struct VmaAllocation_T
    4103 {
    4104  VMA_CLASS_NO_COPY(VmaAllocation_T)
    4105 private:
    4106  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
    4107 
    4108  enum FLAGS
    4109  {
    4110  FLAG_USER_DATA_STRING = 0x01,
    4111  };
    4112 
    4113 public:
    4114  enum ALLOCATION_TYPE
    4115  {
    4116  ALLOCATION_TYPE_NONE,
    4117  ALLOCATION_TYPE_BLOCK,
    4118  ALLOCATION_TYPE_DEDICATED,
    4119  };
    4120 
    4121  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
    4122  m_Alignment(1),
    4123  m_Size(0),
    4124  m_pUserData(VMA_NULL),
    4125  m_LastUseFrameIndex(currentFrameIndex),
    4126  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
    4127  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
    4128  m_MapCount(0),
    4129  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
    4130  {
    4131 #if VMA_STATS_STRING_ENABLED
    4132  m_CreationFrameIndex = currentFrameIndex;
    4133  m_BufferImageUsage = 0;
    4134 #endif
    4135  }
    4136 
    4137  ~VmaAllocation_T()
    4138  {
    4139  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
    4140 
    4141  // Check if owned string was freed.
    4142  VMA_ASSERT(m_pUserData == VMA_NULL);
    4143  }
    4144 
    4145  void InitBlockAllocation(
    4146  VmaPool hPool,
    4147  VmaDeviceMemoryBlock* block,
    4148  VkDeviceSize offset,
    4149  VkDeviceSize alignment,
    4150  VkDeviceSize size,
    4151  VmaSuballocationType suballocationType,
    4152  bool mapped,
    4153  bool canBecomeLost)
    4154  {
    4155  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4156  VMA_ASSERT(block != VMA_NULL);
    4157  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4158  m_Alignment = alignment;
    4159  m_Size = size;
    4160  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4161  m_SuballocationType = (uint8_t)suballocationType;
    4162  m_BlockAllocation.m_hPool = hPool;
    4163  m_BlockAllocation.m_Block = block;
    4164  m_BlockAllocation.m_Offset = offset;
    4165  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
    4166  }
    4167 
    4168  void InitLost()
    4169  {
    4170  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4171  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
    4172  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
    4173  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
    4174  m_BlockAllocation.m_Block = VMA_NULL;
    4175  m_BlockAllocation.m_Offset = 0;
    4176  m_BlockAllocation.m_CanBecomeLost = true;
    4177  }
    4178 
    4179  void ChangeBlockAllocation(
    4180  VmaAllocator hAllocator,
    4181  VmaDeviceMemoryBlock* block,
    4182  VkDeviceSize offset);
    4183 
    4184  // pMappedData not null means allocation is created with MAPPED flag.
    4185  void InitDedicatedAllocation(
    4186  uint32_t memoryTypeIndex,
    4187  VkDeviceMemory hMemory,
    4188  VmaSuballocationType suballocationType,
    4189  void* pMappedData,
    4190  VkDeviceSize size)
    4191  {
    4192  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
    4193  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
    4194  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
    4195  m_Alignment = 0;
    4196  m_Size = size;
    4197  m_SuballocationType = (uint8_t)suballocationType;
    4198  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
    4199  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
    4200  m_DedicatedAllocation.m_hMemory = hMemory;
    4201  m_DedicatedAllocation.m_pMappedData = pMappedData;
    4202  }
    4203 
    4204  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
    4205  VkDeviceSize GetAlignment() const { return m_Alignment; }
    4206  VkDeviceSize GetSize() const { return m_Size; }
    4207  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
    4208  void* GetUserData() const { return m_pUserData; }
    4209  void SetUserData(VmaAllocator hAllocator, void* pUserData);
    4210  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
    4211 
    4212  VmaDeviceMemoryBlock* GetBlock() const
    4213  {
    4214  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    4215  return m_BlockAllocation.m_Block;
    4216  }
    4217  VkDeviceSize GetOffset() const;
    4218  VkDeviceMemory GetMemory() const;
    4219  uint32_t GetMemoryTypeIndex() const;
    4220  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
    4221  void* GetMappedData() const;
    4222  bool CanBecomeLost() const;
    4223  VmaPool GetPool() const;
    4224 
    4225  uint32_t GetLastUseFrameIndex() const
    4226  {
    4227  return m_LastUseFrameIndex.load();
    4228  }
    4229  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
    4230  {
    4231  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
    4232  }
    4233  /*
    4234  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
    4235  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
    4236  - Else, returns false.
    4237 
    4238  If hAllocation is already lost, assert - you should not call it then.
    4239  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
    4240  */
    4241  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4242 
    4243  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
    4244  {
    4245  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
    4246  outInfo.blockCount = 1;
    4247  outInfo.allocationCount = 1;
    4248  outInfo.unusedRangeCount = 0;
    4249  outInfo.usedBytes = m_Size;
    4250  outInfo.unusedBytes = 0;
    4251  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
    4252  outInfo.unusedRangeSizeMin = UINT64_MAX;
    4253  outInfo.unusedRangeSizeMax = 0;
    4254  }
    4255 
    4256  void BlockAllocMap();
    4257  void BlockAllocUnmap();
    4258  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
    4259  void DedicatedAllocUnmap(VmaAllocator hAllocator);
    4260 
    4261 #if VMA_STATS_STRING_ENABLED
    4262  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
    4263  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
    4264 
    4265  void InitBufferImageUsage(uint32_t bufferImageUsage)
    4266  {
    4267  VMA_ASSERT(m_BufferImageUsage == 0);
    4268  m_BufferImageUsage = bufferImageUsage;
    4269  }
    4270 
    4271  void PrintParameters(class VmaJsonWriter& json) const;
    4272 #endif
    4273 
    4274 private:
    4275  VkDeviceSize m_Alignment;
    4276  VkDeviceSize m_Size;
    4277  void* m_pUserData;
    4278  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
    4279  uint8_t m_Type; // ALLOCATION_TYPE
    4280  uint8_t m_SuballocationType; // VmaSuballocationType
    4281  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
    4282  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
    4283  uint8_t m_MapCount;
    4284  uint8_t m_Flags; // enum FLAGS
    4285 
    4286  // Allocation out of VmaDeviceMemoryBlock.
    4287  struct BlockAllocation
    4288  {
    4289  VmaPool m_hPool; // Null if belongs to general memory.
    4290  VmaDeviceMemoryBlock* m_Block;
    4291  VkDeviceSize m_Offset;
    4292  bool m_CanBecomeLost;
    4293  };
    4294 
    4295  // Allocation for an object that has its own private VkDeviceMemory.
    4296  struct DedicatedAllocation
    4297  {
    4298  uint32_t m_MemoryTypeIndex;
    4299  VkDeviceMemory m_hMemory;
    4300  void* m_pMappedData; // Not null means memory is mapped.
    4301  };
    4302 
    4303  union
    4304  {
    4305  // Allocation out of VmaDeviceMemoryBlock.
    4306  BlockAllocation m_BlockAllocation;
    4307  // Allocation for an object that has its own private VkDeviceMemory.
    4308  DedicatedAllocation m_DedicatedAllocation;
    4309  };
    4310 
    4311 #if VMA_STATS_STRING_ENABLED
    4312  uint32_t m_CreationFrameIndex;
    4313  uint32_t m_BufferImageUsage; // 0 if unknown.
    4314 #endif
    4315 
    4316  void FreeUserDataString(VmaAllocator hAllocator);
    4317 };
    4318 
    4319 /*
    4320 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
    4321 allocated memory block or free.
    4322 */
    4323 struct VmaSuballocation
    4324 {
    4325  VkDeviceSize offset;
    4326  VkDeviceSize size;
    4327  VmaAllocation hAllocation;
    4328  VmaSuballocationType type;
    4329 };
    4330 
    4331 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
    4332 
    4333 // Cost of one additional allocation lost, as equivalent in bytes.
    4334 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
    4335 
    4336 /*
    4337 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
    4338 
    4339 If canMakeOtherLost was false:
    4340 - item points to a FREE suballocation.
    4341 - itemsToMakeLostCount is 0.
    4342 
    4343 If canMakeOtherLost was true:
    4344 - item points to first of sequence of suballocations, which are either FREE,
    4345  or point to VmaAllocations that can become lost.
    4346 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
    4347  the requested allocation to succeed.
    4348 */
    4349 struct VmaAllocationRequest
    4350 {
    4351  VkDeviceSize offset;
    4352  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
    4353  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
    4354  VmaSuballocationList::iterator item;
    4355  size_t itemsToMakeLostCount;
    4356 
    4357  VkDeviceSize CalcCost() const
    4358  {
    4359  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
    4360  }
    4361 };
    4362 
    4363 /*
    4364 Data structure used for bookkeeping of allocations and unused ranges of memory
    4365 in a single VkDeviceMemory block.
    4366 */
    4367 class VmaBlockMetadata
    4368 {
    4369  VMA_CLASS_NO_COPY(VmaBlockMetadata)
    4370 public:
    4371  VmaBlockMetadata(VmaAllocator hAllocator);
    4372  ~VmaBlockMetadata();
    4373  void Init(VkDeviceSize size);
    4374 
    4375  // Validates all data structures inside this object. If not valid, returns false.
    4376  bool Validate() const;
    4377  VkDeviceSize GetSize() const { return m_Size; }
    4378  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
    4379  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
    4380  VkDeviceSize GetUnusedRangeSizeMax() const;
    4381  // Returns true if this block is empty - contains only single free suballocation.
    4382  bool IsEmpty() const;
    4383 
    4384  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
    4385  void AddPoolStats(VmaPoolStats& inoutStats) const;
    4386 
    4387 #if VMA_STATS_STRING_ENABLED
    4388  void PrintDetailedMap(class VmaJsonWriter& json) const;
    4389 #endif
    4390 
    4391  // Tries to find a place for suballocation with given parameters inside this block.
    4392  // If succeeded, fills pAllocationRequest and returns true.
    4393  // If failed, returns false.
    4394  bool CreateAllocationRequest(
    4395  uint32_t currentFrameIndex,
    4396  uint32_t frameInUseCount,
    4397  VkDeviceSize bufferImageGranularity,
    4398  VkDeviceSize allocSize,
    4399  VkDeviceSize allocAlignment,
    4400  VmaSuballocationType allocType,
    4401  bool canMakeOtherLost,
    4402  VmaAllocationRequest* pAllocationRequest);
    4403 
    4404  bool MakeRequestedAllocationsLost(
    4405  uint32_t currentFrameIndex,
    4406  uint32_t frameInUseCount,
    4407  VmaAllocationRequest* pAllocationRequest);
    4408 
    4409  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
    4410 
    4411  VkResult CheckCorruption(const void* pBlockData);
    4412 
    4413  // Makes actual allocation based on request. Request must already be checked and valid.
    4414  void Alloc(
    4415  const VmaAllocationRequest& request,
    4416  VmaSuballocationType type,
    4417  VkDeviceSize allocSize,
    4418  VmaAllocation hAllocation);
    4419 
    4420  // Frees suballocation assigned to given memory region.
    4421  void Free(const VmaAllocation allocation);
    4422  void FreeAtOffset(VkDeviceSize offset);
    4423 
    4424 private:
    4425  VkDeviceSize m_Size;
    4426  uint32_t m_FreeCount;
    4427  VkDeviceSize m_SumFreeSize;
    4428  VmaSuballocationList m_Suballocations;
    4429  // Suballocations that are free and have size greater than certain threshold.
    4430  // Sorted by size, ascending.
    4431  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
    4432 
    4433  bool ValidateFreeSuballocationList() const;
    4434 
    4435  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
    4436  // If yes, fills pOffset and returns true. If no, returns false.
    4437  bool CheckAllocation(
    4438  uint32_t currentFrameIndex,
    4439  uint32_t frameInUseCount,
    4440  VkDeviceSize bufferImageGranularity,
    4441  VkDeviceSize allocSize,
    4442  VkDeviceSize allocAlignment,
    4443  VmaSuballocationType allocType,
    4444  VmaSuballocationList::const_iterator suballocItem,
    4445  bool canMakeOtherLost,
    4446  VkDeviceSize* pOffset,
    4447  size_t* itemsToMakeLostCount,
    4448  VkDeviceSize* pSumFreeSize,
    4449  VkDeviceSize* pSumItemSize) const;
    4450  // Given free suballocation, it merges it with following one, which must also be free.
    4451  void MergeFreeWithNext(VmaSuballocationList::iterator item);
    4452  // Releases given suballocation, making it free.
    4453  // Merges it with adjacent free suballocations if applicable.
    4454  // Returns iterator to new free suballocation at this place.
    4455  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
    4456  // Given free suballocation, it inserts it into sorted list of
    4457  // m_FreeSuballocationsBySize if it's suitable.
    4458  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
    4459  // Given free suballocation, it removes it from sorted list of
    4460  // m_FreeSuballocationsBySize if it's suitable.
    4461  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
    4462 };
    4463 
    4464 /*
    4465 Represents a single block of device memory (`VkDeviceMemory`) with all the
    4466 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
    4467 
    4468 Thread-safety: This class must be externally synchronized.
    4469 */
    4470 class VmaDeviceMemoryBlock
    4471 {
    4472  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
    4473 public:
    4474  VmaBlockMetadata m_Metadata;
    4475 
    4476  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
    4477 
    4478  ~VmaDeviceMemoryBlock()
    4479  {
    4480  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
    4481  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    4482  }
    4483 
    4484  // Always call after construction.
    4485  void Init(
    4486  uint32_t newMemoryTypeIndex,
    4487  VkDeviceMemory newMemory,
    4488  VkDeviceSize newSize,
    4489  uint32_t id);
    4490  // Always call before destruction.
    4491  void Destroy(VmaAllocator allocator);
    4492 
    4493  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
    4494  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4495  uint32_t GetId() const { return m_Id; }
    4496  void* GetMappedData() const { return m_pMappedData; }
    4497 
    4498  // Validates all data structures inside this object. If not valid, returns false.
    4499  bool Validate() const;
    4500 
    4501  VkResult CheckCorruption(VmaAllocator hAllocator);
    4502 
    4503  // ppData can be null.
    4504  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
    4505  void Unmap(VmaAllocator hAllocator, uint32_t count);
    4506 
    4507  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4508  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
    4509 
    4510  VkResult BindBufferMemory(
    4511  const VmaAllocator hAllocator,
    4512  const VmaAllocation hAllocation,
    4513  VkBuffer hBuffer);
    4514  VkResult BindImageMemory(
    4515  const VmaAllocator hAllocator,
    4516  const VmaAllocation hAllocation,
    4517  VkImage hImage);
    4518 
    4519 private:
    4520  uint32_t m_MemoryTypeIndex;
    4521  uint32_t m_Id;
    4522  VkDeviceMemory m_hMemory;
    4523 
    4524  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
    4525  // Also protects m_MapCount, m_pMappedData.
    4526  VMA_MUTEX m_Mutex;
    4527  uint32_t m_MapCount;
    4528  void* m_pMappedData;
    4529 };
    4530 
    4531 struct VmaPointerLess
    4532 {
    4533  bool operator()(const void* lhs, const void* rhs) const
    4534  {
    4535  return lhs < rhs;
    4536  }
    4537 };
    4538 
    4539 class VmaDefragmentator;
    4540 
    4541 /*
    4542 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
    4543 Vulkan memory type.
    4544 
    4545 Synchronized internally with a mutex.
    4546 */
    4547 struct VmaBlockVector
    4548 {
    4549  VMA_CLASS_NO_COPY(VmaBlockVector)
    4550 public:
    4551  VmaBlockVector(
    4552  VmaAllocator hAllocator,
    4553  uint32_t memoryTypeIndex,
    4554  VkDeviceSize preferredBlockSize,
    4555  size_t minBlockCount,
    4556  size_t maxBlockCount,
    4557  VkDeviceSize bufferImageGranularity,
    4558  uint32_t frameInUseCount,
    4559  bool isCustomPool);
    4560  ~VmaBlockVector();
    4561 
    4562  VkResult CreateMinBlocks();
    4563 
    4564  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
    4565  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
    4566  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
    4567  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
    4568 
    4569  void GetPoolStats(VmaPoolStats* pStats);
    4570 
    4571  bool IsEmpty() const { return m_Blocks.empty(); }
    4572  bool IsCorruptionDetectionEnabled() const;
    4573 
    4574  VkResult Allocate(
    4575  VmaPool hCurrentPool,
    4576  uint32_t currentFrameIndex,
    4577  VkDeviceSize size,
    4578  VkDeviceSize alignment,
    4579  const VmaAllocationCreateInfo& createInfo,
    4580  VmaSuballocationType suballocType,
    4581  VmaAllocation* pAllocation);
    4582 
    4583  void Free(
    4584  VmaAllocation hAllocation);
    4585 
    4586  // Adds statistics of this BlockVector to pStats.
    4587  void AddStats(VmaStats* pStats);
    4588 
    4589 #if VMA_STATS_STRING_ENABLED
    4590  void PrintDetailedMap(class VmaJsonWriter& json);
    4591 #endif
    4592 
    4593  void MakePoolAllocationsLost(
    4594  uint32_t currentFrameIndex,
    4595  size_t* pLostAllocationCount);
    4596  VkResult CheckCorruption();
    4597 
    4598  VmaDefragmentator* EnsureDefragmentator(
    4599  VmaAllocator hAllocator,
    4600  uint32_t currentFrameIndex);
    4601 
    4602  VkResult Defragment(
    4603  VmaDefragmentationStats* pDefragmentationStats,
    4604  VkDeviceSize& maxBytesToMove,
    4605  uint32_t& maxAllocationsToMove);
    4606 
    4607  void DestroyDefragmentator();
    4608 
    4609 private:
    4610  friend class VmaDefragmentator;
    4611 
    4612  const VmaAllocator m_hAllocator;
    4613  const uint32_t m_MemoryTypeIndex;
    4614  const VkDeviceSize m_PreferredBlockSize;
    4615  const size_t m_MinBlockCount;
    4616  const size_t m_MaxBlockCount;
    4617  const VkDeviceSize m_BufferImageGranularity;
    4618  const uint32_t m_FrameInUseCount;
    4619  const bool m_IsCustomPool;
    4620  VMA_MUTEX m_Mutex;
    4621  // Incrementally sorted by sumFreeSize, ascending.
    4622  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
    4623  /* There can be at most one allocation that is completely empty - a
    4624  hysteresis to avoid pessimistic case of alternating creation and destruction
    4625  of a VkDeviceMemory. */
    4626  bool m_HasEmptyBlock;
    4627  VmaDefragmentator* m_pDefragmentator;
    4628  uint32_t m_NextBlockId;
    4629 
    4630  VkDeviceSize CalcMaxBlockSize() const;
    4631 
    4632  // Finds and removes given block from vector.
    4633  void Remove(VmaDeviceMemoryBlock* pBlock);
    4634 
    4635  // Performs single step in sorting m_Blocks. They may not be fully sorted
    4636  // after this call.
    4637  void IncrementallySortBlocks();
    4638 
    4639  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
    4640 };
    4641 
    4642 struct VmaPool_T
    4643 {
    4644  VMA_CLASS_NO_COPY(VmaPool_T)
    4645 public:
    4646  VmaBlockVector m_BlockVector;
    4647 
    4648  VmaPool_T(
    4649  VmaAllocator hAllocator,
    4650  const VmaPoolCreateInfo& createInfo);
    4651  ~VmaPool_T();
    4652 
    4653  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
    4654  uint32_t GetId() const { return m_Id; }
    4655  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
    4656 
    4657 #if VMA_STATS_STRING_ENABLED
    4658  //void PrintDetailedMap(class VmaStringBuilder& sb);
    4659 #endif
    4660 
    4661 private:
    4662  uint32_t m_Id;
    4663 };
    4664 
    4665 class VmaDefragmentator
    4666 {
    4667  VMA_CLASS_NO_COPY(VmaDefragmentator)
    4668 private:
    4669  const VmaAllocator m_hAllocator;
    4670  VmaBlockVector* const m_pBlockVector;
    4671  uint32_t m_CurrentFrameIndex;
    4672  VkDeviceSize m_BytesMoved;
    4673  uint32_t m_AllocationsMoved;
    4674 
    4675  struct AllocationInfo
    4676  {
    4677  VmaAllocation m_hAllocation;
    4678  VkBool32* m_pChanged;
    4679 
    4680  AllocationInfo() :
    4681  m_hAllocation(VK_NULL_HANDLE),
    4682  m_pChanged(VMA_NULL)
    4683  {
    4684  }
    4685  };
    4686 
    4687  struct AllocationInfoSizeGreater
    4688  {
    4689  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
    4690  {
    4691  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
    4692  }
    4693  };
    4694 
    4695  // Used between AddAllocation and Defragment.
    4696  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4697 
    4698  struct BlockInfo
    4699  {
    4700  VmaDeviceMemoryBlock* m_pBlock;
    4701  bool m_HasNonMovableAllocations;
    4702  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
    4703 
    4704  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
    4705  m_pBlock(VMA_NULL),
    4706  m_HasNonMovableAllocations(true),
    4707  m_Allocations(pAllocationCallbacks),
    4708  m_pMappedDataForDefragmentation(VMA_NULL)
    4709  {
    4710  }
    4711 
    4712  void CalcHasNonMovableAllocations()
    4713  {
    4714  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
    4715  const size_t defragmentAllocCount = m_Allocations.size();
    4716  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
    4717  }
    4718 
    4719  void SortAllocationsBySizeDescecnding()
    4720  {
    4721  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
    4722  }
    4723 
    4724  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
    4725  void Unmap(VmaAllocator hAllocator);
    4726 
    4727  private:
    4728  // Not null if mapped for defragmentation only, not originally mapped.
    4729  void* m_pMappedDataForDefragmentation;
    4730  };
    4731 
    4732  struct BlockPointerLess
    4733  {
    4734  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
    4735  {
    4736  return pLhsBlockInfo->m_pBlock < pRhsBlock;
    4737  }
    4738  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4739  {
    4740  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
    4741  }
    4742  };
    4743 
    4744  // 1. Blocks with some non-movable allocations go first.
    4745  // 2. Blocks with smaller sumFreeSize go first.
    4746  struct BlockInfoCompareMoveDestination
    4747  {
    4748  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
    4749  {
    4750  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
    4751  {
    4752  return true;
    4753  }
    4754  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
    4755  {
    4756  return false;
    4757  }
    4758  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
    4759  {
    4760  return true;
    4761  }
    4762  return false;
    4763  }
    4764  };
    4765 
    4766  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
    4767  BlockInfoVector m_Blocks;
    4768 
    4769  VkResult DefragmentRound(
    4770  VkDeviceSize maxBytesToMove,
    4771  uint32_t maxAllocationsToMove);
    4772 
    4773  static bool MoveMakesSense(
    4774  size_t dstBlockIndex, VkDeviceSize dstOffset,
    4775  size_t srcBlockIndex, VkDeviceSize srcOffset);
    4776 
    4777 public:
    4778  VmaDefragmentator(
    4779  VmaAllocator hAllocator,
    4780  VmaBlockVector* pBlockVector,
    4781  uint32_t currentFrameIndex);
    4782 
    4783  ~VmaDefragmentator();
    4784 
    4785  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
    4786  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
    4787 
    4788  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
    4789 
    4790  VkResult Defragment(
    4791  VkDeviceSize maxBytesToMove,
    4792  uint32_t maxAllocationsToMove);
    4793 };
    4794 
    4795 #if VMA_RECORDING_ENABLED
    4796 
    4797 class VmaRecorder
    4798 {
    4799 public:
    4800  VmaRecorder();
    4801  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
    4802  void WriteConfiguration(
    4803  const VkPhysicalDeviceProperties& devProps,
    4804  const VkPhysicalDeviceMemoryProperties& memProps,
    4805  bool dedicatedAllocationExtensionEnabled);
    4806  ~VmaRecorder();
    4807 
    4808  void RecordCreateAllocator(uint32_t frameIndex);
    4809  void RecordDestroyAllocator(uint32_t frameIndex);
    4810  void RecordCreatePool(uint32_t frameIndex,
    4811  const VmaPoolCreateInfo& createInfo,
    4812  VmaPool pool);
    4813  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
    4814  void RecordAllocateMemory(uint32_t frameIndex,
    4815  const VkMemoryRequirements& vkMemReq,
    4816  const VmaAllocationCreateInfo& createInfo,
    4817  VmaAllocation allocation);
    4818  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    4819  const VkMemoryRequirements& vkMemReq,
    4820  bool requiresDedicatedAllocation,
    4821  bool prefersDedicatedAllocation,
    4822  const VmaAllocationCreateInfo& createInfo,
    4823  VmaAllocation allocation);
    4824  void RecordAllocateMemoryForImage(uint32_t frameIndex,
    4825  const VkMemoryRequirements& vkMemReq,
    4826  bool requiresDedicatedAllocation,
    4827  bool prefersDedicatedAllocation,
    4828  const VmaAllocationCreateInfo& createInfo,
    4829  VmaAllocation allocation);
    4830  void RecordFreeMemory(uint32_t frameIndex,
    4831  VmaAllocation allocation);
    4832  void RecordSetAllocationUserData(uint32_t frameIndex,
    4833  VmaAllocation allocation,
    4834  const void* pUserData);
    4835  void RecordCreateLostAllocation(uint32_t frameIndex,
    4836  VmaAllocation allocation);
    4837  void RecordMapMemory(uint32_t frameIndex,
    4838  VmaAllocation allocation);
    4839  void RecordUnmapMemory(uint32_t frameIndex,
    4840  VmaAllocation allocation);
    4841  void RecordFlushAllocation(uint32_t frameIndex,
    4842  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4843  void RecordInvalidateAllocation(uint32_t frameIndex,
    4844  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
    4845  void RecordCreateBuffer(uint32_t frameIndex,
    4846  const VkBufferCreateInfo& bufCreateInfo,
    4847  const VmaAllocationCreateInfo& allocCreateInfo,
    4848  VmaAllocation allocation);
    4849  void RecordCreateImage(uint32_t frameIndex,
    4850  const VkImageCreateInfo& imageCreateInfo,
    4851  const VmaAllocationCreateInfo& allocCreateInfo,
    4852  VmaAllocation allocation);
    4853  void RecordDestroyBuffer(uint32_t frameIndex,
    4854  VmaAllocation allocation);
    4855  void RecordDestroyImage(uint32_t frameIndex,
    4856  VmaAllocation allocation);
    4857  void RecordTouchAllocation(uint32_t frameIndex,
    4858  VmaAllocation allocation);
    4859  void RecordGetAllocationInfo(uint32_t frameIndex,
    4860  VmaAllocation allocation);
    4861  void RecordMakePoolAllocationsLost(uint32_t frameIndex,
    4862  VmaPool pool);
    4863 
    4864 private:
    4865  struct CallParams
    4866  {
    4867  uint32_t threadId;
    4868  double time;
    4869  };
    4870 
    4871  class UserDataString
    4872  {
    4873  public:
    4874  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
    4875  const char* GetString() const { return m_Str; }
    4876 
    4877  private:
    4878  char m_PtrStr[17];
    4879  const char* m_Str;
    4880  };
    4881 
    4882  bool m_UseMutex;
    4883  VmaRecordFlags m_Flags;
    4884  FILE* m_File;
    4885  VMA_MUTEX m_FileMutex;
    4886  int64_t m_Freq;
    4887  int64_t m_StartCounter;
    4888 
    4889  void GetBasicParams(CallParams& outParams);
    4890  void Flush();
    4891 };
    4892 
    4893 #endif // #if VMA_RECORDING_ENABLED
    4894 
    4895 // Main allocator object.
    4896 struct VmaAllocator_T
    4897 {
    4898  VMA_CLASS_NO_COPY(VmaAllocator_T)
    4899 public:
    4900  bool m_UseMutex;
    4901  bool m_UseKhrDedicatedAllocation;
    4902  VkDevice m_hDevice;
    4903  bool m_AllocationCallbacksSpecified;
    4904  VkAllocationCallbacks m_AllocationCallbacks;
    4905  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
    4906 
    4907  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
    4908  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
    4909  VMA_MUTEX m_HeapSizeLimitMutex;
    4910 
    4911  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
    4912  VkPhysicalDeviceMemoryProperties m_MemProps;
    4913 
    4914  // Default pools.
    4915  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
    4916 
    4917  // Each vector is sorted by memory (handle value).
    4918  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
    4919  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
    4920  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
    4921 
    4922  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
    4923  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
    4924  ~VmaAllocator_T();
    4925 
    4926  const VkAllocationCallbacks* GetAllocationCallbacks() const
    4927  {
    4928  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
    4929  }
    4930  const VmaVulkanFunctions& GetVulkanFunctions() const
    4931  {
    4932  return m_VulkanFunctions;
    4933  }
    4934 
    4935  VkDeviceSize GetBufferImageGranularity() const
    4936  {
    4937  return VMA_MAX(
    4938  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
    4939  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
    4940  }
    4941 
    4942  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
    4943  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
    4944 
    4945  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
    4946  {
    4947  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
    4948  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
    4949  }
    4950  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
    4951  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
    4952  {
    4953  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
    4954  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    4955  }
    4956  // Minimum alignment for all allocations in specific memory type.
    4957  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
    4958  {
    4959  return IsMemoryTypeNonCoherent(memTypeIndex) ?
    4960  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
    4961  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
    4962  }
    4963 
    4964  bool IsIntegratedGpu() const
    4965  {
    4966  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
    4967  }
    4968 
    4969 #if VMA_RECORDING_ENABLED
    4970  VmaRecorder* GetRecorder() const { return m_pRecorder; }
    4971 #endif
    4972 
    4973  void GetBufferMemoryRequirements(
    4974  VkBuffer hBuffer,
    4975  VkMemoryRequirements& memReq,
    4976  bool& requiresDedicatedAllocation,
    4977  bool& prefersDedicatedAllocation) const;
    4978  void GetImageMemoryRequirements(
    4979  VkImage hImage,
    4980  VkMemoryRequirements& memReq,
    4981  bool& requiresDedicatedAllocation,
    4982  bool& prefersDedicatedAllocation) const;
    4983 
    4984  // Main allocation function.
    4985  VkResult AllocateMemory(
    4986  const VkMemoryRequirements& vkMemReq,
    4987  bool requiresDedicatedAllocation,
    4988  bool prefersDedicatedAllocation,
    4989  VkBuffer dedicatedBuffer,
    4990  VkImage dedicatedImage,
    4991  const VmaAllocationCreateInfo& createInfo,
    4992  VmaSuballocationType suballocType,
    4993  VmaAllocation* pAllocation);
    4994 
    4995  // Main deallocation function.
    4996  void FreeMemory(const VmaAllocation allocation);
    4997 
    4998  void CalculateStats(VmaStats* pStats);
    4999 
    5000 #if VMA_STATS_STRING_ENABLED
    5001  void PrintDetailedMap(class VmaJsonWriter& json);
    5002 #endif
    5003 
    5004  VkResult Defragment(
    5005  VmaAllocation* pAllocations,
    5006  size_t allocationCount,
    5007  VkBool32* pAllocationsChanged,
    5008  const VmaDefragmentationInfo* pDefragmentationInfo,
    5009  VmaDefragmentationStats* pDefragmentationStats);
    5010 
    5011  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
    5012  bool TouchAllocation(VmaAllocation hAllocation);
    5013 
    5014  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
    5015  void DestroyPool(VmaPool pool);
    5016  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
    5017 
    5018  void SetCurrentFrameIndex(uint32_t frameIndex);
    5019  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
    5020 
    5021  void MakePoolAllocationsLost(
    5022  VmaPool hPool,
    5023  size_t* pLostAllocationCount);
    5024  VkResult CheckPoolCorruption(VmaPool hPool);
    5025  VkResult CheckCorruption(uint32_t memoryTypeBits);
    5026 
    5027  void CreateLostAllocation(VmaAllocation* pAllocation);
    5028 
    5029  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
    5030  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
    5031 
    5032  VkResult Map(VmaAllocation hAllocation, void** ppData);
    5033  void Unmap(VmaAllocation hAllocation);
    5034 
    5035  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
    5036  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
    5037 
    5038  void FlushOrInvalidateAllocation(
    5039  VmaAllocation hAllocation,
    5040  VkDeviceSize offset, VkDeviceSize size,
    5041  VMA_CACHE_OPERATION op);
    5042 
    5043  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
    5044 
    5045 private:
    5046  VkDeviceSize m_PreferredLargeHeapBlockSize;
    5047 
    5048  VkPhysicalDevice m_PhysicalDevice;
    5049  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
    5050 
    5051  VMA_MUTEX m_PoolsMutex;
    5052  // Protected by m_PoolsMutex. Sorted by pointer value.
    5053  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
    5054  uint32_t m_NextPoolId;
    5055 
    5056  VmaVulkanFunctions m_VulkanFunctions;
    5057 
    5058 #if VMA_RECORDING_ENABLED
    5059  VmaRecorder* m_pRecorder;
    5060 #endif
    5061 
    5062  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
    5063 
    5064  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
    5065 
    5066  VkResult AllocateMemoryOfType(
    5067  VkDeviceSize size,
    5068  VkDeviceSize alignment,
    5069  bool dedicatedAllocation,
    5070  VkBuffer dedicatedBuffer,
    5071  VkImage dedicatedImage,
    5072  const VmaAllocationCreateInfo& createInfo,
    5073  uint32_t memTypeIndex,
    5074  VmaSuballocationType suballocType,
    5075  VmaAllocation* pAllocation);
    5076 
    5077  // Allocates and registers new VkDeviceMemory specifically for single allocation.
    5078  VkResult AllocateDedicatedMemory(
    5079  VkDeviceSize size,
    5080  VmaSuballocationType suballocType,
    5081  uint32_t memTypeIndex,
    5082  bool map,
    5083  bool isUserDataString,
    5084  void* pUserData,
    5085  VkBuffer dedicatedBuffer,
    5086  VkImage dedicatedImage,
    5087  VmaAllocation* pAllocation);
    5088 
    5089  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
    5090  void FreeDedicatedMemory(VmaAllocation allocation);
    5091 };
    5092 
    5094 // Memory allocation #2 after VmaAllocator_T definition
    5095 
    5096 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
    5097 {
    5098  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
    5099 }
    5100 
    5101 static void VmaFree(VmaAllocator hAllocator, void* ptr)
    5102 {
    5103  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
    5104 }
    5105 
    5106 template<typename T>
    5107 static T* VmaAllocate(VmaAllocator hAllocator)
    5108 {
    5109  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
    5110 }
    5111 
    5112 template<typename T>
    5113 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
    5114 {
    5115  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
    5116 }
    5117 
    5118 template<typename T>
    5119 static void vma_delete(VmaAllocator hAllocator, T* ptr)
    5120 {
    5121  if(ptr != VMA_NULL)
    5122  {
    5123  ptr->~T();
    5124  VmaFree(hAllocator, ptr);
    5125  }
    5126 }
    5127 
    5128 template<typename T>
    5129 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
    5130 {
    5131  if(ptr != VMA_NULL)
    5132  {
    5133  for(size_t i = count; i--; )
    5134  ptr[i].~T();
    5135  VmaFree(hAllocator, ptr);
    5136  }
    5137 }
    5138 
    5140 // VmaStringBuilder
    5141 
    5142 #if VMA_STATS_STRING_ENABLED
    5143 
    5144 class VmaStringBuilder
    5145 {
    5146 public:
    5147  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
    5148  size_t GetLength() const { return m_Data.size(); }
    5149  const char* GetData() const { return m_Data.data(); }
    5150 
    5151  void Add(char ch) { m_Data.push_back(ch); }
    5152  void Add(const char* pStr);
    5153  void AddNewLine() { Add('\n'); }
    5154  void AddNumber(uint32_t num);
    5155  void AddNumber(uint64_t num);
    5156  void AddPointer(const void* ptr);
    5157 
    5158 private:
    5159  VmaVector< char, VmaStlAllocator<char> > m_Data;
    5160 };
    5161 
    5162 void VmaStringBuilder::Add(const char* pStr)
    5163 {
    5164  const size_t strLen = strlen(pStr);
    5165  if(strLen > 0)
    5166  {
    5167  const size_t oldCount = m_Data.size();
    5168  m_Data.resize(oldCount + strLen);
    5169  memcpy(m_Data.data() + oldCount, pStr, strLen);
    5170  }
    5171 }
    5172 
    5173 void VmaStringBuilder::AddNumber(uint32_t num)
    5174 {
    5175  char buf[11];
    5176  VmaUint32ToStr(buf, sizeof(buf), num);
    5177  Add(buf);
    5178 }
    5179 
    5180 void VmaStringBuilder::AddNumber(uint64_t num)
    5181 {
    5182  char buf[21];
    5183  VmaUint64ToStr(buf, sizeof(buf), num);
    5184  Add(buf);
    5185 }
    5186 
    5187 void VmaStringBuilder::AddPointer(const void* ptr)
    5188 {
    5189  char buf[21];
    5190  VmaPtrToStr(buf, sizeof(buf), ptr);
    5191  Add(buf);
    5192 }
    5193 
    5194 #endif // #if VMA_STATS_STRING_ENABLED
    5195 
    5197 // VmaJsonWriter
    5198 
    5199 #if VMA_STATS_STRING_ENABLED
    5200 
    5201 class VmaJsonWriter
    5202 {
    5203  VMA_CLASS_NO_COPY(VmaJsonWriter)
    5204 public:
    5205  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
    5206  ~VmaJsonWriter();
    5207 
    5208  void BeginObject(bool singleLine = false);
    5209  void EndObject();
    5210 
    5211  void BeginArray(bool singleLine = false);
    5212  void EndArray();
    5213 
    5214  void WriteString(const char* pStr);
    5215  void BeginString(const char* pStr = VMA_NULL);
    5216  void ContinueString(const char* pStr);
    5217  void ContinueString(uint32_t n);
    5218  void ContinueString(uint64_t n);
    5219  void ContinueString_Pointer(const void* ptr);
    5220  void EndString(const char* pStr = VMA_NULL);
    5221 
    5222  void WriteNumber(uint32_t n);
    5223  void WriteNumber(uint64_t n);
    5224  void WriteBool(bool b);
    5225  void WriteNull();
    5226 
    5227 private:
    5228  static const char* const INDENT;
    5229 
    5230  enum COLLECTION_TYPE
    5231  {
    5232  COLLECTION_TYPE_OBJECT,
    5233  COLLECTION_TYPE_ARRAY,
    5234  };
    5235  struct StackItem
    5236  {
    5237  COLLECTION_TYPE type;
    5238  uint32_t valueCount;
    5239  bool singleLineMode;
    5240  };
    5241 
    5242  VmaStringBuilder& m_SB;
    5243  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
    5244  bool m_InsideString;
    5245 
    5246  void BeginValue(bool isString);
    5247  void WriteIndent(bool oneLess = false);
    5248 };
    5249 
    5250 const char* const VmaJsonWriter::INDENT = " ";
    5251 
    5252 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
    5253  m_SB(sb),
    5254  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
    5255  m_InsideString(false)
    5256 {
    5257 }
    5258 
    5259 VmaJsonWriter::~VmaJsonWriter()
    5260 {
    5261  VMA_ASSERT(!m_InsideString);
    5262  VMA_ASSERT(m_Stack.empty());
    5263 }
    5264 
    5265 void VmaJsonWriter::BeginObject(bool singleLine)
    5266 {
    5267  VMA_ASSERT(!m_InsideString);
    5268 
    5269  BeginValue(false);
    5270  m_SB.Add('{');
    5271 
    5272  StackItem item;
    5273  item.type = COLLECTION_TYPE_OBJECT;
    5274  item.valueCount = 0;
    5275  item.singleLineMode = singleLine;
    5276  m_Stack.push_back(item);
    5277 }
    5278 
    5279 void VmaJsonWriter::EndObject()
    5280 {
    5281  VMA_ASSERT(!m_InsideString);
    5282 
    5283  WriteIndent(true);
    5284  m_SB.Add('}');
    5285 
    5286  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
    5287  m_Stack.pop_back();
    5288 }
    5289 
    5290 void VmaJsonWriter::BeginArray(bool singleLine)
    5291 {
    5292  VMA_ASSERT(!m_InsideString);
    5293 
    5294  BeginValue(false);
    5295  m_SB.Add('[');
    5296 
    5297  StackItem item;
    5298  item.type = COLLECTION_TYPE_ARRAY;
    5299  item.valueCount = 0;
    5300  item.singleLineMode = singleLine;
    5301  m_Stack.push_back(item);
    5302 }
    5303 
    5304 void VmaJsonWriter::EndArray()
    5305 {
    5306  VMA_ASSERT(!m_InsideString);
    5307 
    5308  WriteIndent(true);
    5309  m_SB.Add(']');
    5310 
    5311  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
    5312  m_Stack.pop_back();
    5313 }
    5314 
    5315 void VmaJsonWriter::WriteString(const char* pStr)
    5316 {
    5317  BeginString(pStr);
    5318  EndString();
    5319 }
    5320 
    5321 void VmaJsonWriter::BeginString(const char* pStr)
    5322 {
    5323  VMA_ASSERT(!m_InsideString);
    5324 
    5325  BeginValue(true);
    5326  m_SB.Add('"');
    5327  m_InsideString = true;
    5328  if(pStr != VMA_NULL && pStr[0] != '\0')
    5329  {
    5330  ContinueString(pStr);
    5331  }
    5332 }
    5333 
    5334 void VmaJsonWriter::ContinueString(const char* pStr)
    5335 {
    5336  VMA_ASSERT(m_InsideString);
    5337 
    5338  const size_t strLen = strlen(pStr);
    5339  for(size_t i = 0; i < strLen; ++i)
    5340  {
    5341  char ch = pStr[i];
    5342  if(ch == '\\')
    5343  {
    5344  m_SB.Add("\\\\");
    5345  }
    5346  else if(ch == '"')
    5347  {
    5348  m_SB.Add("\\\"");
    5349  }
    5350  else if(ch >= 32)
    5351  {
    5352  m_SB.Add(ch);
    5353  }
    5354  else switch(ch)
    5355  {
    5356  case '\b':
    5357  m_SB.Add("\\b");
    5358  break;
    5359  case '\f':
    5360  m_SB.Add("\\f");
    5361  break;
    5362  case '\n':
    5363  m_SB.Add("\\n");
    5364  break;
    5365  case '\r':
    5366  m_SB.Add("\\r");
    5367  break;
    5368  case '\t':
    5369  m_SB.Add("\\t");
    5370  break;
    5371  default:
    5372  VMA_ASSERT(0 && "Character not currently supported.");
    5373  break;
    5374  }
    5375  }
    5376 }
    5377 
    5378 void VmaJsonWriter::ContinueString(uint32_t n)
    5379 {
    5380  VMA_ASSERT(m_InsideString);
    5381  m_SB.AddNumber(n);
    5382 }
    5383 
    5384 void VmaJsonWriter::ContinueString(uint64_t n)
    5385 {
    5386  VMA_ASSERT(m_InsideString);
    5387  m_SB.AddNumber(n);
    5388 }
    5389 
    5390 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
    5391 {
    5392  VMA_ASSERT(m_InsideString);
    5393  m_SB.AddPointer(ptr);
    5394 }
    5395 
    5396 void VmaJsonWriter::EndString(const char* pStr)
    5397 {
    5398  VMA_ASSERT(m_InsideString);
    5399  if(pStr != VMA_NULL && pStr[0] != '\0')
    5400  {
    5401  ContinueString(pStr);
    5402  }
    5403  m_SB.Add('"');
    5404  m_InsideString = false;
    5405 }
    5406 
    5407 void VmaJsonWriter::WriteNumber(uint32_t n)
    5408 {
    5409  VMA_ASSERT(!m_InsideString);
    5410  BeginValue(false);
    5411  m_SB.AddNumber(n);
    5412 }
    5413 
    5414 void VmaJsonWriter::WriteNumber(uint64_t n)
    5415 {
    5416  VMA_ASSERT(!m_InsideString);
    5417  BeginValue(false);
    5418  m_SB.AddNumber(n);
    5419 }
    5420 
    5421 void VmaJsonWriter::WriteBool(bool b)
    5422 {
    5423  VMA_ASSERT(!m_InsideString);
    5424  BeginValue(false);
    5425  m_SB.Add(b ? "true" : "false");
    5426 }
    5427 
    5428 void VmaJsonWriter::WriteNull()
    5429 {
    5430  VMA_ASSERT(!m_InsideString);
    5431  BeginValue(false);
    5432  m_SB.Add("null");
    5433 }
    5434 
    5435 void VmaJsonWriter::BeginValue(bool isString)
    5436 {
    5437  if(!m_Stack.empty())
    5438  {
    5439  StackItem& currItem = m_Stack.back();
    5440  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5441  currItem.valueCount % 2 == 0)
    5442  {
    5443  VMA_ASSERT(isString);
    5444  }
    5445 
    5446  if(currItem.type == COLLECTION_TYPE_OBJECT &&
    5447  currItem.valueCount % 2 != 0)
    5448  {
    5449  m_SB.Add(": ");
    5450  }
    5451  else if(currItem.valueCount > 0)
    5452  {
    5453  m_SB.Add(", ");
    5454  WriteIndent();
    5455  }
    5456  else
    5457  {
    5458  WriteIndent();
    5459  }
    5460  ++currItem.valueCount;
    5461  }
    5462 }
    5463 
    5464 void VmaJsonWriter::WriteIndent(bool oneLess)
    5465 {
    5466  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
    5467  {
    5468  m_SB.AddNewLine();
    5469 
    5470  size_t count = m_Stack.size();
    5471  if(count > 0 && oneLess)
    5472  {
    5473  --count;
    5474  }
    5475  for(size_t i = 0; i < count; ++i)
    5476  {
    5477  m_SB.Add(INDENT);
    5478  }
    5479  }
    5480 }
    5481 
    5482 #endif // #if VMA_STATS_STRING_ENABLED
    5483 
    5485 
    5486 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
    5487 {
    5488  if(IsUserDataString())
    5489  {
    5490  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
    5491 
    5492  FreeUserDataString(hAllocator);
    5493 
    5494  if(pUserData != VMA_NULL)
    5495  {
    5496  const char* const newStrSrc = (char*)pUserData;
    5497  const size_t newStrLen = strlen(newStrSrc);
    5498  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
    5499  memcpy(newStrDst, newStrSrc, newStrLen + 1);
    5500  m_pUserData = newStrDst;
    5501  }
    5502  }
    5503  else
    5504  {
    5505  m_pUserData = pUserData;
    5506  }
    5507 }
    5508 
    5509 void VmaAllocation_T::ChangeBlockAllocation(
    5510  VmaAllocator hAllocator,
    5511  VmaDeviceMemoryBlock* block,
    5512  VkDeviceSize offset)
    5513 {
    5514  VMA_ASSERT(block != VMA_NULL);
    5515  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5516 
    5517  // Move mapping reference counter from old block to new block.
    5518  if(block != m_BlockAllocation.m_Block)
    5519  {
    5520  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
    5521  if(IsPersistentMap())
    5522  ++mapRefCount;
    5523  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
    5524  block->Map(hAllocator, mapRefCount, VMA_NULL);
    5525  }
    5526 
    5527  m_BlockAllocation.m_Block = block;
    5528  m_BlockAllocation.m_Offset = offset;
    5529 }
    5530 
    5531 VkDeviceSize VmaAllocation_T::GetOffset() const
    5532 {
    5533  switch(m_Type)
    5534  {
    5535  case ALLOCATION_TYPE_BLOCK:
    5536  return m_BlockAllocation.m_Offset;
    5537  case ALLOCATION_TYPE_DEDICATED:
    5538  return 0;
    5539  default:
    5540  VMA_ASSERT(0);
    5541  return 0;
    5542  }
    5543 }
    5544 
    5545 VkDeviceMemory VmaAllocation_T::GetMemory() const
    5546 {
    5547  switch(m_Type)
    5548  {
    5549  case ALLOCATION_TYPE_BLOCK:
    5550  return m_BlockAllocation.m_Block->GetDeviceMemory();
    5551  case ALLOCATION_TYPE_DEDICATED:
    5552  return m_DedicatedAllocation.m_hMemory;
    5553  default:
    5554  VMA_ASSERT(0);
    5555  return VK_NULL_HANDLE;
    5556  }
    5557 }
    5558 
    5559 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
    5560 {
    5561  switch(m_Type)
    5562  {
    5563  case ALLOCATION_TYPE_BLOCK:
    5564  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
    5565  case ALLOCATION_TYPE_DEDICATED:
    5566  return m_DedicatedAllocation.m_MemoryTypeIndex;
    5567  default:
    5568  VMA_ASSERT(0);
    5569  return UINT32_MAX;
    5570  }
    5571 }
    5572 
    5573 void* VmaAllocation_T::GetMappedData() const
    5574 {
    5575  switch(m_Type)
    5576  {
    5577  case ALLOCATION_TYPE_BLOCK:
    5578  if(m_MapCount != 0)
    5579  {
    5580  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
    5581  VMA_ASSERT(pBlockData != VMA_NULL);
    5582  return (char*)pBlockData + m_BlockAllocation.m_Offset;
    5583  }
    5584  else
    5585  {
    5586  return VMA_NULL;
    5587  }
    5588  break;
    5589  case ALLOCATION_TYPE_DEDICATED:
    5590  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
    5591  return m_DedicatedAllocation.m_pMappedData;
    5592  default:
    5593  VMA_ASSERT(0);
    5594  return VMA_NULL;
    5595  }
    5596 }
    5597 
    5598 bool VmaAllocation_T::CanBecomeLost() const
    5599 {
    5600  switch(m_Type)
    5601  {
    5602  case ALLOCATION_TYPE_BLOCK:
    5603  return m_BlockAllocation.m_CanBecomeLost;
    5604  case ALLOCATION_TYPE_DEDICATED:
    5605  return false;
    5606  default:
    5607  VMA_ASSERT(0);
    5608  return false;
    5609  }
    5610 }
    5611 
    5612 VmaPool VmaAllocation_T::GetPool() const
    5613 {
    5614  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
    5615  return m_BlockAllocation.m_hPool;
    5616 }
    5617 
    5618 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    5619 {
    5620  VMA_ASSERT(CanBecomeLost());
    5621 
    5622  /*
    5623  Warning: This is a carefully designed algorithm.
    5624  Do not modify unless you really know what you're doing :)
    5625  */
    5626  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
    5627  for(;;)
    5628  {
    5629  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    5630  {
    5631  VMA_ASSERT(0);
    5632  return false;
    5633  }
    5634  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
    5635  {
    5636  return false;
    5637  }
    5638  else // Last use time earlier than current time.
    5639  {
    5640  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
    5641  {
    5642  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
    5643  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
    5644  return true;
    5645  }
    5646  }
    5647  }
    5648 }
    5649 
    5650 #if VMA_STATS_STRING_ENABLED
    5651 
    5652 // Correspond to values of enum VmaSuballocationType.
    5653 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
    5654  "FREE",
    5655  "UNKNOWN",
    5656  "BUFFER",
    5657  "IMAGE_UNKNOWN",
    5658  "IMAGE_LINEAR",
    5659  "IMAGE_OPTIMAL",
    5660 };
    5661 
    5662 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
    5663 {
    5664  json.WriteString("Type");
    5665  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
    5666 
    5667  json.WriteString("Size");
    5668  json.WriteNumber(m_Size);
    5669 
    5670  if(m_pUserData != VMA_NULL)
    5671  {
    5672  json.WriteString("UserData");
    5673  if(IsUserDataString())
    5674  {
    5675  json.WriteString((const char*)m_pUserData);
    5676  }
    5677  else
    5678  {
    5679  json.BeginString();
    5680  json.ContinueString_Pointer(m_pUserData);
    5681  json.EndString();
    5682  }
    5683  }
    5684 
    5685  json.WriteString("CreationFrameIndex");
    5686  json.WriteNumber(m_CreationFrameIndex);
    5687 
    5688  json.WriteString("LastUseFrameIndex");
    5689  json.WriteNumber(GetLastUseFrameIndex());
    5690 
    5691  if(m_BufferImageUsage != 0)
    5692  {
    5693  json.WriteString("Usage");
    5694  json.WriteNumber(m_BufferImageUsage);
    5695  }
    5696 }
    5697 
    5698 #endif
    5699 
    5700 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
    5701 {
    5702  VMA_ASSERT(IsUserDataString());
    5703  if(m_pUserData != VMA_NULL)
    5704  {
    5705  char* const oldStr = (char*)m_pUserData;
    5706  const size_t oldStrLen = strlen(oldStr);
    5707  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
    5708  m_pUserData = VMA_NULL;
    5709  }
    5710 }
    5711 
    5712 void VmaAllocation_T::BlockAllocMap()
    5713 {
    5714  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5715 
    5716  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5717  {
    5718  ++m_MapCount;
    5719  }
    5720  else
    5721  {
    5722  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
    5723  }
    5724 }
    5725 
    5726 void VmaAllocation_T::BlockAllocUnmap()
    5727 {
    5728  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
    5729 
    5730  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5731  {
    5732  --m_MapCount;
    5733  }
    5734  else
    5735  {
    5736  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
    5737  }
    5738 }
    5739 
    5740 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
    5741 {
    5742  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5743 
    5744  if(m_MapCount != 0)
    5745  {
    5746  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
    5747  {
    5748  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
    5749  *ppData = m_DedicatedAllocation.m_pMappedData;
    5750  ++m_MapCount;
    5751  return VK_SUCCESS;
    5752  }
    5753  else
    5754  {
    5755  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
    5756  return VK_ERROR_MEMORY_MAP_FAILED;
    5757  }
    5758  }
    5759  else
    5760  {
    5761  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    5762  hAllocator->m_hDevice,
    5763  m_DedicatedAllocation.m_hMemory,
    5764  0, // offset
    5765  VK_WHOLE_SIZE,
    5766  0, // flags
    5767  ppData);
    5768  if(result == VK_SUCCESS)
    5769  {
    5770  m_DedicatedAllocation.m_pMappedData = *ppData;
    5771  m_MapCount = 1;
    5772  }
    5773  return result;
    5774  }
    5775 }
    5776 
    5777 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
    5778 {
    5779  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
    5780 
    5781  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
    5782  {
    5783  --m_MapCount;
    5784  if(m_MapCount == 0)
    5785  {
    5786  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
    5787  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
    5788  hAllocator->m_hDevice,
    5789  m_DedicatedAllocation.m_hMemory);
    5790  }
    5791  }
    5792  else
    5793  {
    5794  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
    5795  }
    5796 }
    5797 
    5798 #if VMA_STATS_STRING_ENABLED
    5799 
    5800 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
    5801 {
    5802  json.BeginObject();
    5803 
    5804  json.WriteString("Blocks");
    5805  json.WriteNumber(stat.blockCount);
    5806 
    5807  json.WriteString("Allocations");
    5808  json.WriteNumber(stat.allocationCount);
    5809 
    5810  json.WriteString("UnusedRanges");
    5811  json.WriteNumber(stat.unusedRangeCount);
    5812 
    5813  json.WriteString("UsedBytes");
    5814  json.WriteNumber(stat.usedBytes);
    5815 
    5816  json.WriteString("UnusedBytes");
    5817  json.WriteNumber(stat.unusedBytes);
    5818 
    5819  if(stat.allocationCount > 1)
    5820  {
    5821  json.WriteString("AllocationSize");
    5822  json.BeginObject(true);
    5823  json.WriteString("Min");
    5824  json.WriteNumber(stat.allocationSizeMin);
    5825  json.WriteString("Avg");
    5826  json.WriteNumber(stat.allocationSizeAvg);
    5827  json.WriteString("Max");
    5828  json.WriteNumber(stat.allocationSizeMax);
    5829  json.EndObject();
    5830  }
    5831 
    5832  if(stat.unusedRangeCount > 1)
    5833  {
    5834  json.WriteString("UnusedRangeSize");
    5835  json.BeginObject(true);
    5836  json.WriteString("Min");
    5837  json.WriteNumber(stat.unusedRangeSizeMin);
    5838  json.WriteString("Avg");
    5839  json.WriteNumber(stat.unusedRangeSizeAvg);
    5840  json.WriteString("Max");
    5841  json.WriteNumber(stat.unusedRangeSizeMax);
    5842  json.EndObject();
    5843  }
    5844 
    5845  json.EndObject();
    5846 }
    5847 
    5848 #endif // #if VMA_STATS_STRING_ENABLED
    5849 
    5850 struct VmaSuballocationItemSizeLess
    5851 {
    5852  bool operator()(
    5853  const VmaSuballocationList::iterator lhs,
    5854  const VmaSuballocationList::iterator rhs) const
    5855  {
    5856  return lhs->size < rhs->size;
    5857  }
    5858  bool operator()(
    5859  const VmaSuballocationList::iterator lhs,
    5860  VkDeviceSize rhsSize) const
    5861  {
    5862  return lhs->size < rhsSize;
    5863  }
    5864 };
    5865 
    5867 // class VmaBlockMetadata
    5868 
    5869 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
    5870  m_Size(0),
    5871  m_FreeCount(0),
    5872  m_SumFreeSize(0),
    5873  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
    5874  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
    5875 {
    5876 }
    5877 
    5878 VmaBlockMetadata::~VmaBlockMetadata()
    5879 {
    5880 }
    5881 
    5882 void VmaBlockMetadata::Init(VkDeviceSize size)
    5883 {
    5884  m_Size = size;
    5885  m_FreeCount = 1;
    5886  m_SumFreeSize = size;
    5887 
    5888  VmaSuballocation suballoc = {};
    5889  suballoc.offset = 0;
    5890  suballoc.size = size;
    5891  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    5892  suballoc.hAllocation = VK_NULL_HANDLE;
    5893 
    5894  m_Suballocations.push_back(suballoc);
    5895  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
    5896  --suballocItem;
    5897  m_FreeSuballocationsBySize.push_back(suballocItem);
    5898 }
    5899 
    5900 bool VmaBlockMetadata::Validate() const
    5901 {
    5902  if(m_Suballocations.empty())
    5903  {
    5904  return false;
    5905  }
    5906 
    5907  // Expected offset of new suballocation as calculates from previous ones.
    5908  VkDeviceSize calculatedOffset = 0;
    5909  // Expected number of free suballocations as calculated from traversing their list.
    5910  uint32_t calculatedFreeCount = 0;
    5911  // Expected sum size of free suballocations as calculated from traversing their list.
    5912  VkDeviceSize calculatedSumFreeSize = 0;
    5913  // Expected number of free suballocations that should be registered in
    5914  // m_FreeSuballocationsBySize calculated from traversing their list.
    5915  size_t freeSuballocationsToRegister = 0;
    5916  // True if previous visited suballocation was free.
    5917  bool prevFree = false;
    5918 
    5919  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    5920  suballocItem != m_Suballocations.cend();
    5921  ++suballocItem)
    5922  {
    5923  const VmaSuballocation& subAlloc = *suballocItem;
    5924 
    5925  // Actual offset of this suballocation doesn't match expected one.
    5926  if(subAlloc.offset != calculatedOffset)
    5927  {
    5928  return false;
    5929  }
    5930 
    5931  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
    5932  // Two adjacent free suballocations are invalid. They should be merged.
    5933  if(prevFree && currFree)
    5934  {
    5935  return false;
    5936  }
    5937 
    5938  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
    5939  {
    5940  return false;
    5941  }
    5942 
    5943  if(currFree)
    5944  {
    5945  calculatedSumFreeSize += subAlloc.size;
    5946  ++calculatedFreeCount;
    5947  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    5948  {
    5949  ++freeSuballocationsToRegister;
    5950  }
    5951 
    5952  // Margin required between allocations - every free space must be at least that large.
    5953  if(subAlloc.size < VMA_DEBUG_MARGIN)
    5954  {
    5955  return false;
    5956  }
    5957  }
    5958  else
    5959  {
    5960  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
    5961  {
    5962  return false;
    5963  }
    5964  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
    5965  {
    5966  return false;
    5967  }
    5968 
    5969  // Margin required between allocations - previous allocation must be free.
    5970  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
    5971  {
    5972  return false;
    5973  }
    5974  }
    5975 
    5976  calculatedOffset += subAlloc.size;
    5977  prevFree = currFree;
    5978  }
    5979 
    5980  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
    5981  // match expected one.
    5982  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
    5983  {
    5984  return false;
    5985  }
    5986 
    5987  VkDeviceSize lastSize = 0;
    5988  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
    5989  {
    5990  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
    5991 
    5992  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
    5993  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
    5994  {
    5995  return false;
    5996  }
    5997  // They must be sorted by size ascending.
    5998  if(suballocItem->size < lastSize)
    5999  {
    6000  return false;
    6001  }
    6002 
    6003  lastSize = suballocItem->size;
    6004  }
    6005 
    6006  // Check if totals match calculacted values.
    6007  if(!ValidateFreeSuballocationList() ||
    6008  (calculatedOffset != m_Size) ||
    6009  (calculatedSumFreeSize != m_SumFreeSize) ||
    6010  (calculatedFreeCount != m_FreeCount))
    6011  {
    6012  return false;
    6013  }
    6014 
    6015  return true;
    6016 }
    6017 
    6018 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
    6019 {
    6020  if(!m_FreeSuballocationsBySize.empty())
    6021  {
    6022  return m_FreeSuballocationsBySize.back()->size;
    6023  }
    6024  else
    6025  {
    6026  return 0;
    6027  }
    6028 }
    6029 
    6030 bool VmaBlockMetadata::IsEmpty() const
    6031 {
    6032  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
    6033 }
    6034 
    6035 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
    6036 {
    6037  outInfo.blockCount = 1;
    6038 
    6039  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    6040  outInfo.allocationCount = rangeCount - m_FreeCount;
    6041  outInfo.unusedRangeCount = m_FreeCount;
    6042 
    6043  outInfo.unusedBytes = m_SumFreeSize;
    6044  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
    6045 
    6046  outInfo.allocationSizeMin = UINT64_MAX;
    6047  outInfo.allocationSizeMax = 0;
    6048  outInfo.unusedRangeSizeMin = UINT64_MAX;
    6049  outInfo.unusedRangeSizeMax = 0;
    6050 
    6051  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    6052  suballocItem != m_Suballocations.cend();
    6053  ++suballocItem)
    6054  {
    6055  const VmaSuballocation& suballoc = *suballocItem;
    6056  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
    6057  {
    6058  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
    6059  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
    6060  }
    6061  else
    6062  {
    6063  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
    6064  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
    6065  }
    6066  }
    6067 }
    6068 
    6069 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
    6070 {
    6071  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
    6072 
    6073  inoutStats.size += m_Size;
    6074  inoutStats.unusedSize += m_SumFreeSize;
    6075  inoutStats.allocationCount += rangeCount - m_FreeCount;
    6076  inoutStats.unusedRangeCount += m_FreeCount;
    6077  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
    6078 }
    6079 
    6080 #if VMA_STATS_STRING_ENABLED
    6081 
    6082 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
    6083 {
    6084  json.BeginObject();
    6085 
    6086  json.WriteString("TotalBytes");
    6087  json.WriteNumber(m_Size);
    6088 
    6089  json.WriteString("UnusedBytes");
    6090  json.WriteNumber(m_SumFreeSize);
    6091 
    6092  json.WriteString("Allocations");
    6093  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
    6094 
    6095  json.WriteString("UnusedRanges");
    6096  json.WriteNumber(m_FreeCount);
    6097 
    6098  json.WriteString("Suballocations");
    6099  json.BeginArray();
    6100  size_t i = 0;
    6101  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
    6102  suballocItem != m_Suballocations.cend();
    6103  ++suballocItem, ++i)
    6104  {
    6105  json.BeginObject(true);
    6106 
    6107  json.WriteString("Offset");
    6108  json.WriteNumber(suballocItem->offset);
    6109 
    6110  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6111  {
    6112  json.WriteString("Type");
    6113  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
    6114 
    6115  json.WriteString("Size");
    6116  json.WriteNumber(suballocItem->size);
    6117  }
    6118  else
    6119  {
    6120  suballocItem->hAllocation->PrintParameters(json);
    6121  }
    6122 
    6123  json.EndObject();
    6124  }
    6125  json.EndArray();
    6126 
    6127  json.EndObject();
    6128 }
    6129 
    6130 #endif // #if VMA_STATS_STRING_ENABLED
    6131 
    6132 /*
    6133 How many suitable free suballocations to analyze before choosing best one.
    6134 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
    6135  be chosen.
    6136 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
    6137  suballocations will be analized and best one will be chosen.
    6138 - Any other value is also acceptable.
    6139 */
    6140 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
    6141 
    6142 bool VmaBlockMetadata::CreateAllocationRequest(
    6143  uint32_t currentFrameIndex,
    6144  uint32_t frameInUseCount,
    6145  VkDeviceSize bufferImageGranularity,
    6146  VkDeviceSize allocSize,
    6147  VkDeviceSize allocAlignment,
    6148  VmaSuballocationType allocType,
    6149  bool canMakeOtherLost,
    6150  VmaAllocationRequest* pAllocationRequest)
    6151 {
    6152  VMA_ASSERT(allocSize > 0);
    6153  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6154  VMA_ASSERT(pAllocationRequest != VMA_NULL);
    6155  VMA_HEAVY_ASSERT(Validate());
    6156 
    6157  // There is not enough total free space in this block to fullfill the request: Early return.
    6158  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
    6159  {
    6160  return false;
    6161  }
    6162 
    6163  // New algorithm, efficiently searching freeSuballocationsBySize.
    6164  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
    6165  if(freeSuballocCount > 0)
    6166  {
    6167  if(VMA_BEST_FIT)
    6168  {
    6169  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
    6170  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6171  m_FreeSuballocationsBySize.data(),
    6172  m_FreeSuballocationsBySize.data() + freeSuballocCount,
    6173  allocSize + 2 * VMA_DEBUG_MARGIN,
    6174  VmaSuballocationItemSizeLess());
    6175  size_t index = it - m_FreeSuballocationsBySize.data();
    6176  for(; index < freeSuballocCount; ++index)
    6177  {
    6178  if(CheckAllocation(
    6179  currentFrameIndex,
    6180  frameInUseCount,
    6181  bufferImageGranularity,
    6182  allocSize,
    6183  allocAlignment,
    6184  allocType,
    6185  m_FreeSuballocationsBySize[index],
    6186  false, // canMakeOtherLost
    6187  &pAllocationRequest->offset,
    6188  &pAllocationRequest->itemsToMakeLostCount,
    6189  &pAllocationRequest->sumFreeSize,
    6190  &pAllocationRequest->sumItemSize))
    6191  {
    6192  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6193  return true;
    6194  }
    6195  }
    6196  }
    6197  else
    6198  {
    6199  // Search staring from biggest suballocations.
    6200  for(size_t index = freeSuballocCount; index--; )
    6201  {
    6202  if(CheckAllocation(
    6203  currentFrameIndex,
    6204  frameInUseCount,
    6205  bufferImageGranularity,
    6206  allocSize,
    6207  allocAlignment,
    6208  allocType,
    6209  m_FreeSuballocationsBySize[index],
    6210  false, // canMakeOtherLost
    6211  &pAllocationRequest->offset,
    6212  &pAllocationRequest->itemsToMakeLostCount,
    6213  &pAllocationRequest->sumFreeSize,
    6214  &pAllocationRequest->sumItemSize))
    6215  {
    6216  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
    6217  return true;
    6218  }
    6219  }
    6220  }
    6221  }
    6222 
    6223  if(canMakeOtherLost)
    6224  {
    6225  // Brute-force algorithm. TODO: Come up with something better.
    6226 
    6227  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
    6228  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
    6229 
    6230  VmaAllocationRequest tmpAllocRequest = {};
    6231  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
    6232  suballocIt != m_Suballocations.end();
    6233  ++suballocIt)
    6234  {
    6235  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
    6236  suballocIt->hAllocation->CanBecomeLost())
    6237  {
    6238  if(CheckAllocation(
    6239  currentFrameIndex,
    6240  frameInUseCount,
    6241  bufferImageGranularity,
    6242  allocSize,
    6243  allocAlignment,
    6244  allocType,
    6245  suballocIt,
    6246  canMakeOtherLost,
    6247  &tmpAllocRequest.offset,
    6248  &tmpAllocRequest.itemsToMakeLostCount,
    6249  &tmpAllocRequest.sumFreeSize,
    6250  &tmpAllocRequest.sumItemSize))
    6251  {
    6252  tmpAllocRequest.item = suballocIt;
    6253 
    6254  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
    6255  {
    6256  *pAllocationRequest = tmpAllocRequest;
    6257  }
    6258  }
    6259  }
    6260  }
    6261 
    6262  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
    6263  {
    6264  return true;
    6265  }
    6266  }
    6267 
    6268  return false;
    6269 }
    6270 
    6271 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
    6272  uint32_t currentFrameIndex,
    6273  uint32_t frameInUseCount,
    6274  VmaAllocationRequest* pAllocationRequest)
    6275 {
    6276  while(pAllocationRequest->itemsToMakeLostCount > 0)
    6277  {
    6278  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
    6279  {
    6280  ++pAllocationRequest->item;
    6281  }
    6282  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6283  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
    6284  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
    6285  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6286  {
    6287  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
    6288  --pAllocationRequest->itemsToMakeLostCount;
    6289  }
    6290  else
    6291  {
    6292  return false;
    6293  }
    6294  }
    6295 
    6296  VMA_HEAVY_ASSERT(Validate());
    6297  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
    6298  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6299 
    6300  return true;
    6301 }
    6302 
    6303 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
    6304 {
    6305  uint32_t lostAllocationCount = 0;
    6306  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6307  it != m_Suballocations.end();
    6308  ++it)
    6309  {
    6310  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
    6311  it->hAllocation->CanBecomeLost() &&
    6312  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
    6313  {
    6314  it = FreeSuballocation(it);
    6315  ++lostAllocationCount;
    6316  }
    6317  }
    6318  return lostAllocationCount;
    6319 }
    6320 
    6321 VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData)
    6322 {
    6323  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
    6324  it != m_Suballocations.end();
    6325  ++it)
    6326  {
    6327  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6328  {
    6329  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
    6330  {
    6331  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
    6332  return VK_ERROR_VALIDATION_FAILED_EXT;
    6333  }
    6334  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
    6335  {
    6336  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
    6337  return VK_ERROR_VALIDATION_FAILED_EXT;
    6338  }
    6339  }
    6340  }
    6341 
    6342  return VK_SUCCESS;
    6343 }
    6344 
    6345 void VmaBlockMetadata::Alloc(
    6346  const VmaAllocationRequest& request,
    6347  VmaSuballocationType type,
    6348  VkDeviceSize allocSize,
    6349  VmaAllocation hAllocation)
    6350 {
    6351  VMA_ASSERT(request.item != m_Suballocations.end());
    6352  VmaSuballocation& suballoc = *request.item;
    6353  // Given suballocation is a free block.
    6354  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6355  // Given offset is inside this suballocation.
    6356  VMA_ASSERT(request.offset >= suballoc.offset);
    6357  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
    6358  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
    6359  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
    6360 
    6361  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
    6362  // it to become used.
    6363  UnregisterFreeSuballocation(request.item);
    6364 
    6365  suballoc.offset = request.offset;
    6366  suballoc.size = allocSize;
    6367  suballoc.type = type;
    6368  suballoc.hAllocation = hAllocation;
    6369 
    6370  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
    6371  if(paddingEnd)
    6372  {
    6373  VmaSuballocation paddingSuballoc = {};
    6374  paddingSuballoc.offset = request.offset + allocSize;
    6375  paddingSuballoc.size = paddingEnd;
    6376  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6377  VmaSuballocationList::iterator next = request.item;
    6378  ++next;
    6379  const VmaSuballocationList::iterator paddingEndItem =
    6380  m_Suballocations.insert(next, paddingSuballoc);
    6381  RegisterFreeSuballocation(paddingEndItem);
    6382  }
    6383 
    6384  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
    6385  if(paddingBegin)
    6386  {
    6387  VmaSuballocation paddingSuballoc = {};
    6388  paddingSuballoc.offset = request.offset - paddingBegin;
    6389  paddingSuballoc.size = paddingBegin;
    6390  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6391  const VmaSuballocationList::iterator paddingBeginItem =
    6392  m_Suballocations.insert(request.item, paddingSuballoc);
    6393  RegisterFreeSuballocation(paddingBeginItem);
    6394  }
    6395 
    6396  // Update totals.
    6397  m_FreeCount = m_FreeCount - 1;
    6398  if(paddingBegin > 0)
    6399  {
    6400  ++m_FreeCount;
    6401  }
    6402  if(paddingEnd > 0)
    6403  {
    6404  ++m_FreeCount;
    6405  }
    6406  m_SumFreeSize -= allocSize;
    6407 }
    6408 
    6409 void VmaBlockMetadata::Free(const VmaAllocation allocation)
    6410 {
    6411  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6412  suballocItem != m_Suballocations.end();
    6413  ++suballocItem)
    6414  {
    6415  VmaSuballocation& suballoc = *suballocItem;
    6416  if(suballoc.hAllocation == allocation)
    6417  {
    6418  FreeSuballocation(suballocItem);
    6419  VMA_HEAVY_ASSERT(Validate());
    6420  return;
    6421  }
    6422  }
    6423  VMA_ASSERT(0 && "Not found!");
    6424 }
    6425 
    6426 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
    6427 {
    6428  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
    6429  suballocItem != m_Suballocations.end();
    6430  ++suballocItem)
    6431  {
    6432  VmaSuballocation& suballoc = *suballocItem;
    6433  if(suballoc.offset == offset)
    6434  {
    6435  FreeSuballocation(suballocItem);
    6436  return;
    6437  }
    6438  }
    6439  VMA_ASSERT(0 && "Not found!");
    6440 }
    6441 
    6442 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
    6443 {
    6444  VkDeviceSize lastSize = 0;
    6445  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
    6446  {
    6447  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
    6448 
    6449  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
    6450  {
    6451  VMA_ASSERT(0);
    6452  return false;
    6453  }
    6454  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6455  {
    6456  VMA_ASSERT(0);
    6457  return false;
    6458  }
    6459  if(it->size < lastSize)
    6460  {
    6461  VMA_ASSERT(0);
    6462  return false;
    6463  }
    6464 
    6465  lastSize = it->size;
    6466  }
    6467  return true;
    6468 }
    6469 
    6470 bool VmaBlockMetadata::CheckAllocation(
    6471  uint32_t currentFrameIndex,
    6472  uint32_t frameInUseCount,
    6473  VkDeviceSize bufferImageGranularity,
    6474  VkDeviceSize allocSize,
    6475  VkDeviceSize allocAlignment,
    6476  VmaSuballocationType allocType,
    6477  VmaSuballocationList::const_iterator suballocItem,
    6478  bool canMakeOtherLost,
    6479  VkDeviceSize* pOffset,
    6480  size_t* itemsToMakeLostCount,
    6481  VkDeviceSize* pSumFreeSize,
    6482  VkDeviceSize* pSumItemSize) const
    6483 {
    6484  VMA_ASSERT(allocSize > 0);
    6485  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
    6486  VMA_ASSERT(suballocItem != m_Suballocations.cend());
    6487  VMA_ASSERT(pOffset != VMA_NULL);
    6488 
    6489  *itemsToMakeLostCount = 0;
    6490  *pSumFreeSize = 0;
    6491  *pSumItemSize = 0;
    6492 
    6493  if(canMakeOtherLost)
    6494  {
    6495  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6496  {
    6497  *pSumFreeSize = suballocItem->size;
    6498  }
    6499  else
    6500  {
    6501  if(suballocItem->hAllocation->CanBecomeLost() &&
    6502  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6503  {
    6504  ++*itemsToMakeLostCount;
    6505  *pSumItemSize = suballocItem->size;
    6506  }
    6507  else
    6508  {
    6509  return false;
    6510  }
    6511  }
    6512 
    6513  // Remaining size is too small for this request: Early return.
    6514  if(m_Size - suballocItem->offset < allocSize)
    6515  {
    6516  return false;
    6517  }
    6518 
    6519  // Start from offset equal to beginning of this suballocation.
    6520  *pOffset = suballocItem->offset;
    6521 
    6522  // Apply VMA_DEBUG_MARGIN at the beginning.
    6523  if(VMA_DEBUG_MARGIN > 0)
    6524  {
    6525  *pOffset += VMA_DEBUG_MARGIN;
    6526  }
    6527 
    6528  // Apply alignment.
    6529  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6530 
    6531  // Check previous suballocations for BufferImageGranularity conflicts.
    6532  // Make bigger alignment if necessary.
    6533  if(bufferImageGranularity > 1)
    6534  {
    6535  bool bufferImageGranularityConflict = false;
    6536  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6537  while(prevSuballocItem != m_Suballocations.cbegin())
    6538  {
    6539  --prevSuballocItem;
    6540  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6541  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6542  {
    6543  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6544  {
    6545  bufferImageGranularityConflict = true;
    6546  break;
    6547  }
    6548  }
    6549  else
    6550  // Already on previous page.
    6551  break;
    6552  }
    6553  if(bufferImageGranularityConflict)
    6554  {
    6555  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6556  }
    6557  }
    6558 
    6559  // Now that we have final *pOffset, check if we are past suballocItem.
    6560  // If yes, return false - this function should be called for another suballocItem as starting point.
    6561  if(*pOffset >= suballocItem->offset + suballocItem->size)
    6562  {
    6563  return false;
    6564  }
    6565 
    6566  // Calculate padding at the beginning based on current offset.
    6567  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
    6568 
    6569  // Calculate required margin at the end.
    6570  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6571 
    6572  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
    6573  // Another early return check.
    6574  if(suballocItem->offset + totalSize > m_Size)
    6575  {
    6576  return false;
    6577  }
    6578 
    6579  // Advance lastSuballocItem until desired size is reached.
    6580  // Update itemsToMakeLostCount.
    6581  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
    6582  if(totalSize > suballocItem->size)
    6583  {
    6584  VkDeviceSize remainingSize = totalSize - suballocItem->size;
    6585  while(remainingSize > 0)
    6586  {
    6587  ++lastSuballocItem;
    6588  if(lastSuballocItem == m_Suballocations.cend())
    6589  {
    6590  return false;
    6591  }
    6592  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6593  {
    6594  *pSumFreeSize += lastSuballocItem->size;
    6595  }
    6596  else
    6597  {
    6598  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
    6599  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
    6600  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6601  {
    6602  ++*itemsToMakeLostCount;
    6603  *pSumItemSize += lastSuballocItem->size;
    6604  }
    6605  else
    6606  {
    6607  return false;
    6608  }
    6609  }
    6610  remainingSize = (lastSuballocItem->size < remainingSize) ?
    6611  remainingSize - lastSuballocItem->size : 0;
    6612  }
    6613  }
    6614 
    6615  // Check next suballocations for BufferImageGranularity conflicts.
    6616  // If conflict exists, we must mark more allocations lost or fail.
    6617  if(bufferImageGranularity > 1)
    6618  {
    6619  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
    6620  ++nextSuballocItem;
    6621  while(nextSuballocItem != m_Suballocations.cend())
    6622  {
    6623  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6624  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6625  {
    6626  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6627  {
    6628  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
    6629  if(nextSuballoc.hAllocation->CanBecomeLost() &&
    6630  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
    6631  {
    6632  ++*itemsToMakeLostCount;
    6633  }
    6634  else
    6635  {
    6636  return false;
    6637  }
    6638  }
    6639  }
    6640  else
    6641  {
    6642  // Already on next page.
    6643  break;
    6644  }
    6645  ++nextSuballocItem;
    6646  }
    6647  }
    6648  }
    6649  else
    6650  {
    6651  const VmaSuballocation& suballoc = *suballocItem;
    6652  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
    6653 
    6654  *pSumFreeSize = suballoc.size;
    6655 
    6656  // Size of this suballocation is too small for this request: Early return.
    6657  if(suballoc.size < allocSize)
    6658  {
    6659  return false;
    6660  }
    6661 
    6662  // Start from offset equal to beginning of this suballocation.
    6663  *pOffset = suballoc.offset;
    6664 
    6665  // Apply VMA_DEBUG_MARGIN at the beginning.
    6666  if(VMA_DEBUG_MARGIN > 0)
    6667  {
    6668  *pOffset += VMA_DEBUG_MARGIN;
    6669  }
    6670 
    6671  // Apply alignment.
    6672  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
    6673 
    6674  // Check previous suballocations for BufferImageGranularity conflicts.
    6675  // Make bigger alignment if necessary.
    6676  if(bufferImageGranularity > 1)
    6677  {
    6678  bool bufferImageGranularityConflict = false;
    6679  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
    6680  while(prevSuballocItem != m_Suballocations.cbegin())
    6681  {
    6682  --prevSuballocItem;
    6683  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
    6684  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
    6685  {
    6686  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
    6687  {
    6688  bufferImageGranularityConflict = true;
    6689  break;
    6690  }
    6691  }
    6692  else
    6693  // Already on previous page.
    6694  break;
    6695  }
    6696  if(bufferImageGranularityConflict)
    6697  {
    6698  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
    6699  }
    6700  }
    6701 
    6702  // Calculate padding at the beginning based on current offset.
    6703  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
    6704 
    6705  // Calculate required margin at the end.
    6706  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
    6707 
    6708  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
    6709  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
    6710  {
    6711  return false;
    6712  }
    6713 
    6714  // Check next suballocations for BufferImageGranularity conflicts.
    6715  // If conflict exists, allocation cannot be made here.
    6716  if(bufferImageGranularity > 1)
    6717  {
    6718  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
    6719  ++nextSuballocItem;
    6720  while(nextSuballocItem != m_Suballocations.cend())
    6721  {
    6722  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
    6723  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
    6724  {
    6725  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
    6726  {
    6727  return false;
    6728  }
    6729  }
    6730  else
    6731  {
    6732  // Already on next page.
    6733  break;
    6734  }
    6735  ++nextSuballocItem;
    6736  }
    6737  }
    6738  }
    6739 
    6740  // All tests passed: Success. pOffset is already filled.
    6741  return true;
    6742 }
    6743 
    6744 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
    6745 {
    6746  VMA_ASSERT(item != m_Suballocations.end());
    6747  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6748 
    6749  VmaSuballocationList::iterator nextItem = item;
    6750  ++nextItem;
    6751  VMA_ASSERT(nextItem != m_Suballocations.end());
    6752  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
    6753 
    6754  item->size += nextItem->size;
    6755  --m_FreeCount;
    6756  m_Suballocations.erase(nextItem);
    6757 }
    6758 
    6759 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
    6760 {
    6761  // Change this suballocation to be marked as free.
    6762  VmaSuballocation& suballoc = *suballocItem;
    6763  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
    6764  suballoc.hAllocation = VK_NULL_HANDLE;
    6765 
    6766  // Update totals.
    6767  ++m_FreeCount;
    6768  m_SumFreeSize += suballoc.size;
    6769 
    6770  // Merge with previous and/or next suballocation if it's also free.
    6771  bool mergeWithNext = false;
    6772  bool mergeWithPrev = false;
    6773 
    6774  VmaSuballocationList::iterator nextItem = suballocItem;
    6775  ++nextItem;
    6776  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
    6777  {
    6778  mergeWithNext = true;
    6779  }
    6780 
    6781  VmaSuballocationList::iterator prevItem = suballocItem;
    6782  if(suballocItem != m_Suballocations.begin())
    6783  {
    6784  --prevItem;
    6785  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
    6786  {
    6787  mergeWithPrev = true;
    6788  }
    6789  }
    6790 
    6791  if(mergeWithNext)
    6792  {
    6793  UnregisterFreeSuballocation(nextItem);
    6794  MergeFreeWithNext(suballocItem);
    6795  }
    6796 
    6797  if(mergeWithPrev)
    6798  {
    6799  UnregisterFreeSuballocation(prevItem);
    6800  MergeFreeWithNext(prevItem);
    6801  RegisterFreeSuballocation(prevItem);
    6802  return prevItem;
    6803  }
    6804  else
    6805  {
    6806  RegisterFreeSuballocation(suballocItem);
    6807  return suballocItem;
    6808  }
    6809 }
    6810 
    6811 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
    6812 {
    6813  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6814  VMA_ASSERT(item->size > 0);
    6815 
    6816  // You may want to enable this validation at the beginning or at the end of
    6817  // this function, depending on what do you want to check.
    6818  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6819 
    6820  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6821  {
    6822  if(m_FreeSuballocationsBySize.empty())
    6823  {
    6824  m_FreeSuballocationsBySize.push_back(item);
    6825  }
    6826  else
    6827  {
    6828  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
    6829  }
    6830  }
    6831 
    6832  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6833 }
    6834 
    6835 
    6836 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
    6837 {
    6838  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
    6839  VMA_ASSERT(item->size > 0);
    6840 
    6841  // You may want to enable this validation at the beginning or at the end of
    6842  // this function, depending on what do you want to check.
    6843  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6844 
    6845  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
    6846  {
    6847  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
    6848  m_FreeSuballocationsBySize.data(),
    6849  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
    6850  item,
    6851  VmaSuballocationItemSizeLess());
    6852  for(size_t index = it - m_FreeSuballocationsBySize.data();
    6853  index < m_FreeSuballocationsBySize.size();
    6854  ++index)
    6855  {
    6856  if(m_FreeSuballocationsBySize[index] == item)
    6857  {
    6858  VmaVectorRemove(m_FreeSuballocationsBySize, index);
    6859  return;
    6860  }
    6861  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
    6862  }
    6863  VMA_ASSERT(0 && "Not found.");
    6864  }
    6865 
    6866  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
    6867 }
    6868 
    6870 // class VmaDeviceMemoryBlock
    6871 
    6872 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
    6873  m_Metadata(hAllocator),
    6874  m_MemoryTypeIndex(UINT32_MAX),
    6875  m_Id(0),
    6876  m_hMemory(VK_NULL_HANDLE),
    6877  m_MapCount(0),
    6878  m_pMappedData(VMA_NULL)
    6879 {
    6880 }
    6881 
    6882 void VmaDeviceMemoryBlock::Init(
    6883  uint32_t newMemoryTypeIndex,
    6884  VkDeviceMemory newMemory,
    6885  VkDeviceSize newSize,
    6886  uint32_t id)
    6887 {
    6888  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
    6889 
    6890  m_MemoryTypeIndex = newMemoryTypeIndex;
    6891  m_Id = id;
    6892  m_hMemory = newMemory;
    6893 
    6894  m_Metadata.Init(newSize);
    6895 }
    6896 
    6897 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
    6898 {
    6899  // This is the most important assert in the entire library.
    6900  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
    6901  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
    6902 
    6903  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
    6904  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
    6905  m_hMemory = VK_NULL_HANDLE;
    6906 }
    6907 
    6908 bool VmaDeviceMemoryBlock::Validate() const
    6909 {
    6910  if((m_hMemory == VK_NULL_HANDLE) ||
    6911  (m_Metadata.GetSize() == 0))
    6912  {
    6913  return false;
    6914  }
    6915 
    6916  return m_Metadata.Validate();
    6917 }
    6918 
    6919 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
    6920 {
    6921  void* pData = nullptr;
    6922  VkResult res = Map(hAllocator, 1, &pData);
    6923  if(res != VK_SUCCESS)
    6924  {
    6925  return res;
    6926  }
    6927 
    6928  res = m_Metadata.CheckCorruption(pData);
    6929 
    6930  Unmap(hAllocator, 1);
    6931 
    6932  return res;
    6933 }
    6934 
    6935 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
    6936 {
    6937  if(count == 0)
    6938  {
    6939  return VK_SUCCESS;
    6940  }
    6941 
    6942  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6943  if(m_MapCount != 0)
    6944  {
    6945  m_MapCount += count;
    6946  VMA_ASSERT(m_pMappedData != VMA_NULL);
    6947  if(ppData != VMA_NULL)
    6948  {
    6949  *ppData = m_pMappedData;
    6950  }
    6951  return VK_SUCCESS;
    6952  }
    6953  else
    6954  {
    6955  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
    6956  hAllocator->m_hDevice,
    6957  m_hMemory,
    6958  0, // offset
    6959  VK_WHOLE_SIZE,
    6960  0, // flags
    6961  &m_pMappedData);
    6962  if(result == VK_SUCCESS)
    6963  {
    6964  if(ppData != VMA_NULL)
    6965  {
    6966  *ppData = m_pMappedData;
    6967  }
    6968  m_MapCount = count;
    6969  }
    6970  return result;
    6971  }
    6972 }
    6973 
    6974 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
    6975 {
    6976  if(count == 0)
    6977  {
    6978  return;
    6979  }
    6980 
    6981  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    6982  if(m_MapCount >= count)
    6983  {
    6984  m_MapCount -= count;
    6985  if(m_MapCount == 0)
    6986  {
    6987  m_pMappedData = VMA_NULL;
    6988  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
    6989  }
    6990  }
    6991  else
    6992  {
    6993  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
    6994  }
    6995 }
    6996 
    6997 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    6998 {
    6999  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    7000  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    7001 
    7002  void* pData;
    7003  VkResult res = Map(hAllocator, 1, &pData);
    7004  if(res != VK_SUCCESS)
    7005  {
    7006  return res;
    7007  }
    7008 
    7009  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
    7010  VmaWriteMagicValue(pData, allocOffset + allocSize);
    7011 
    7012  Unmap(hAllocator, 1);
    7013 
    7014  return VK_SUCCESS;
    7015 }
    7016 
    7017 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
    7018 {
    7019  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
    7020  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
    7021 
    7022  void* pData;
    7023  VkResult res = Map(hAllocator, 1, &pData);
    7024  if(res != VK_SUCCESS)
    7025  {
    7026  return res;
    7027  }
    7028 
    7029  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
    7030  {
    7031  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
    7032  }
    7033  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
    7034  {
    7035  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
    7036  }
    7037 
    7038  Unmap(hAllocator, 1);
    7039 
    7040  return VK_SUCCESS;
    7041 }
    7042 
    7043 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
    7044  const VmaAllocator hAllocator,
    7045  const VmaAllocation hAllocation,
    7046  VkBuffer hBuffer)
    7047 {
    7048  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    7049  hAllocation->GetBlock() == this);
    7050  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    7051  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    7052  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
    7053  hAllocator->m_hDevice,
    7054  hBuffer,
    7055  m_hMemory,
    7056  hAllocation->GetOffset());
    7057 }
    7058 
    7059 VkResult VmaDeviceMemoryBlock::BindImageMemory(
    7060  const VmaAllocator hAllocator,
    7061  const VmaAllocation hAllocation,
    7062  VkImage hImage)
    7063 {
    7064  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
    7065  hAllocation->GetBlock() == this);
    7066  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
    7067  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
    7068  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
    7069  hAllocator->m_hDevice,
    7070  hImage,
    7071  m_hMemory,
    7072  hAllocation->GetOffset());
    7073 }
    7074 
    7075 static void InitStatInfo(VmaStatInfo& outInfo)
    7076 {
    7077  memset(&outInfo, 0, sizeof(outInfo));
    7078  outInfo.allocationSizeMin = UINT64_MAX;
    7079  outInfo.unusedRangeSizeMin = UINT64_MAX;
    7080 }
    7081 
    7082 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
    7083 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
    7084 {
    7085  inoutInfo.blockCount += srcInfo.blockCount;
    7086  inoutInfo.allocationCount += srcInfo.allocationCount;
    7087  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
    7088  inoutInfo.usedBytes += srcInfo.usedBytes;
    7089  inoutInfo.unusedBytes += srcInfo.unusedBytes;
    7090  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
    7091  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
    7092  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
    7093  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
    7094 }
    7095 
    7096 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
    7097 {
    7098  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
    7099  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
    7100  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
    7101  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
    7102 }
    7103 
    7104 VmaPool_T::VmaPool_T(
    7105  VmaAllocator hAllocator,
    7106  const VmaPoolCreateInfo& createInfo) :
    7107  m_BlockVector(
    7108  hAllocator,
    7109  createInfo.memoryTypeIndex,
    7110  createInfo.blockSize,
    7111  createInfo.minBlockCount,
    7112  createInfo.maxBlockCount,
    7113  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
    7114  createInfo.frameInUseCount,
    7115  true), // isCustomPool
    7116  m_Id(0)
    7117 {
    7118 }
    7119 
    7120 VmaPool_T::~VmaPool_T()
    7121 {
    7122 }
    7123 
    7124 #if VMA_STATS_STRING_ENABLED
    7125 
    7126 #endif // #if VMA_STATS_STRING_ENABLED
    7127 
    7128 VmaBlockVector::VmaBlockVector(
    7129  VmaAllocator hAllocator,
    7130  uint32_t memoryTypeIndex,
    7131  VkDeviceSize preferredBlockSize,
    7132  size_t minBlockCount,
    7133  size_t maxBlockCount,
    7134  VkDeviceSize bufferImageGranularity,
    7135  uint32_t frameInUseCount,
    7136  bool isCustomPool) :
    7137  m_hAllocator(hAllocator),
    7138  m_MemoryTypeIndex(memoryTypeIndex),
    7139  m_PreferredBlockSize(preferredBlockSize),
    7140  m_MinBlockCount(minBlockCount),
    7141  m_MaxBlockCount(maxBlockCount),
    7142  m_BufferImageGranularity(bufferImageGranularity),
    7143  m_FrameInUseCount(frameInUseCount),
    7144  m_IsCustomPool(isCustomPool),
    7145  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
    7146  m_HasEmptyBlock(false),
    7147  m_pDefragmentator(VMA_NULL),
    7148  m_NextBlockId(0)
    7149 {
    7150 }
    7151 
    7152 VmaBlockVector::~VmaBlockVector()
    7153 {
    7154  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
    7155 
    7156  for(size_t i = m_Blocks.size(); i--; )
    7157  {
    7158  m_Blocks[i]->Destroy(m_hAllocator);
    7159  vma_delete(m_hAllocator, m_Blocks[i]);
    7160  }
    7161 }
    7162 
    7163 VkResult VmaBlockVector::CreateMinBlocks()
    7164 {
    7165  for(size_t i = 0; i < m_MinBlockCount; ++i)
    7166  {
    7167  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
    7168  if(res != VK_SUCCESS)
    7169  {
    7170  return res;
    7171  }
    7172  }
    7173  return VK_SUCCESS;
    7174 }
    7175 
    7176 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
    7177 {
    7178  pStats->size = 0;
    7179  pStats->unusedSize = 0;
    7180  pStats->allocationCount = 0;
    7181  pStats->unusedRangeCount = 0;
    7182  pStats->unusedRangeSizeMax = 0;
    7183 
    7184  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7185 
    7186  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7187  {
    7188  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7189  VMA_ASSERT(pBlock);
    7190  VMA_HEAVY_ASSERT(pBlock->Validate());
    7191  pBlock->m_Metadata.AddPoolStats(*pStats);
    7192  }
    7193 }
    7194 
    7195 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
    7196 {
    7197  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    7198  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
    7199  (VMA_DEBUG_MARGIN > 0) &&
    7200  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
    7201 }
    7202 
    7203 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
    7204 
    7205 VkResult VmaBlockVector::Allocate(
    7206  VmaPool hCurrentPool,
    7207  uint32_t currentFrameIndex,
    7208  VkDeviceSize size,
    7209  VkDeviceSize alignment,
    7210  const VmaAllocationCreateInfo& createInfo,
    7211  VmaSuballocationType suballocType,
    7212  VmaAllocation* pAllocation)
    7213 {
    7214  // Early reject: requested allocation size is larger that maximum block size for this block vector.
    7215  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
    7216  {
    7217  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7218  }
    7219 
    7220  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    7221  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
    7222 
    7223  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7224 
    7225  // 1. Search existing allocations. Try to allocate without making other allocations lost.
    7226  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7227  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7228  {
    7229  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7230  VMA_ASSERT(pCurrBlock);
    7231  VmaAllocationRequest currRequest = {};
    7232  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7233  currentFrameIndex,
    7234  m_FrameInUseCount,
    7235  m_BufferImageGranularity,
    7236  size,
    7237  alignment,
    7238  suballocType,
    7239  false, // canMakeOtherLost
    7240  &currRequest))
    7241  {
    7242  // Allocate from pCurrBlock.
    7243  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
    7244 
    7245  if(mapped)
    7246  {
    7247  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
    7248  if(res != VK_SUCCESS)
    7249  {
    7250  return res;
    7251  }
    7252  }
    7253 
    7254  // We no longer have an empty Allocation.
    7255  if(pCurrBlock->m_Metadata.IsEmpty())
    7256  {
    7257  m_HasEmptyBlock = false;
    7258  }
    7259 
    7260  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7261  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation);
    7262  (*pAllocation)->InitBlockAllocation(
    7263  hCurrentPool,
    7264  pCurrBlock,
    7265  currRequest.offset,
    7266  alignment,
    7267  size,
    7268  suballocType,
    7269  mapped,
    7270  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7271  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
    7272  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7273  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7274  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7275  {
    7276  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7277  }
    7278  if(IsCorruptionDetectionEnabled())
    7279  {
    7280  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
    7281  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7282  }
    7283  return VK_SUCCESS;
    7284  }
    7285  }
    7286 
    7287  const bool canCreateNewBlock =
    7288  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
    7289  (m_Blocks.size() < m_MaxBlockCount);
    7290 
    7291  // 2. Try to create new block.
    7292  if(canCreateNewBlock)
    7293  {
    7294  // Calculate optimal size for new block.
    7295  VkDeviceSize newBlockSize = m_PreferredBlockSize;
    7296  uint32_t newBlockSizeShift = 0;
    7297  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
    7298 
    7299  // Allocating blocks of other sizes is allowed only in default pools.
    7300  // In custom pools block size is fixed.
    7301  if(m_IsCustomPool == false)
    7302  {
    7303  // Allocate 1/8, 1/4, 1/2 as first blocks.
    7304  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
    7305  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
    7306  {
    7307  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7308  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
    7309  {
    7310  newBlockSize = smallerNewBlockSize;
    7311  ++newBlockSizeShift;
    7312  }
    7313  else
    7314  {
    7315  break;
    7316  }
    7317  }
    7318  }
    7319 
    7320  size_t newBlockIndex = 0;
    7321  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
    7322  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
    7323  if(m_IsCustomPool == false)
    7324  {
    7325  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
    7326  {
    7327  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
    7328  if(smallerNewBlockSize >= size)
    7329  {
    7330  newBlockSize = smallerNewBlockSize;
    7331  ++newBlockSizeShift;
    7332  res = CreateBlock(newBlockSize, &newBlockIndex);
    7333  }
    7334  else
    7335  {
    7336  break;
    7337  }
    7338  }
    7339  }
    7340 
    7341  if(res == VK_SUCCESS)
    7342  {
    7343  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
    7344  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size);
    7345 
    7346  if(mapped)
    7347  {
    7348  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
    7349  if(res != VK_SUCCESS)
    7350  {
    7351  return res;
    7352  }
    7353  }
    7354 
    7355  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
    7356  VmaAllocationRequest allocRequest;
    7357  if(pBlock->m_Metadata.CreateAllocationRequest(
    7358  currentFrameIndex,
    7359  m_FrameInUseCount,
    7360  m_BufferImageGranularity,
    7361  size,
    7362  alignment,
    7363  suballocType,
    7364  false, // canMakeOtherLost
    7365  &allocRequest))
    7366  {
    7367  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7368  pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation);
    7369  (*pAllocation)->InitBlockAllocation(
    7370  hCurrentPool,
    7371  pBlock,
    7372  allocRequest.offset,
    7373  alignment,
    7374  size,
    7375  suballocType,
    7376  mapped,
    7377  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7378  VMA_HEAVY_ASSERT(pBlock->Validate());
    7379  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
    7380  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7381  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7382  {
    7383  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7384  }
    7385  if(IsCorruptionDetectionEnabled())
    7386  {
    7387  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
    7388  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7389  }
    7390  return VK_SUCCESS;
    7391  }
    7392  else
    7393  {
    7394  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
    7395  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7396  }
    7397  }
    7398  }
    7399 
    7400  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
    7401 
    7402  // 3. Try to allocate from existing blocks with making other allocations lost.
    7403  if(canMakeOtherLost)
    7404  {
    7405  uint32_t tryIndex = 0;
    7406  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
    7407  {
    7408  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
    7409  VmaAllocationRequest bestRequest = {};
    7410  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
    7411 
    7412  // 1. Search existing allocations.
    7413  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
    7414  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
    7415  {
    7416  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
    7417  VMA_ASSERT(pCurrBlock);
    7418  VmaAllocationRequest currRequest = {};
    7419  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
    7420  currentFrameIndex,
    7421  m_FrameInUseCount,
    7422  m_BufferImageGranularity,
    7423  size,
    7424  alignment,
    7425  suballocType,
    7426  canMakeOtherLost,
    7427  &currRequest))
    7428  {
    7429  const VkDeviceSize currRequestCost = currRequest.CalcCost();
    7430  if(pBestRequestBlock == VMA_NULL ||
    7431  currRequestCost < bestRequestCost)
    7432  {
    7433  pBestRequestBlock = pCurrBlock;
    7434  bestRequest = currRequest;
    7435  bestRequestCost = currRequestCost;
    7436 
    7437  if(bestRequestCost == 0)
    7438  {
    7439  break;
    7440  }
    7441  }
    7442  }
    7443  }
    7444 
    7445  if(pBestRequestBlock != VMA_NULL)
    7446  {
    7447  if(mapped)
    7448  {
    7449  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
    7450  if(res != VK_SUCCESS)
    7451  {
    7452  return res;
    7453  }
    7454  }
    7455 
    7456  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
    7457  currentFrameIndex,
    7458  m_FrameInUseCount,
    7459  &bestRequest))
    7460  {
    7461  // We no longer have an empty Allocation.
    7462  if(pBestRequestBlock->m_Metadata.IsEmpty())
    7463  {
    7464  m_HasEmptyBlock = false;
    7465  }
    7466  // Allocate from this pBlock.
    7467  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
    7468  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation);
    7469  (*pAllocation)->InitBlockAllocation(
    7470  hCurrentPool,
    7471  pBestRequestBlock,
    7472  bestRequest.offset,
    7473  alignment,
    7474  size,
    7475  suballocType,
    7476  mapped,
    7477  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
    7478  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
    7479  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
    7480  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
    7481  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    7482  {
    7483  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    7484  }
    7485  if(IsCorruptionDetectionEnabled())
    7486  {
    7487  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
    7488  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
    7489  }
    7490  return VK_SUCCESS;
    7491  }
    7492  // else: Some allocations must have been touched while we are here. Next try.
    7493  }
    7494  else
    7495  {
    7496  // Could not find place in any of the blocks - break outer loop.
    7497  break;
    7498  }
    7499  }
    7500  /* Maximum number of tries exceeded - a very unlike event when many other
    7501  threads are simultaneously touching allocations making it impossible to make
    7502  lost at the same time as we try to allocate. */
    7503  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
    7504  {
    7505  return VK_ERROR_TOO_MANY_OBJECTS;
    7506  }
    7507  }
    7508 
    7509  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    7510 }
    7511 
    7512 void VmaBlockVector::Free(
    7513  VmaAllocation hAllocation)
    7514 {
    7515  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
    7516 
    7517  // Scope for lock.
    7518  {
    7519  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7520 
    7521  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    7522 
    7523  if(IsCorruptionDetectionEnabled())
    7524  {
    7525  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
    7526  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
    7527  }
    7528 
    7529  if(hAllocation->IsPersistentMap())
    7530  {
    7531  pBlock->Unmap(m_hAllocator, 1);
    7532  }
    7533 
    7534  pBlock->m_Metadata.Free(hAllocation);
    7535  VMA_HEAVY_ASSERT(pBlock->Validate());
    7536 
    7537  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
    7538 
    7539  // pBlock became empty after this deallocation.
    7540  if(pBlock->m_Metadata.IsEmpty())
    7541  {
    7542  // Already has empty Allocation. We don't want to have two, so delete this one.
    7543  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
    7544  {
    7545  pBlockToDelete = pBlock;
    7546  Remove(pBlock);
    7547  }
    7548  // We now have first empty block.
    7549  else
    7550  {
    7551  m_HasEmptyBlock = true;
    7552  }
    7553  }
    7554  // pBlock didn't become empty, but we have another empty block - find and free that one.
    7555  // (This is optional, heuristics.)
    7556  else if(m_HasEmptyBlock)
    7557  {
    7558  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
    7559  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
    7560  {
    7561  pBlockToDelete = pLastBlock;
    7562  m_Blocks.pop_back();
    7563  m_HasEmptyBlock = false;
    7564  }
    7565  }
    7566 
    7567  IncrementallySortBlocks();
    7568  }
    7569 
    7570  // Destruction of a free Allocation. Deferred until this point, outside of mutex
    7571  // lock, for performance reason.
    7572  if(pBlockToDelete != VMA_NULL)
    7573  {
    7574  VMA_DEBUG_LOG(" Deleted empty allocation");
    7575  pBlockToDelete->Destroy(m_hAllocator);
    7576  vma_delete(m_hAllocator, pBlockToDelete);
    7577  }
    7578 }
    7579 
    7580 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
    7581 {
    7582  VkDeviceSize result = 0;
    7583  for(size_t i = m_Blocks.size(); i--; )
    7584  {
    7585  result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize());
    7586  if(result >= m_PreferredBlockSize)
    7587  {
    7588  break;
    7589  }
    7590  }
    7591  return result;
    7592 }
    7593 
    7594 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
    7595 {
    7596  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7597  {
    7598  if(m_Blocks[blockIndex] == pBlock)
    7599  {
    7600  VmaVectorRemove(m_Blocks, blockIndex);
    7601  return;
    7602  }
    7603  }
    7604  VMA_ASSERT(0);
    7605 }
    7606 
    7607 void VmaBlockVector::IncrementallySortBlocks()
    7608 {
    7609  // Bubble sort only until first swap.
    7610  for(size_t i = 1; i < m_Blocks.size(); ++i)
    7611  {
    7612  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
    7613  {
    7614  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
    7615  return;
    7616  }
    7617  }
    7618 }
    7619 
    7620 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
    7621 {
    7622  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    7623  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
    7624  allocInfo.allocationSize = blockSize;
    7625  VkDeviceMemory mem = VK_NULL_HANDLE;
    7626  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
    7627  if(res < 0)
    7628  {
    7629  return res;
    7630  }
    7631 
    7632  // New VkDeviceMemory successfully created.
    7633 
    7634  // Create new Allocation for it.
    7635  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
    7636  pBlock->Init(
    7637  m_MemoryTypeIndex,
    7638  mem,
    7639  allocInfo.allocationSize,
    7640  m_NextBlockId++);
    7641 
    7642  m_Blocks.push_back(pBlock);
    7643  if(pNewBlockIndex != VMA_NULL)
    7644  {
    7645  *pNewBlockIndex = m_Blocks.size() - 1;
    7646  }
    7647 
    7648  return VK_SUCCESS;
    7649 }
    7650 
    7651 #if VMA_STATS_STRING_ENABLED
    7652 
    7653 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
    7654 {
    7655  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7656 
    7657  json.BeginObject();
    7658 
    7659  if(m_IsCustomPool)
    7660  {
    7661  json.WriteString("MemoryTypeIndex");
    7662  json.WriteNumber(m_MemoryTypeIndex);
    7663 
    7664  json.WriteString("BlockSize");
    7665  json.WriteNumber(m_PreferredBlockSize);
    7666 
    7667  json.WriteString("BlockCount");
    7668  json.BeginObject(true);
    7669  if(m_MinBlockCount > 0)
    7670  {
    7671  json.WriteString("Min");
    7672  json.WriteNumber((uint64_t)m_MinBlockCount);
    7673  }
    7674  if(m_MaxBlockCount < SIZE_MAX)
    7675  {
    7676  json.WriteString("Max");
    7677  json.WriteNumber((uint64_t)m_MaxBlockCount);
    7678  }
    7679  json.WriteString("Cur");
    7680  json.WriteNumber((uint64_t)m_Blocks.size());
    7681  json.EndObject();
    7682 
    7683  if(m_FrameInUseCount > 0)
    7684  {
    7685  json.WriteString("FrameInUseCount");
    7686  json.WriteNumber(m_FrameInUseCount);
    7687  }
    7688  }
    7689  else
    7690  {
    7691  json.WriteString("PreferredBlockSize");
    7692  json.WriteNumber(m_PreferredBlockSize);
    7693  }
    7694 
    7695  json.WriteString("Blocks");
    7696  json.BeginObject();
    7697  for(size_t i = 0; i < m_Blocks.size(); ++i)
    7698  {
    7699  json.BeginString();
    7700  json.ContinueString(m_Blocks[i]->GetId());
    7701  json.EndString();
    7702 
    7703  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
    7704  }
    7705  json.EndObject();
    7706 
    7707  json.EndObject();
    7708 }
    7709 
    7710 #endif // #if VMA_STATS_STRING_ENABLED
    7711 
    7712 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
    7713  VmaAllocator hAllocator,
    7714  uint32_t currentFrameIndex)
    7715 {
    7716  if(m_pDefragmentator == VMA_NULL)
    7717  {
    7718  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
    7719  hAllocator,
    7720  this,
    7721  currentFrameIndex);
    7722  }
    7723 
    7724  return m_pDefragmentator;
    7725 }
    7726 
    7727 VkResult VmaBlockVector::Defragment(
    7728  VmaDefragmentationStats* pDefragmentationStats,
    7729  VkDeviceSize& maxBytesToMove,
    7730  uint32_t& maxAllocationsToMove)
    7731 {
    7732  if(m_pDefragmentator == VMA_NULL)
    7733  {
    7734  return VK_SUCCESS;
    7735  }
    7736 
    7737  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7738 
    7739  // Defragment.
    7740  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
    7741 
    7742  // Accumulate statistics.
    7743  if(pDefragmentationStats != VMA_NULL)
    7744  {
    7745  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
    7746  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
    7747  pDefragmentationStats->bytesMoved += bytesMoved;
    7748  pDefragmentationStats->allocationsMoved += allocationsMoved;
    7749  VMA_ASSERT(bytesMoved <= maxBytesToMove);
    7750  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
    7751  maxBytesToMove -= bytesMoved;
    7752  maxAllocationsToMove -= allocationsMoved;
    7753  }
    7754 
    7755  // Free empty blocks.
    7756  m_HasEmptyBlock = false;
    7757  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
    7758  {
    7759  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
    7760  if(pBlock->m_Metadata.IsEmpty())
    7761  {
    7762  if(m_Blocks.size() > m_MinBlockCount)
    7763  {
    7764  if(pDefragmentationStats != VMA_NULL)
    7765  {
    7766  ++pDefragmentationStats->deviceMemoryBlocksFreed;
    7767  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
    7768  }
    7769 
    7770  VmaVectorRemove(m_Blocks, blockIndex);
    7771  pBlock->Destroy(m_hAllocator);
    7772  vma_delete(m_hAllocator, pBlock);
    7773  }
    7774  else
    7775  {
    7776  m_HasEmptyBlock = true;
    7777  }
    7778  }
    7779  }
    7780 
    7781  return result;
    7782 }
    7783 
    7784 void VmaBlockVector::DestroyDefragmentator()
    7785 {
    7786  if(m_pDefragmentator != VMA_NULL)
    7787  {
    7788  vma_delete(m_hAllocator, m_pDefragmentator);
    7789  m_pDefragmentator = VMA_NULL;
    7790  }
    7791 }
    7792 
    7793 void VmaBlockVector::MakePoolAllocationsLost(
    7794  uint32_t currentFrameIndex,
    7795  size_t* pLostAllocationCount)
    7796 {
    7797  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7798  size_t lostAllocationCount = 0;
    7799  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7800  {
    7801  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7802  VMA_ASSERT(pBlock);
    7803  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
    7804  }
    7805  if(pLostAllocationCount != VMA_NULL)
    7806  {
    7807  *pLostAllocationCount = lostAllocationCount;
    7808  }
    7809 }
    7810 
    7811 VkResult VmaBlockVector::CheckCorruption()
    7812 {
    7813  if(!IsCorruptionDetectionEnabled())
    7814  {
    7815  return VK_ERROR_FEATURE_NOT_PRESENT;
    7816  }
    7817 
    7818  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7819  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7820  {
    7821  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7822  VMA_ASSERT(pBlock);
    7823  VkResult res = pBlock->CheckCorruption(m_hAllocator);
    7824  if(res != VK_SUCCESS)
    7825  {
    7826  return res;
    7827  }
    7828  }
    7829  return VK_SUCCESS;
    7830 }
    7831 
    7832 void VmaBlockVector::AddStats(VmaStats* pStats)
    7833 {
    7834  const uint32_t memTypeIndex = m_MemoryTypeIndex;
    7835  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
    7836 
    7837  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
    7838 
    7839  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
    7840  {
    7841  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
    7842  VMA_ASSERT(pBlock);
    7843  VMA_HEAVY_ASSERT(pBlock->Validate());
    7844  VmaStatInfo allocationStatInfo;
    7845  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
    7846  VmaAddStatInfo(pStats->total, allocationStatInfo);
    7847  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    7848  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    7849  }
    7850 }
    7851 
    7853 // VmaDefragmentator members definition
    7854 
    7855 VmaDefragmentator::VmaDefragmentator(
    7856  VmaAllocator hAllocator,
    7857  VmaBlockVector* pBlockVector,
    7858  uint32_t currentFrameIndex) :
    7859  m_hAllocator(hAllocator),
    7860  m_pBlockVector(pBlockVector),
    7861  m_CurrentFrameIndex(currentFrameIndex),
    7862  m_BytesMoved(0),
    7863  m_AllocationsMoved(0),
    7864  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
    7865  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
    7866 {
    7867 }
    7868 
    7869 VmaDefragmentator::~VmaDefragmentator()
    7870 {
    7871  for(size_t i = m_Blocks.size(); i--; )
    7872  {
    7873  vma_delete(m_hAllocator, m_Blocks[i]);
    7874  }
    7875 }
    7876 
    7877 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
    7878 {
    7879  AllocationInfo allocInfo;
    7880  allocInfo.m_hAllocation = hAlloc;
    7881  allocInfo.m_pChanged = pChanged;
    7882  m_Allocations.push_back(allocInfo);
    7883 }
    7884 
    7885 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
    7886 {
    7887  // It has already been mapped for defragmentation.
    7888  if(m_pMappedDataForDefragmentation)
    7889  {
    7890  *ppMappedData = m_pMappedDataForDefragmentation;
    7891  return VK_SUCCESS;
    7892  }
    7893 
    7894  // It is originally mapped.
    7895  if(m_pBlock->GetMappedData())
    7896  {
    7897  *ppMappedData = m_pBlock->GetMappedData();
    7898  return VK_SUCCESS;
    7899  }
    7900 
    7901  // Map on first usage.
    7902  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
    7903  *ppMappedData = m_pMappedDataForDefragmentation;
    7904  return res;
    7905 }
    7906 
    7907 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
    7908 {
    7909  if(m_pMappedDataForDefragmentation != VMA_NULL)
    7910  {
    7911  m_pBlock->Unmap(hAllocator, 1);
    7912  }
    7913 }
    7914 
    7915 VkResult VmaDefragmentator::DefragmentRound(
    7916  VkDeviceSize maxBytesToMove,
    7917  uint32_t maxAllocationsToMove)
    7918 {
    7919  if(m_Blocks.empty())
    7920  {
    7921  return VK_SUCCESS;
    7922  }
    7923 
    7924  size_t srcBlockIndex = m_Blocks.size() - 1;
    7925  size_t srcAllocIndex = SIZE_MAX;
    7926  for(;;)
    7927  {
    7928  // 1. Find next allocation to move.
    7929  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
    7930  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
    7931  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
    7932  {
    7933  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
    7934  {
    7935  // Finished: no more allocations to process.
    7936  if(srcBlockIndex == 0)
    7937  {
    7938  return VK_SUCCESS;
    7939  }
    7940  else
    7941  {
    7942  --srcBlockIndex;
    7943  srcAllocIndex = SIZE_MAX;
    7944  }
    7945  }
    7946  else
    7947  {
    7948  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
    7949  }
    7950  }
    7951 
    7952  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
    7953  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
    7954 
    7955  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
    7956  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
    7957  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
    7958  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
    7959 
    7960  // 2. Try to find new place for this allocation in preceding or current block.
    7961  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
    7962  {
    7963  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
    7964  VmaAllocationRequest dstAllocRequest;
    7965  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
    7966  m_CurrentFrameIndex,
    7967  m_pBlockVector->GetFrameInUseCount(),
    7968  m_pBlockVector->GetBufferImageGranularity(),
    7969  size,
    7970  alignment,
    7971  suballocType,
    7972  false, // canMakeOtherLost
    7973  &dstAllocRequest) &&
    7974  MoveMakesSense(
    7975  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
    7976  {
    7977  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
    7978 
    7979  // Reached limit on number of allocations or bytes to move.
    7980  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
    7981  (m_BytesMoved + size > maxBytesToMove))
    7982  {
    7983  return VK_INCOMPLETE;
    7984  }
    7985 
    7986  void* pDstMappedData = VMA_NULL;
    7987  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
    7988  if(res != VK_SUCCESS)
    7989  {
    7990  return res;
    7991  }
    7992 
    7993  void* pSrcMappedData = VMA_NULL;
    7994  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
    7995  if(res != VK_SUCCESS)
    7996  {
    7997  return res;
    7998  }
    7999 
    8000  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
    8001  memcpy(
    8002  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
    8003  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
    8004  static_cast<size_t>(size));
    8005 
    8006  if(VMA_DEBUG_MARGIN > 0)
    8007  {
    8008  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
    8009  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
    8010  }
    8011 
    8012  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
    8013  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
    8014 
    8015  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
    8016 
    8017  if(allocInfo.m_pChanged != VMA_NULL)
    8018  {
    8019  *allocInfo.m_pChanged = VK_TRUE;
    8020  }
    8021 
    8022  ++m_AllocationsMoved;
    8023  m_BytesMoved += size;
    8024 
    8025  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
    8026 
    8027  break;
    8028  }
    8029  }
    8030 
    8031  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
    8032 
    8033  if(srcAllocIndex > 0)
    8034  {
    8035  --srcAllocIndex;
    8036  }
    8037  else
    8038  {
    8039  if(srcBlockIndex > 0)
    8040  {
    8041  --srcBlockIndex;
    8042  srcAllocIndex = SIZE_MAX;
    8043  }
    8044  else
    8045  {
    8046  return VK_SUCCESS;
    8047  }
    8048  }
    8049  }
    8050 }
    8051 
    8052 VkResult VmaDefragmentator::Defragment(
    8053  VkDeviceSize maxBytesToMove,
    8054  uint32_t maxAllocationsToMove)
    8055 {
    8056  if(m_Allocations.empty())
    8057  {
    8058  return VK_SUCCESS;
    8059  }
    8060 
    8061  // Create block info for each block.
    8062  const size_t blockCount = m_pBlockVector->m_Blocks.size();
    8063  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8064  {
    8065  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
    8066  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
    8067  m_Blocks.push_back(pBlockInfo);
    8068  }
    8069 
    8070  // Sort them by m_pBlock pointer value.
    8071  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
    8072 
    8073  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
    8074  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
    8075  {
    8076  AllocationInfo& allocInfo = m_Allocations[blockIndex];
    8077  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
    8078  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    8079  {
    8080  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
    8081  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
    8082  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
    8083  {
    8084  (*it)->m_Allocations.push_back(allocInfo);
    8085  }
    8086  else
    8087  {
    8088  VMA_ASSERT(0);
    8089  }
    8090  }
    8091  }
    8092  m_Allocations.clear();
    8093 
    8094  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8095  {
    8096  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
    8097  pBlockInfo->CalcHasNonMovableAllocations();
    8098  pBlockInfo->SortAllocationsBySizeDescecnding();
    8099  }
    8100 
    8101  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
    8102  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
    8103 
    8104  // Execute defragmentation rounds (the main part).
    8105  VkResult result = VK_SUCCESS;
    8106  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
    8107  {
    8108  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
    8109  }
    8110 
    8111  // Unmap blocks that were mapped for defragmentation.
    8112  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
    8113  {
    8114  m_Blocks[blockIndex]->Unmap(m_hAllocator);
    8115  }
    8116 
    8117  return result;
    8118 }
    8119 
    8120 bool VmaDefragmentator::MoveMakesSense(
    8121  size_t dstBlockIndex, VkDeviceSize dstOffset,
    8122  size_t srcBlockIndex, VkDeviceSize srcOffset)
    8123 {
    8124  if(dstBlockIndex < srcBlockIndex)
    8125  {
    8126  return true;
    8127  }
    8128  if(dstBlockIndex > srcBlockIndex)
    8129  {
    8130  return false;
    8131  }
    8132  if(dstOffset < srcOffset)
    8133  {
    8134  return true;
    8135  }
    8136  return false;
    8137 }
    8138 
    8140 // VmaRecorder
    8141 
    8142 #if VMA_RECORDING_ENABLED
    8143 
    8144 VmaRecorder::VmaRecorder() :
    8145  m_UseMutex(true),
    8146  m_Flags(0),
    8147  m_File(VMA_NULL),
    8148  m_Freq(INT64_MAX),
    8149  m_StartCounter(INT64_MAX)
    8150 {
    8151 }
    8152 
    8153 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
    8154 {
    8155  m_UseMutex = useMutex;
    8156  m_Flags = settings.flags;
    8157 
    8158  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
    8159  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
    8160 
    8161  // Open file for writing.
    8162  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
    8163  if(err != 0)
    8164  {
    8165  return VK_ERROR_INITIALIZATION_FAILED;
    8166  }
    8167 
    8168  // Write header.
    8169  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
    8170  fprintf(m_File, "%s\n", "1,3");
    8171 
    8172  return VK_SUCCESS;
    8173 }
    8174 
    8175 VmaRecorder::~VmaRecorder()
    8176 {
    8177  if(m_File != VMA_NULL)
    8178  {
    8179  fclose(m_File);
    8180  }
    8181 }
    8182 
    8183 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
    8184 {
    8185  CallParams callParams;
    8186  GetBasicParams(callParams);
    8187 
    8188  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8189  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8190  Flush();
    8191 }
    8192 
    8193 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
    8194 {
    8195  CallParams callParams;
    8196  GetBasicParams(callParams);
    8197 
    8198  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8199  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
    8200  Flush();
    8201 }
    8202 
    8203 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
    8204 {
    8205  CallParams callParams;
    8206  GetBasicParams(callParams);
    8207 
    8208  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8209  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
    8210  createInfo.memoryTypeIndex,
    8211  createInfo.flags,
    8212  createInfo.blockSize,
    8213  createInfo.minBlockCount,
    8214  createInfo.maxBlockCount,
    8215  createInfo.frameInUseCount,
    8216  pool);
    8217  Flush();
    8218 }
    8219 
    8220 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
    8221 {
    8222  CallParams callParams;
    8223  GetBasicParams(callParams);
    8224 
    8225  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8226  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
    8227  pool);
    8228  Flush();
    8229 }
    8230 
    8231 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
    8232  const VkMemoryRequirements& vkMemReq,
    8233  const VmaAllocationCreateInfo& createInfo,
    8234  VmaAllocation allocation)
    8235 {
    8236  CallParams callParams;
    8237  GetBasicParams(callParams);
    8238 
    8239  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8240  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8241  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8242  vkMemReq.size,
    8243  vkMemReq.alignment,
    8244  vkMemReq.memoryTypeBits,
    8245  createInfo.flags,
    8246  createInfo.usage,
    8247  createInfo.requiredFlags,
    8248  createInfo.preferredFlags,
    8249  createInfo.memoryTypeBits,
    8250  createInfo.pool,
    8251  allocation,
    8252  userDataStr.GetString());
    8253  Flush();
    8254 }
    8255 
    8256 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
    8257  const VkMemoryRequirements& vkMemReq,
    8258  bool requiresDedicatedAllocation,
    8259  bool prefersDedicatedAllocation,
    8260  const VmaAllocationCreateInfo& createInfo,
    8261  VmaAllocation allocation)
    8262 {
    8263  CallParams callParams;
    8264  GetBasicParams(callParams);
    8265 
    8266  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8267  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8268  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8269  vkMemReq.size,
    8270  vkMemReq.alignment,
    8271  vkMemReq.memoryTypeBits,
    8272  requiresDedicatedAllocation ? 1 : 0,
    8273  prefersDedicatedAllocation ? 1 : 0,
    8274  createInfo.flags,
    8275  createInfo.usage,
    8276  createInfo.requiredFlags,
    8277  createInfo.preferredFlags,
    8278  createInfo.memoryTypeBits,
    8279  createInfo.pool,
    8280  allocation,
    8281  userDataStr.GetString());
    8282  Flush();
    8283 }
    8284 
    8285 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
    8286  const VkMemoryRequirements& vkMemReq,
    8287  bool requiresDedicatedAllocation,
    8288  bool prefersDedicatedAllocation,
    8289  const VmaAllocationCreateInfo& createInfo,
    8290  VmaAllocation allocation)
    8291 {
    8292  CallParams callParams;
    8293  GetBasicParams(callParams);
    8294 
    8295  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8296  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
    8297  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8298  vkMemReq.size,
    8299  vkMemReq.alignment,
    8300  vkMemReq.memoryTypeBits,
    8301  requiresDedicatedAllocation ? 1 : 0,
    8302  prefersDedicatedAllocation ? 1 : 0,
    8303  createInfo.flags,
    8304  createInfo.usage,
    8305  createInfo.requiredFlags,
    8306  createInfo.preferredFlags,
    8307  createInfo.memoryTypeBits,
    8308  createInfo.pool,
    8309  allocation,
    8310  userDataStr.GetString());
    8311  Flush();
    8312 }
    8313 
    8314 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
    8315  VmaAllocation allocation)
    8316 {
    8317  CallParams callParams;
    8318  GetBasicParams(callParams);
    8319 
    8320  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8321  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8322  allocation);
    8323  Flush();
    8324 }
    8325 
    8326 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
    8327  VmaAllocation allocation,
    8328  const void* pUserData)
    8329 {
    8330  CallParams callParams;
    8331  GetBasicParams(callParams);
    8332 
    8333  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8334  UserDataString userDataStr(
    8335  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
    8336  pUserData);
    8337  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8338  allocation,
    8339  userDataStr.GetString());
    8340  Flush();
    8341 }
    8342 
    8343 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
    8344  VmaAllocation allocation)
    8345 {
    8346  CallParams callParams;
    8347  GetBasicParams(callParams);
    8348 
    8349  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8350  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
    8351  allocation);
    8352  Flush();
    8353 }
    8354 
    8355 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
    8356  VmaAllocation allocation)
    8357 {
    8358  CallParams callParams;
    8359  GetBasicParams(callParams);
    8360 
    8361  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8362  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8363  allocation);
    8364  Flush();
    8365 }
    8366 
    8367 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
    8368  VmaAllocation allocation)
    8369 {
    8370  CallParams callParams;
    8371  GetBasicParams(callParams);
    8372 
    8373  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8374  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
    8375  allocation);
    8376  Flush();
    8377 }
    8378 
    8379 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
    8380  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8381 {
    8382  CallParams callParams;
    8383  GetBasicParams(callParams);
    8384 
    8385  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8386  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8387  allocation,
    8388  offset,
    8389  size);
    8390  Flush();
    8391 }
    8392 
    8393 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
    8394  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    8395 {
    8396  CallParams callParams;
    8397  GetBasicParams(callParams);
    8398 
    8399  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8400  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
    8401  allocation,
    8402  offset,
    8403  size);
    8404  Flush();
    8405 }
    8406 
    8407 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
    8408  const VkBufferCreateInfo& bufCreateInfo,
    8409  const VmaAllocationCreateInfo& allocCreateInfo,
    8410  VmaAllocation allocation)
    8411 {
    8412  CallParams callParams;
    8413  GetBasicParams(callParams);
    8414 
    8415  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8416  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8417  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8418  bufCreateInfo.flags,
    8419  bufCreateInfo.size,
    8420  bufCreateInfo.usage,
    8421  bufCreateInfo.sharingMode,
    8422  allocCreateInfo.flags,
    8423  allocCreateInfo.usage,
    8424  allocCreateInfo.requiredFlags,
    8425  allocCreateInfo.preferredFlags,
    8426  allocCreateInfo.memoryTypeBits,
    8427  allocCreateInfo.pool,
    8428  allocation,
    8429  userDataStr.GetString());
    8430  Flush();
    8431 }
    8432 
    8433 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
    8434  const VkImageCreateInfo& imageCreateInfo,
    8435  const VmaAllocationCreateInfo& allocCreateInfo,
    8436  VmaAllocation allocation)
    8437 {
    8438  CallParams callParams;
    8439  GetBasicParams(callParams);
    8440 
    8441  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8442  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
    8443  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
    8444  imageCreateInfo.flags,
    8445  imageCreateInfo.imageType,
    8446  imageCreateInfo.format,
    8447  imageCreateInfo.extent.width,
    8448  imageCreateInfo.extent.height,
    8449  imageCreateInfo.extent.depth,
    8450  imageCreateInfo.mipLevels,
    8451  imageCreateInfo.arrayLayers,
    8452  imageCreateInfo.samples,
    8453  imageCreateInfo.tiling,
    8454  imageCreateInfo.usage,
    8455  imageCreateInfo.sharingMode,
    8456  imageCreateInfo.initialLayout,
    8457  allocCreateInfo.flags,
    8458  allocCreateInfo.usage,
    8459  allocCreateInfo.requiredFlags,
    8460  allocCreateInfo.preferredFlags,
    8461  allocCreateInfo.memoryTypeBits,
    8462  allocCreateInfo.pool,
    8463  allocation,
    8464  userDataStr.GetString());
    8465  Flush();
    8466 }
    8467 
    8468 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
    8469  VmaAllocation allocation)
    8470 {
    8471  CallParams callParams;
    8472  GetBasicParams(callParams);
    8473 
    8474  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8475  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
    8476  allocation);
    8477  Flush();
    8478 }
    8479 
    8480 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
    8481  VmaAllocation allocation)
    8482 {
    8483  CallParams callParams;
    8484  GetBasicParams(callParams);
    8485 
    8486  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8487  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
    8488  allocation);
    8489  Flush();
    8490 }
    8491 
    8492 void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex,
    8493  VmaAllocation allocation)
    8494 {
    8495  CallParams callParams;
    8496  GetBasicParams(callParams);
    8497 
    8498  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8499  fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
    8500  allocation);
    8501  Flush();
    8502 }
    8503 
    8504 void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex,
    8505  VmaAllocation allocation)
    8506 {
    8507  CallParams callParams;
    8508  GetBasicParams(callParams);
    8509 
    8510  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8511  fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex,
    8512  allocation);
    8513  Flush();
    8514 }
    8515 
    8516 void VmaRecorder::RecordMakePoolAllocationsLost(uint32_t frameIndex,
    8517  VmaPool pool)
    8518 {
    8519  CallParams callParams;
    8520  GetBasicParams(callParams);
    8521 
    8522  VmaMutexLock lock(m_FileMutex, m_UseMutex);
    8523  fprintf(m_File, "%u,%.3f,%u,vmaMakePoolAllocationsLost,%p\n", callParams.threadId, callParams.time, frameIndex,
    8524  pool);
    8525  Flush();
    8526 }
    8527 
    8528 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
    8529 {
    8530  if(pUserData != VMA_NULL)
    8531  {
    8532  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
    8533  {
    8534  m_Str = (const char*)pUserData;
    8535  }
    8536  else
    8537  {
    8538  sprintf_s(m_PtrStr, "%p", pUserData);
    8539  m_Str = m_PtrStr;
    8540  }
    8541  }
    8542  else
    8543  {
    8544  m_Str = "";
    8545  }
    8546 }
    8547 
    8548 void VmaRecorder::WriteConfiguration(
    8549  const VkPhysicalDeviceProperties& devProps,
    8550  const VkPhysicalDeviceMemoryProperties& memProps,
    8551  bool dedicatedAllocationExtensionEnabled)
    8552 {
    8553  fprintf(m_File, "Config,Begin\n");
    8554 
    8555  fprintf(m_File, "PhysicalDevice,apiVersion,%u\n", devProps.apiVersion);
    8556  fprintf(m_File, "PhysicalDevice,driverVersion,%u\n", devProps.driverVersion);
    8557  fprintf(m_File, "PhysicalDevice,vendorID,%u\n", devProps.vendorID);
    8558  fprintf(m_File, "PhysicalDevice,deviceID,%u\n", devProps.deviceID);
    8559  fprintf(m_File, "PhysicalDevice,deviceType,%u\n", devProps.deviceType);
    8560  fprintf(m_File, "PhysicalDevice,deviceName,%s\n", devProps.deviceName);
    8561 
    8562  fprintf(m_File, "PhysicalDeviceLimits,maxMemoryAllocationCount,%u\n", devProps.limits.maxMemoryAllocationCount);
    8563  fprintf(m_File, "PhysicalDeviceLimits,bufferImageGranularity,%llu\n", devProps.limits.bufferImageGranularity);
    8564  fprintf(m_File, "PhysicalDeviceLimits,nonCoherentAtomSize,%llu\n", devProps.limits.nonCoherentAtomSize);
    8565 
    8566  fprintf(m_File, "PhysicalDeviceMemory,HeapCount,%u\n", memProps.memoryHeapCount);
    8567  for(uint32_t i = 0; i < memProps.memoryHeapCount; ++i)
    8568  {
    8569  fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,size,%llu\n", i, memProps.memoryHeaps[i].size);
    8570  fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,flags,%u\n", i, memProps.memoryHeaps[i].flags);
    8571  }
    8572  fprintf(m_File, "PhysicalDeviceMemory,TypeCount,%u\n", memProps.memoryTypeCount);
    8573  for(uint32_t i = 0; i < memProps.memoryTypeCount; ++i)
    8574  {
    8575  fprintf(m_File, "PhysicalDeviceMemory,Type,%u,heapIndex,%u\n", i, memProps.memoryTypes[i].heapIndex);
    8576  fprintf(m_File, "PhysicalDeviceMemory,Type,%u,propertyFlags,%u\n", i, memProps.memoryTypes[i].propertyFlags);
    8577  }
    8578 
    8579  fprintf(m_File, "Extension,VK_KHR_dedicated_allocation,%u\n", dedicatedAllocationExtensionEnabled ? 1 : 0);
    8580 
    8581  fprintf(m_File, "Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,%u\n", VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ? 1 : 0);
    8582  fprintf(m_File, "Macro,VMA_DEBUG_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_DEBUG_ALIGNMENT);
    8583  fprintf(m_File, "Macro,VMA_DEBUG_MARGIN,%llu\n", (VkDeviceSize)VMA_DEBUG_MARGIN);
    8584  fprintf(m_File, "Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS,%u\n", VMA_DEBUG_INITIALIZE_ALLOCATIONS ? 1 : 0);
    8585  fprintf(m_File, "Macro,VMA_DEBUG_DETECT_CORRUPTION,%u\n", VMA_DEBUG_DETECT_CORRUPTION ? 1 : 0);
    8586  fprintf(m_File, "Macro,VMA_DEBUG_GLOBAL_MUTEX,%u\n", VMA_DEBUG_GLOBAL_MUTEX ? 1 : 0);
    8587  fprintf(m_File, "Macro,VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY,%llu\n", (VkDeviceSize)VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY);
    8588  fprintf(m_File, "Macro,VMA_SMALL_HEAP_MAX_SIZE,%llu\n", (VkDeviceSize)VMA_SMALL_HEAP_MAX_SIZE);
    8589  fprintf(m_File, "Macro,VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE,%llu\n", (VkDeviceSize)VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
    8590 
    8591  fprintf(m_File, "Config,End\n");
    8592 }
    8593 
    8594 void VmaRecorder::GetBasicParams(CallParams& outParams)
    8595 {
    8596  outParams.threadId = GetCurrentThreadId();
    8597 
    8598  LARGE_INTEGER counter;
    8599  QueryPerformanceCounter(&counter);
    8600  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
    8601 }
    8602 
    8603 void VmaRecorder::Flush()
    8604 {
    8605  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
    8606  {
    8607  fflush(m_File);
    8608  }
    8609 }
    8610 
    8611 #endif // #if VMA_RECORDING_ENABLED
    8612 
    8614 // VmaAllocator_T
    8615 
    8616 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
    8617  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
    8618  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
    8619  m_hDevice(pCreateInfo->device),
    8620  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
    8621  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
    8622  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
    8623  m_PreferredLargeHeapBlockSize(0),
    8624  m_PhysicalDevice(pCreateInfo->physicalDevice),
    8625  m_CurrentFrameIndex(0),
    8626  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
    8627  m_NextPoolId(0)
    8628 #if VMA_RECORDING_ENABLED
    8629  ,m_pRecorder(VMA_NULL)
    8630 #endif
    8631 {
    8632  if(VMA_DEBUG_DETECT_CORRUPTION)
    8633  {
    8634  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
    8635  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
    8636  }
    8637 
    8638  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
    8639 
    8640 #if !(VMA_DEDICATED_ALLOCATION)
    8641  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
    8642  {
    8643  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
    8644  }
    8645 #endif
    8646 
    8647  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
    8648  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
    8649  memset(&m_MemProps, 0, sizeof(m_MemProps));
    8650 
    8651  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
    8652  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
    8653 
    8654  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    8655  {
    8656  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
    8657  }
    8658 
    8659  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
    8660  {
    8661  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
    8662  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
    8663  }
    8664 
    8665  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
    8666 
    8667  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
    8668  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
    8669 
    8670  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
    8671  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
    8672 
    8673  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
    8674  {
    8675  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
    8676  {
    8677  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
    8678  if(limit != VK_WHOLE_SIZE)
    8679  {
    8680  m_HeapSizeLimit[heapIndex] = limit;
    8681  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
    8682  {
    8683  m_MemProps.memoryHeaps[heapIndex].size = limit;
    8684  }
    8685  }
    8686  }
    8687  }
    8688 
    8689  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    8690  {
    8691  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
    8692 
    8693  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
    8694  this,
    8695  memTypeIndex,
    8696  preferredBlockSize,
    8697  0,
    8698  SIZE_MAX,
    8699  GetBufferImageGranularity(),
    8700  pCreateInfo->frameInUseCount,
    8701  false); // isCustomPool
    8702  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
    8703  // becase minBlockCount is 0.
    8704  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
    8705 
    8706  }
    8707 }
    8708 
    8709 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
    8710 {
    8711  VkResult res = VK_SUCCESS;
    8712 
    8713  if(pCreateInfo->pRecordSettings != VMA_NULL &&
    8714  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
    8715  {
    8716 #if VMA_RECORDING_ENABLED
    8717  m_pRecorder = vma_new(this, VmaRecorder)();
    8718  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
    8719  if(res != VK_SUCCESS)
    8720  {
    8721  return res;
    8722  }
    8723  m_pRecorder->WriteConfiguration(
    8724  m_PhysicalDeviceProperties,
    8725  m_MemProps,
    8726  m_UseKhrDedicatedAllocation);
    8727  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
    8728 #else
    8729  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
    8730  return VK_ERROR_FEATURE_NOT_PRESENT;
    8731 #endif
    8732  }
    8733 
    8734  return res;
    8735 }
    8736 
    8737 VmaAllocator_T::~VmaAllocator_T()
    8738 {
    8739 #if VMA_RECORDING_ENABLED
    8740  if(m_pRecorder != VMA_NULL)
    8741  {
    8742  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
    8743  vma_delete(this, m_pRecorder);
    8744  }
    8745 #endif
    8746 
    8747  VMA_ASSERT(m_Pools.empty());
    8748 
    8749  for(size_t i = GetMemoryTypeCount(); i--; )
    8750  {
    8751  vma_delete(this, m_pDedicatedAllocations[i]);
    8752  vma_delete(this, m_pBlockVectors[i]);
    8753  }
    8754 }
    8755 
    8756 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
    8757 {
    8758 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8759  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
    8760  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
    8761  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
    8762  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
    8763  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
    8764  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
    8765  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
    8766  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
    8767  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
    8768  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
    8769  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
    8770  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
    8771  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
    8772  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
    8773  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
    8774  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
    8775 #if VMA_DEDICATED_ALLOCATION
    8776  if(m_UseKhrDedicatedAllocation)
    8777  {
    8778  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
    8779  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
    8780  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
    8781  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
    8782  }
    8783 #endif // #if VMA_DEDICATED_ALLOCATION
    8784 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
    8785 
    8786 #define VMA_COPY_IF_NOT_NULL(funcName) \
    8787  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
    8788 
    8789  if(pVulkanFunctions != VMA_NULL)
    8790  {
    8791  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
    8792  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
    8793  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
    8794  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
    8795  VMA_COPY_IF_NOT_NULL(vkMapMemory);
    8796  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
    8797  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
    8798  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
    8799  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
    8800  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
    8801  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
    8802  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
    8803  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
    8804  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
    8805  VMA_COPY_IF_NOT_NULL(vkCreateImage);
    8806  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
    8807 #if VMA_DEDICATED_ALLOCATION
    8808  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
    8809  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
    8810 #endif
    8811  }
    8812 
    8813 #undef VMA_COPY_IF_NOT_NULL
    8814 
    8815  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
    8816  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
    8817  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
    8818  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
    8819  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
    8820  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
    8821  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
    8822  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
    8823  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
    8824  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
    8825  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
    8826  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
    8827  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
    8828  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
    8829  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
    8830  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
    8831  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
    8832  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
    8833 #if VMA_DEDICATED_ALLOCATION
    8834  if(m_UseKhrDedicatedAllocation)
    8835  {
    8836  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
    8837  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
    8838  }
    8839 #endif
    8840 }
    8841 
    8842 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
    8843 {
    8844  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    8845  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
    8846  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
    8847  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
    8848 }
    8849 
    8850 VkResult VmaAllocator_T::AllocateMemoryOfType(
    8851  VkDeviceSize size,
    8852  VkDeviceSize alignment,
    8853  bool dedicatedAllocation,
    8854  VkBuffer dedicatedBuffer,
    8855  VkImage dedicatedImage,
    8856  const VmaAllocationCreateInfo& createInfo,
    8857  uint32_t memTypeIndex,
    8858  VmaSuballocationType suballocType,
    8859  VmaAllocation* pAllocation)
    8860 {
    8861  VMA_ASSERT(pAllocation != VMA_NULL);
    8862  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
    8863 
    8864  VmaAllocationCreateInfo finalCreateInfo = createInfo;
    8865 
    8866  // If memory type is not HOST_VISIBLE, disable MAPPED.
    8867  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    8868  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    8869  {
    8870  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
    8871  }
    8872 
    8873  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
    8874  VMA_ASSERT(blockVector);
    8875 
    8876  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
    8877  bool preferDedicatedMemory =
    8878  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
    8879  dedicatedAllocation ||
    8880  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
    8881  size > preferredBlockSize / 2;
    8882 
    8883  if(preferDedicatedMemory &&
    8884  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
    8885  finalCreateInfo.pool == VK_NULL_HANDLE)
    8886  {
    8887  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
    8888  }
    8889 
    8890  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
    8891  {
    8892  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8893  {
    8894  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8895  }
    8896  else
    8897  {
    8898  return AllocateDedicatedMemory(
    8899  size,
    8900  suballocType,
    8901  memTypeIndex,
    8902  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8903  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8904  finalCreateInfo.pUserData,
    8905  dedicatedBuffer,
    8906  dedicatedImage,
    8907  pAllocation);
    8908  }
    8909  }
    8910  else
    8911  {
    8912  VkResult res = blockVector->Allocate(
    8913  VK_NULL_HANDLE, // hCurrentPool
    8914  m_CurrentFrameIndex.load(),
    8915  size,
    8916  alignment,
    8917  finalCreateInfo,
    8918  suballocType,
    8919  pAllocation);
    8920  if(res == VK_SUCCESS)
    8921  {
    8922  return res;
    8923  }
    8924 
    8925  // 5. Try dedicated memory.
    8926  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    8927  {
    8928  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    8929  }
    8930  else
    8931  {
    8932  res = AllocateDedicatedMemory(
    8933  size,
    8934  suballocType,
    8935  memTypeIndex,
    8936  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
    8937  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
    8938  finalCreateInfo.pUserData,
    8939  dedicatedBuffer,
    8940  dedicatedImage,
    8941  pAllocation);
    8942  if(res == VK_SUCCESS)
    8943  {
    8944  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
    8945  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
    8946  return VK_SUCCESS;
    8947  }
    8948  else
    8949  {
    8950  // Everything failed: Return error code.
    8951  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8952  return res;
    8953  }
    8954  }
    8955  }
    8956 }
    8957 
    8958 VkResult VmaAllocator_T::AllocateDedicatedMemory(
    8959  VkDeviceSize size,
    8960  VmaSuballocationType suballocType,
    8961  uint32_t memTypeIndex,
    8962  bool map,
    8963  bool isUserDataString,
    8964  void* pUserData,
    8965  VkBuffer dedicatedBuffer,
    8966  VkImage dedicatedImage,
    8967  VmaAllocation* pAllocation)
    8968 {
    8969  VMA_ASSERT(pAllocation);
    8970 
    8971  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
    8972  allocInfo.memoryTypeIndex = memTypeIndex;
    8973  allocInfo.allocationSize = size;
    8974 
    8975 #if VMA_DEDICATED_ALLOCATION
    8976  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
    8977  if(m_UseKhrDedicatedAllocation)
    8978  {
    8979  if(dedicatedBuffer != VK_NULL_HANDLE)
    8980  {
    8981  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
    8982  dedicatedAllocInfo.buffer = dedicatedBuffer;
    8983  allocInfo.pNext = &dedicatedAllocInfo;
    8984  }
    8985  else if(dedicatedImage != VK_NULL_HANDLE)
    8986  {
    8987  dedicatedAllocInfo.image = dedicatedImage;
    8988  allocInfo.pNext = &dedicatedAllocInfo;
    8989  }
    8990  }
    8991 #endif // #if VMA_DEDICATED_ALLOCATION
    8992 
    8993  // Allocate VkDeviceMemory.
    8994  VkDeviceMemory hMemory = VK_NULL_HANDLE;
    8995  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
    8996  if(res < 0)
    8997  {
    8998  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
    8999  return res;
    9000  }
    9001 
    9002  void* pMappedData = VMA_NULL;
    9003  if(map)
    9004  {
    9005  res = (*m_VulkanFunctions.vkMapMemory)(
    9006  m_hDevice,
    9007  hMemory,
    9008  0,
    9009  VK_WHOLE_SIZE,
    9010  0,
    9011  &pMappedData);
    9012  if(res < 0)
    9013  {
    9014  VMA_DEBUG_LOG(" vkMapMemory FAILED");
    9015  FreeVulkanMemory(memTypeIndex, size, hMemory);
    9016  return res;
    9017  }
    9018  }
    9019 
    9020  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
    9021  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
    9022  (*pAllocation)->SetUserData(this, pUserData);
    9023  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    9024  {
    9025  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
    9026  }
    9027 
    9028  // Register it in m_pDedicatedAllocations.
    9029  {
    9030  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9031  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    9032  VMA_ASSERT(pDedicatedAllocations);
    9033  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
    9034  }
    9035 
    9036  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
    9037 
    9038  return VK_SUCCESS;
    9039 }
    9040 
    9041 void VmaAllocator_T::GetBufferMemoryRequirements(
    9042  VkBuffer hBuffer,
    9043  VkMemoryRequirements& memReq,
    9044  bool& requiresDedicatedAllocation,
    9045  bool& prefersDedicatedAllocation) const
    9046 {
    9047 #if VMA_DEDICATED_ALLOCATION
    9048  if(m_UseKhrDedicatedAllocation)
    9049  {
    9050  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
    9051  memReqInfo.buffer = hBuffer;
    9052 
    9053  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    9054 
    9055  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    9056  memReq2.pNext = &memDedicatedReq;
    9057 
    9058  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    9059 
    9060  memReq = memReq2.memoryRequirements;
    9061  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    9062  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    9063  }
    9064  else
    9065 #endif // #if VMA_DEDICATED_ALLOCATION
    9066  {
    9067  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
    9068  requiresDedicatedAllocation = false;
    9069  prefersDedicatedAllocation = false;
    9070  }
    9071 }
    9072 
    9073 void VmaAllocator_T::GetImageMemoryRequirements(
    9074  VkImage hImage,
    9075  VkMemoryRequirements& memReq,
    9076  bool& requiresDedicatedAllocation,
    9077  bool& prefersDedicatedAllocation) const
    9078 {
    9079 #if VMA_DEDICATED_ALLOCATION
    9080  if(m_UseKhrDedicatedAllocation)
    9081  {
    9082  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
    9083  memReqInfo.image = hImage;
    9084 
    9085  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
    9086 
    9087  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
    9088  memReq2.pNext = &memDedicatedReq;
    9089 
    9090  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
    9091 
    9092  memReq = memReq2.memoryRequirements;
    9093  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
    9094  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
    9095  }
    9096  else
    9097 #endif // #if VMA_DEDICATED_ALLOCATION
    9098  {
    9099  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
    9100  requiresDedicatedAllocation = false;
    9101  prefersDedicatedAllocation = false;
    9102  }
    9103 }
    9104 
    9105 VkResult VmaAllocator_T::AllocateMemory(
    9106  const VkMemoryRequirements& vkMemReq,
    9107  bool requiresDedicatedAllocation,
    9108  bool prefersDedicatedAllocation,
    9109  VkBuffer dedicatedBuffer,
    9110  VkImage dedicatedImage,
    9111  const VmaAllocationCreateInfo& createInfo,
    9112  VmaSuballocationType suballocType,
    9113  VmaAllocation* pAllocation)
    9114 {
    9115  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
    9116  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    9117  {
    9118  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
    9119  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9120  }
    9121  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
    9122  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
    9123  {
    9124  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
    9125  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9126  }
    9127  if(requiresDedicatedAllocation)
    9128  {
    9129  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
    9130  {
    9131  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
    9132  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9133  }
    9134  if(createInfo.pool != VK_NULL_HANDLE)
    9135  {
    9136  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
    9137  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9138  }
    9139  }
    9140  if((createInfo.pool != VK_NULL_HANDLE) &&
    9141  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
    9142  {
    9143  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
    9144  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9145  }
    9146 
    9147  if(createInfo.pool != VK_NULL_HANDLE)
    9148  {
    9149  const VkDeviceSize alignmentForPool = VMA_MAX(
    9150  vkMemReq.alignment,
    9151  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
    9152  return createInfo.pool->m_BlockVector.Allocate(
    9153  createInfo.pool,
    9154  m_CurrentFrameIndex.load(),
    9155  vkMemReq.size,
    9156  alignmentForPool,
    9157  createInfo,
    9158  suballocType,
    9159  pAllocation);
    9160  }
    9161  else
    9162  {
    9163  // Bit mask of memory Vulkan types acceptable for this allocation.
    9164  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
    9165  uint32_t memTypeIndex = UINT32_MAX;
    9166  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9167  if(res == VK_SUCCESS)
    9168  {
    9169  VkDeviceSize alignmentForMemType = VMA_MAX(
    9170  vkMemReq.alignment,
    9171  GetMemoryTypeMinAlignment(memTypeIndex));
    9172 
    9173  res = AllocateMemoryOfType(
    9174  vkMemReq.size,
    9175  alignmentForMemType,
    9176  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9177  dedicatedBuffer,
    9178  dedicatedImage,
    9179  createInfo,
    9180  memTypeIndex,
    9181  suballocType,
    9182  pAllocation);
    9183  // Succeeded on first try.
    9184  if(res == VK_SUCCESS)
    9185  {
    9186  return res;
    9187  }
    9188  // Allocation from this memory type failed. Try other compatible memory types.
    9189  else
    9190  {
    9191  for(;;)
    9192  {
    9193  // Remove old memTypeIndex from list of possibilities.
    9194  memoryTypeBits &= ~(1u << memTypeIndex);
    9195  // Find alternative memTypeIndex.
    9196  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
    9197  if(res == VK_SUCCESS)
    9198  {
    9199  alignmentForMemType = VMA_MAX(
    9200  vkMemReq.alignment,
    9201  GetMemoryTypeMinAlignment(memTypeIndex));
    9202 
    9203  res = AllocateMemoryOfType(
    9204  vkMemReq.size,
    9205  alignmentForMemType,
    9206  requiresDedicatedAllocation || prefersDedicatedAllocation,
    9207  dedicatedBuffer,
    9208  dedicatedImage,
    9209  createInfo,
    9210  memTypeIndex,
    9211  suballocType,
    9212  pAllocation);
    9213  // Allocation from this alternative memory type succeeded.
    9214  if(res == VK_SUCCESS)
    9215  {
    9216  return res;
    9217  }
    9218  // else: Allocation from this memory type failed. Try next one - next loop iteration.
    9219  }
    9220  // No other matching memory type index could be found.
    9221  else
    9222  {
    9223  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
    9224  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9225  }
    9226  }
    9227  }
    9228  }
    9229  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
    9230  else
    9231  return res;
    9232  }
    9233 }
    9234 
    9235 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
    9236 {
    9237  VMA_ASSERT(allocation);
    9238 
    9239  if(allocation->CanBecomeLost() == false ||
    9240  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
    9241  {
    9242  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
    9243  {
    9244  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
    9245  }
    9246 
    9247  switch(allocation->GetType())
    9248  {
    9249  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9250  {
    9251  VmaBlockVector* pBlockVector = VMA_NULL;
    9252  VmaPool hPool = allocation->GetPool();
    9253  if(hPool != VK_NULL_HANDLE)
    9254  {
    9255  pBlockVector = &hPool->m_BlockVector;
    9256  }
    9257  else
    9258  {
    9259  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9260  pBlockVector = m_pBlockVectors[memTypeIndex];
    9261  }
    9262  pBlockVector->Free(allocation);
    9263  }
    9264  break;
    9265  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9266  FreeDedicatedMemory(allocation);
    9267  break;
    9268  default:
    9269  VMA_ASSERT(0);
    9270  }
    9271  }
    9272 
    9273  allocation->SetUserData(this, VMA_NULL);
    9274  vma_delete(this, allocation);
    9275 }
    9276 
    9277 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
    9278 {
    9279  // Initialize.
    9280  InitStatInfo(pStats->total);
    9281  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
    9282  InitStatInfo(pStats->memoryType[i]);
    9283  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
    9284  InitStatInfo(pStats->memoryHeap[i]);
    9285 
    9286  // Process default pools.
    9287  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9288  {
    9289  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9290  VMA_ASSERT(pBlockVector);
    9291  pBlockVector->AddStats(pStats);
    9292  }
    9293 
    9294  // Process custom pools.
    9295  {
    9296  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9297  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9298  {
    9299  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
    9300  }
    9301  }
    9302 
    9303  // Process dedicated allocations.
    9304  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9305  {
    9306  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
    9307  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9308  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9309  VMA_ASSERT(pDedicatedAllocVector);
    9310  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
    9311  {
    9312  VmaStatInfo allocationStatInfo;
    9313  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
    9314  VmaAddStatInfo(pStats->total, allocationStatInfo);
    9315  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
    9316  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
    9317  }
    9318  }
    9319 
    9320  // Postprocess.
    9321  VmaPostprocessCalcStatInfo(pStats->total);
    9322  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
    9323  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
    9324  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
    9325  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
    9326 }
    9327 
    9328 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
    9329 
    9330 VkResult VmaAllocator_T::Defragment(
    9331  VmaAllocation* pAllocations,
    9332  size_t allocationCount,
    9333  VkBool32* pAllocationsChanged,
    9334  const VmaDefragmentationInfo* pDefragmentationInfo,
    9335  VmaDefragmentationStats* pDefragmentationStats)
    9336 {
    9337  if(pAllocationsChanged != VMA_NULL)
    9338  {
    9339  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
    9340  }
    9341  if(pDefragmentationStats != VMA_NULL)
    9342  {
    9343  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
    9344  }
    9345 
    9346  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
    9347 
    9348  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
    9349 
    9350  const size_t poolCount = m_Pools.size();
    9351 
    9352  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
    9353  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
    9354  {
    9355  VmaAllocation hAlloc = pAllocations[allocIndex];
    9356  VMA_ASSERT(hAlloc);
    9357  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
    9358  // DedicatedAlloc cannot be defragmented.
    9359  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
    9360  // Only HOST_VISIBLE memory types can be defragmented.
    9361  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
    9362  // Lost allocation cannot be defragmented.
    9363  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
    9364  {
    9365  VmaBlockVector* pAllocBlockVector = VMA_NULL;
    9366 
    9367  const VmaPool hAllocPool = hAlloc->GetPool();
    9368  // This allocation belongs to custom pool.
    9369  if(hAllocPool != VK_NULL_HANDLE)
    9370  {
    9371  pAllocBlockVector = &hAllocPool->GetBlockVector();
    9372  }
    9373  // This allocation belongs to general pool.
    9374  else
    9375  {
    9376  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
    9377  }
    9378 
    9379  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
    9380 
    9381  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
    9382  &pAllocationsChanged[allocIndex] : VMA_NULL;
    9383  pDefragmentator->AddAllocation(hAlloc, pChanged);
    9384  }
    9385  }
    9386 
    9387  VkResult result = VK_SUCCESS;
    9388 
    9389  // ======== Main processing.
    9390 
    9391  VkDeviceSize maxBytesToMove = SIZE_MAX;
    9392  uint32_t maxAllocationsToMove = UINT32_MAX;
    9393  if(pDefragmentationInfo != VMA_NULL)
    9394  {
    9395  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
    9396  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
    9397  }
    9398 
    9399  // Process standard memory.
    9400  for(uint32_t memTypeIndex = 0;
    9401  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
    9402  ++memTypeIndex)
    9403  {
    9404  // Only HOST_VISIBLE memory types can be defragmented.
    9405  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9406  {
    9407  result = m_pBlockVectors[memTypeIndex]->Defragment(
    9408  pDefragmentationStats,
    9409  maxBytesToMove,
    9410  maxAllocationsToMove);
    9411  }
    9412  }
    9413 
    9414  // Process custom pools.
    9415  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
    9416  {
    9417  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
    9418  pDefragmentationStats,
    9419  maxBytesToMove,
    9420  maxAllocationsToMove);
    9421  }
    9422 
    9423  // ======== Destroy defragmentators.
    9424 
    9425  // Process custom pools.
    9426  for(size_t poolIndex = poolCount; poolIndex--; )
    9427  {
    9428  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
    9429  }
    9430 
    9431  // Process standard memory.
    9432  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
    9433  {
    9434  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9435  {
    9436  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
    9437  }
    9438  }
    9439 
    9440  return result;
    9441 }
    9442 
    9443 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
    9444 {
    9445  if(hAllocation->CanBecomeLost())
    9446  {
    9447  /*
    9448  Warning: This is a carefully designed algorithm.
    9449  Do not modify unless you really know what you're doing :)
    9450  */
    9451  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9452  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9453  for(;;)
    9454  {
    9455  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9456  {
    9457  pAllocationInfo->memoryType = UINT32_MAX;
    9458  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
    9459  pAllocationInfo->offset = 0;
    9460  pAllocationInfo->size = hAllocation->GetSize();
    9461  pAllocationInfo->pMappedData = VMA_NULL;
    9462  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9463  return;
    9464  }
    9465  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9466  {
    9467  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9468  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9469  pAllocationInfo->offset = hAllocation->GetOffset();
    9470  pAllocationInfo->size = hAllocation->GetSize();
    9471  pAllocationInfo->pMappedData = VMA_NULL;
    9472  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9473  return;
    9474  }
    9475  else // Last use time earlier than current time.
    9476  {
    9477  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9478  {
    9479  localLastUseFrameIndex = localCurrFrameIndex;
    9480  }
    9481  }
    9482  }
    9483  }
    9484  else
    9485  {
    9486 #if VMA_STATS_STRING_ENABLED
    9487  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9488  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9489  for(;;)
    9490  {
    9491  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9492  if(localLastUseFrameIndex == localCurrFrameIndex)
    9493  {
    9494  break;
    9495  }
    9496  else // Last use time earlier than current time.
    9497  {
    9498  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9499  {
    9500  localLastUseFrameIndex = localCurrFrameIndex;
    9501  }
    9502  }
    9503  }
    9504 #endif
    9505 
    9506  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
    9507  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
    9508  pAllocationInfo->offset = hAllocation->GetOffset();
    9509  pAllocationInfo->size = hAllocation->GetSize();
    9510  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
    9511  pAllocationInfo->pUserData = hAllocation->GetUserData();
    9512  }
    9513 }
    9514 
    9515 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
    9516 {
    9517  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
    9518  if(hAllocation->CanBecomeLost())
    9519  {
    9520  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9521  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9522  for(;;)
    9523  {
    9524  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
    9525  {
    9526  return false;
    9527  }
    9528  else if(localLastUseFrameIndex == localCurrFrameIndex)
    9529  {
    9530  return true;
    9531  }
    9532  else // Last use time earlier than current time.
    9533  {
    9534  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9535  {
    9536  localLastUseFrameIndex = localCurrFrameIndex;
    9537  }
    9538  }
    9539  }
    9540  }
    9541  else
    9542  {
    9543 #if VMA_STATS_STRING_ENABLED
    9544  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
    9545  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
    9546  for(;;)
    9547  {
    9548  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
    9549  if(localLastUseFrameIndex == localCurrFrameIndex)
    9550  {
    9551  break;
    9552  }
    9553  else // Last use time earlier than current time.
    9554  {
    9555  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
    9556  {
    9557  localLastUseFrameIndex = localCurrFrameIndex;
    9558  }
    9559  }
    9560  }
    9561 #endif
    9562 
    9563  return true;
    9564  }
    9565 }
    9566 
    9567 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
    9568 {
    9569  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
    9570 
    9571  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
    9572 
    9573  if(newCreateInfo.maxBlockCount == 0)
    9574  {
    9575  newCreateInfo.maxBlockCount = SIZE_MAX;
    9576  }
    9577  if(newCreateInfo.blockSize == 0)
    9578  {
    9579  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
    9580  }
    9581 
    9582  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
    9583 
    9584  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
    9585  if(res != VK_SUCCESS)
    9586  {
    9587  vma_delete(this, *pPool);
    9588  *pPool = VMA_NULL;
    9589  return res;
    9590  }
    9591 
    9592  // Add to m_Pools.
    9593  {
    9594  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9595  (*pPool)->SetId(m_NextPoolId++);
    9596  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
    9597  }
    9598 
    9599  return VK_SUCCESS;
    9600 }
    9601 
    9602 void VmaAllocator_T::DestroyPool(VmaPool pool)
    9603 {
    9604  // Remove from m_Pools.
    9605  {
    9606  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9607  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
    9608  VMA_ASSERT(success && "Pool not found in Allocator.");
    9609  }
    9610 
    9611  vma_delete(this, pool);
    9612 }
    9613 
    9614 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
    9615 {
    9616  pool->m_BlockVector.GetPoolStats(pPoolStats);
    9617 }
    9618 
    9619 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
    9620 {
    9621  m_CurrentFrameIndex.store(frameIndex);
    9622 }
    9623 
    9624 void VmaAllocator_T::MakePoolAllocationsLost(
    9625  VmaPool hPool,
    9626  size_t* pLostAllocationCount)
    9627 {
    9628  hPool->m_BlockVector.MakePoolAllocationsLost(
    9629  m_CurrentFrameIndex.load(),
    9630  pLostAllocationCount);
    9631 }
    9632 
    9633 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
    9634 {
    9635  return hPool->m_BlockVector.CheckCorruption();
    9636 }
    9637 
    9638 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
    9639 {
    9640  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
    9641 
    9642  // Process default pools.
    9643  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9644  {
    9645  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
    9646  {
    9647  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
    9648  VMA_ASSERT(pBlockVector);
    9649  VkResult localRes = pBlockVector->CheckCorruption();
    9650  switch(localRes)
    9651  {
    9652  case VK_ERROR_FEATURE_NOT_PRESENT:
    9653  break;
    9654  case VK_SUCCESS:
    9655  finalRes = VK_SUCCESS;
    9656  break;
    9657  default:
    9658  return localRes;
    9659  }
    9660  }
    9661  }
    9662 
    9663  // Process custom pools.
    9664  {
    9665  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    9666  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
    9667  {
    9668  if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0)
    9669  {
    9670  VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption();
    9671  switch(localRes)
    9672  {
    9673  case VK_ERROR_FEATURE_NOT_PRESENT:
    9674  break;
    9675  case VK_SUCCESS:
    9676  finalRes = VK_SUCCESS;
    9677  break;
    9678  default:
    9679  return localRes;
    9680  }
    9681  }
    9682  }
    9683  }
    9684 
    9685  return finalRes;
    9686 }
    9687 
    9688 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
    9689 {
    9690  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
    9691  (*pAllocation)->InitLost();
    9692 }
    9693 
    9694 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
    9695 {
    9696  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
    9697 
    9698  VkResult res;
    9699  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9700  {
    9701  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9702  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
    9703  {
    9704  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9705  if(res == VK_SUCCESS)
    9706  {
    9707  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
    9708  }
    9709  }
    9710  else
    9711  {
    9712  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
    9713  }
    9714  }
    9715  else
    9716  {
    9717  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
    9718  }
    9719 
    9720  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
    9721  {
    9722  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
    9723  }
    9724 
    9725  return res;
    9726 }
    9727 
    9728 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
    9729 {
    9730  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
    9731  {
    9732  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
    9733  }
    9734 
    9735  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
    9736 
    9737  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
    9738  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
    9739  {
    9740  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
    9741  m_HeapSizeLimit[heapIndex] += size;
    9742  }
    9743 }
    9744 
    9745 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
    9746 {
    9747  if(hAllocation->CanBecomeLost())
    9748  {
    9749  return VK_ERROR_MEMORY_MAP_FAILED;
    9750  }
    9751 
    9752  switch(hAllocation->GetType())
    9753  {
    9754  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9755  {
    9756  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9757  char *pBytes = VMA_NULL;
    9758  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
    9759  if(res == VK_SUCCESS)
    9760  {
    9761  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
    9762  hAllocation->BlockAllocMap();
    9763  }
    9764  return res;
    9765  }
    9766  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9767  return hAllocation->DedicatedAllocMap(this, ppData);
    9768  default:
    9769  VMA_ASSERT(0);
    9770  return VK_ERROR_MEMORY_MAP_FAILED;
    9771  }
    9772 }
    9773 
    9774 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
    9775 {
    9776  switch(hAllocation->GetType())
    9777  {
    9778  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9779  {
    9780  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
    9781  hAllocation->BlockAllocUnmap();
    9782  pBlock->Unmap(this, 1);
    9783  }
    9784  break;
    9785  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9786  hAllocation->DedicatedAllocUnmap(this);
    9787  break;
    9788  default:
    9789  VMA_ASSERT(0);
    9790  }
    9791 }
    9792 
    9793 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
    9794 {
    9795  VkResult res = VK_SUCCESS;
    9796  switch(hAllocation->GetType())
    9797  {
    9798  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9799  res = GetVulkanFunctions().vkBindBufferMemory(
    9800  m_hDevice,
    9801  hBuffer,
    9802  hAllocation->GetMemory(),
    9803  0); //memoryOffset
    9804  break;
    9805  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9806  {
    9807  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9808  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
    9809  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
    9810  break;
    9811  }
    9812  default:
    9813  VMA_ASSERT(0);
    9814  }
    9815  return res;
    9816 }
    9817 
    9818 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
    9819 {
    9820  VkResult res = VK_SUCCESS;
    9821  switch(hAllocation->GetType())
    9822  {
    9823  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9824  res = GetVulkanFunctions().vkBindImageMemory(
    9825  m_hDevice,
    9826  hImage,
    9827  hAllocation->GetMemory(),
    9828  0); //memoryOffset
    9829  break;
    9830  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9831  {
    9832  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
    9833  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
    9834  res = pBlock->BindImageMemory(this, hAllocation, hImage);
    9835  break;
    9836  }
    9837  default:
    9838  VMA_ASSERT(0);
    9839  }
    9840  return res;
    9841 }
    9842 
    9843 void VmaAllocator_T::FlushOrInvalidateAllocation(
    9844  VmaAllocation hAllocation,
    9845  VkDeviceSize offset, VkDeviceSize size,
    9846  VMA_CACHE_OPERATION op)
    9847 {
    9848  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
    9849  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
    9850  {
    9851  const VkDeviceSize allocationSize = hAllocation->GetSize();
    9852  VMA_ASSERT(offset <= allocationSize);
    9853 
    9854  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
    9855 
    9856  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
    9857  memRange.memory = hAllocation->GetMemory();
    9858 
    9859  switch(hAllocation->GetType())
    9860  {
    9861  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
    9862  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9863  if(size == VK_WHOLE_SIZE)
    9864  {
    9865  memRange.size = allocationSize - memRange.offset;
    9866  }
    9867  else
    9868  {
    9869  VMA_ASSERT(offset + size <= allocationSize);
    9870  memRange.size = VMA_MIN(
    9871  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
    9872  allocationSize - memRange.offset);
    9873  }
    9874  break;
    9875 
    9876  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
    9877  {
    9878  // 1. Still within this allocation.
    9879  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
    9880  if(size == VK_WHOLE_SIZE)
    9881  {
    9882  size = allocationSize - offset;
    9883  }
    9884  else
    9885  {
    9886  VMA_ASSERT(offset + size <= allocationSize);
    9887  }
    9888  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
    9889 
    9890  // 2. Adjust to whole block.
    9891  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
    9892  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
    9893  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize();
    9894  memRange.offset += allocationOffset;
    9895  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
    9896 
    9897  break;
    9898  }
    9899 
    9900  default:
    9901  VMA_ASSERT(0);
    9902  }
    9903 
    9904  switch(op)
    9905  {
    9906  case VMA_CACHE_FLUSH:
    9907  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9908  break;
    9909  case VMA_CACHE_INVALIDATE:
    9910  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
    9911  break;
    9912  default:
    9913  VMA_ASSERT(0);
    9914  }
    9915  }
    9916  // else: Just ignore this call.
    9917 }
    9918 
    9919 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
    9920 {
    9921  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
    9922 
    9923  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
    9924  {
    9925  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9926  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
    9927  VMA_ASSERT(pDedicatedAllocations);
    9928  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
    9929  VMA_ASSERT(success);
    9930  }
    9931 
    9932  VkDeviceMemory hMemory = allocation->GetMemory();
    9933 
    9934  if(allocation->GetMappedData() != VMA_NULL)
    9935  {
    9936  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
    9937  }
    9938 
    9939  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
    9940 
    9941  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
    9942 }
    9943 
    9944 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
    9945 {
    9946  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
    9947  !hAllocation->CanBecomeLost() &&
    9948  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    9949  {
    9950  void* pData = VMA_NULL;
    9951  VkResult res = Map(hAllocation, &pData);
    9952  if(res == VK_SUCCESS)
    9953  {
    9954  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
    9955  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
    9956  Unmap(hAllocation);
    9957  }
    9958  else
    9959  {
    9960  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
    9961  }
    9962  }
    9963 }
    9964 
    9965 #if VMA_STATS_STRING_ENABLED
    9966 
    9967 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
    9968 {
    9969  bool dedicatedAllocationsStarted = false;
    9970  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    9971  {
    9972  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
    9973  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
    9974  VMA_ASSERT(pDedicatedAllocVector);
    9975  if(pDedicatedAllocVector->empty() == false)
    9976  {
    9977  if(dedicatedAllocationsStarted == false)
    9978  {
    9979  dedicatedAllocationsStarted = true;
    9980  json.WriteString("DedicatedAllocations");
    9981  json.BeginObject();
    9982  }
    9983 
    9984  json.BeginString("Type ");
    9985  json.ContinueString(memTypeIndex);
    9986  json.EndString();
    9987 
    9988  json.BeginArray();
    9989 
    9990  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
    9991  {
    9992  json.BeginObject(true);
    9993  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
    9994  hAlloc->PrintParameters(json);
    9995  json.EndObject();
    9996  }
    9997 
    9998  json.EndArray();
    9999  }
    10000  }
    10001  if(dedicatedAllocationsStarted)
    10002  {
    10003  json.EndObject();
    10004  }
    10005 
    10006  {
    10007  bool allocationsStarted = false;
    10008  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
    10009  {
    10010  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
    10011  {
    10012  if(allocationsStarted == false)
    10013  {
    10014  allocationsStarted = true;
    10015  json.WriteString("DefaultPools");
    10016  json.BeginObject();
    10017  }
    10018 
    10019  json.BeginString("Type ");
    10020  json.ContinueString(memTypeIndex);
    10021  json.EndString();
    10022 
    10023  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
    10024  }
    10025  }
    10026  if(allocationsStarted)
    10027  {
    10028  json.EndObject();
    10029  }
    10030  }
    10031 
    10032  {
    10033  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
    10034  const size_t poolCount = m_Pools.size();
    10035  if(poolCount > 0)
    10036  {
    10037  json.WriteString("Pools");
    10038  json.BeginObject();
    10039  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
    10040  {
    10041  json.BeginString();
    10042  json.ContinueString(m_Pools[poolIndex]->GetId());
    10043  json.EndString();
    10044 
    10045  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
    10046  }
    10047  json.EndObject();
    10048  }
    10049  }
    10050 }
    10051 
    10052 #endif // #if VMA_STATS_STRING_ENABLED
    10053 
    10055 // Public interface
    10056 
    10057 VkResult vmaCreateAllocator(
    10058  const VmaAllocatorCreateInfo* pCreateInfo,
    10059  VmaAllocator* pAllocator)
    10060 {
    10061  VMA_ASSERT(pCreateInfo && pAllocator);
    10062  VMA_DEBUG_LOG("vmaCreateAllocator");
    10063  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
    10064  return (*pAllocator)->Init(pCreateInfo);
    10065 }
    10066 
    10067 void vmaDestroyAllocator(
    10068  VmaAllocator allocator)
    10069 {
    10070  if(allocator != VK_NULL_HANDLE)
    10071  {
    10072  VMA_DEBUG_LOG("vmaDestroyAllocator");
    10073  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
    10074  vma_delete(&allocationCallbacks, allocator);
    10075  }
    10076 }
    10077 
    10078 void vmaGetPhysicalDeviceProperties(
    10079  VmaAllocator allocator,
    10080  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
    10081 {
    10082  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
    10083  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
    10084 }
    10085 
    10086 void vmaGetMemoryProperties(
    10087  VmaAllocator allocator,
    10088  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
    10089 {
    10090  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
    10091  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
    10092 }
    10093 
    10094 void vmaGetMemoryTypeProperties(
    10095  VmaAllocator allocator,
    10096  uint32_t memoryTypeIndex,
    10097  VkMemoryPropertyFlags* pFlags)
    10098 {
    10099  VMA_ASSERT(allocator && pFlags);
    10100  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
    10101  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
    10102 }
    10103 
    10104 void vmaSetCurrentFrameIndex(
    10105  VmaAllocator allocator,
    10106  uint32_t frameIndex)
    10107 {
    10108  VMA_ASSERT(allocator);
    10109  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
    10110 
    10111  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10112 
    10113  allocator->SetCurrentFrameIndex(frameIndex);
    10114 }
    10115 
    10116 void vmaCalculateStats(
    10117  VmaAllocator allocator,
    10118  VmaStats* pStats)
    10119 {
    10120  VMA_ASSERT(allocator && pStats);
    10121  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10122  allocator->CalculateStats(pStats);
    10123 }
    10124 
    10125 #if VMA_STATS_STRING_ENABLED
    10126 
    10127 void vmaBuildStatsString(
    10128  VmaAllocator allocator,
    10129  char** ppStatsString,
    10130  VkBool32 detailedMap)
    10131 {
    10132  VMA_ASSERT(allocator && ppStatsString);
    10133  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10134 
    10135  VmaStringBuilder sb(allocator);
    10136  {
    10137  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
    10138  json.BeginObject();
    10139 
    10140  VmaStats stats;
    10141  allocator->CalculateStats(&stats);
    10142 
    10143  json.WriteString("Total");
    10144  VmaPrintStatInfo(json, stats.total);
    10145 
    10146  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
    10147  {
    10148  json.BeginString("Heap ");
    10149  json.ContinueString(heapIndex);
    10150  json.EndString();
    10151  json.BeginObject();
    10152 
    10153  json.WriteString("Size");
    10154  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
    10155 
    10156  json.WriteString("Flags");
    10157  json.BeginArray(true);
    10158  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
    10159  {
    10160  json.WriteString("DEVICE_LOCAL");
    10161  }
    10162  json.EndArray();
    10163 
    10164  if(stats.memoryHeap[heapIndex].blockCount > 0)
    10165  {
    10166  json.WriteString("Stats");
    10167  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
    10168  }
    10169 
    10170  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
    10171  {
    10172  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
    10173  {
    10174  json.BeginString("Type ");
    10175  json.ContinueString(typeIndex);
    10176  json.EndString();
    10177 
    10178  json.BeginObject();
    10179 
    10180  json.WriteString("Flags");
    10181  json.BeginArray(true);
    10182  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
    10183  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
    10184  {
    10185  json.WriteString("DEVICE_LOCAL");
    10186  }
    10187  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
    10188  {
    10189  json.WriteString("HOST_VISIBLE");
    10190  }
    10191  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
    10192  {
    10193  json.WriteString("HOST_COHERENT");
    10194  }
    10195  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
    10196  {
    10197  json.WriteString("HOST_CACHED");
    10198  }
    10199  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
    10200  {
    10201  json.WriteString("LAZILY_ALLOCATED");
    10202  }
    10203  json.EndArray();
    10204 
    10205  if(stats.memoryType[typeIndex].blockCount > 0)
    10206  {
    10207  json.WriteString("Stats");
    10208  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
    10209  }
    10210 
    10211  json.EndObject();
    10212  }
    10213  }
    10214 
    10215  json.EndObject();
    10216  }
    10217  if(detailedMap == VK_TRUE)
    10218  {
    10219  allocator->PrintDetailedMap(json);
    10220  }
    10221 
    10222  json.EndObject();
    10223  }
    10224 
    10225  const size_t len = sb.GetLength();
    10226  char* const pChars = vma_new_array(allocator, char, len + 1);
    10227  if(len > 0)
    10228  {
    10229  memcpy(pChars, sb.GetData(), len);
    10230  }
    10231  pChars[len] = '\0';
    10232  *ppStatsString = pChars;
    10233 }
    10234 
    10235 void vmaFreeStatsString(
    10236  VmaAllocator allocator,
    10237  char* pStatsString)
    10238 {
    10239  if(pStatsString != VMA_NULL)
    10240  {
    10241  VMA_ASSERT(allocator);
    10242  size_t len = strlen(pStatsString);
    10243  vma_delete_array(allocator, pStatsString, len + 1);
    10244  }
    10245 }
    10246 
    10247 #endif // #if VMA_STATS_STRING_ENABLED
    10248 
    10249 /*
    10250 This function is not protected by any mutex because it just reads immutable data.
    10251 */
    10252 VkResult vmaFindMemoryTypeIndex(
    10253  VmaAllocator allocator,
    10254  uint32_t memoryTypeBits,
    10255  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10256  uint32_t* pMemoryTypeIndex)
    10257 {
    10258  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10259  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10260  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10261 
    10262  if(pAllocationCreateInfo->memoryTypeBits != 0)
    10263  {
    10264  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
    10265  }
    10266 
    10267  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
    10268  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
    10269 
    10270  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
    10271  if(mapped)
    10272  {
    10273  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10274  }
    10275 
    10276  // Convert usage to requiredFlags and preferredFlags.
    10277  switch(pAllocationCreateInfo->usage)
    10278  {
    10279  case VMA_MEMORY_USAGE_UNKNOWN:
    10280  break;
    10281  case VMA_MEMORY_USAGE_GPU_ONLY:
    10282  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10283  {
    10284  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10285  }
    10286  break;
    10287  case VMA_MEMORY_USAGE_CPU_ONLY:
    10288  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    10289  break;
    10290  case VMA_MEMORY_USAGE_CPU_TO_GPU:
    10291  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10292  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
    10293  {
    10294  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
    10295  }
    10296  break;
    10297  case VMA_MEMORY_USAGE_GPU_TO_CPU:
    10298  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    10299  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
    10300  break;
    10301  default:
    10302  break;
    10303  }
    10304 
    10305  *pMemoryTypeIndex = UINT32_MAX;
    10306  uint32_t minCost = UINT32_MAX;
    10307  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
    10308  memTypeIndex < allocator->GetMemoryTypeCount();
    10309  ++memTypeIndex, memTypeBit <<= 1)
    10310  {
    10311  // This memory type is acceptable according to memoryTypeBits bitmask.
    10312  if((memTypeBit & memoryTypeBits) != 0)
    10313  {
    10314  const VkMemoryPropertyFlags currFlags =
    10315  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
    10316  // This memory type contains requiredFlags.
    10317  if((requiredFlags & ~currFlags) == 0)
    10318  {
    10319  // Calculate cost as number of bits from preferredFlags not present in this memory type.
    10320  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
    10321  // Remember memory type with lowest cost.
    10322  if(currCost < minCost)
    10323  {
    10324  *pMemoryTypeIndex = memTypeIndex;
    10325  if(currCost == 0)
    10326  {
    10327  return VK_SUCCESS;
    10328  }
    10329  minCost = currCost;
    10330  }
    10331  }
    10332  }
    10333  }
    10334  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
    10335 }
    10336 
    10337 VkResult vmaFindMemoryTypeIndexForBufferInfo(
    10338  VmaAllocator allocator,
    10339  const VkBufferCreateInfo* pBufferCreateInfo,
    10340  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10341  uint32_t* pMemoryTypeIndex)
    10342 {
    10343  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10344  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
    10345  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10346  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10347 
    10348  const VkDevice hDev = allocator->m_hDevice;
    10349  VkBuffer hBuffer = VK_NULL_HANDLE;
    10350  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
    10351  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
    10352  if(res == VK_SUCCESS)
    10353  {
    10354  VkMemoryRequirements memReq = {};
    10355  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
    10356  hDev, hBuffer, &memReq);
    10357 
    10358  res = vmaFindMemoryTypeIndex(
    10359  allocator,
    10360  memReq.memoryTypeBits,
    10361  pAllocationCreateInfo,
    10362  pMemoryTypeIndex);
    10363 
    10364  allocator->GetVulkanFunctions().vkDestroyBuffer(
    10365  hDev, hBuffer, allocator->GetAllocationCallbacks());
    10366  }
    10367  return res;
    10368 }
    10369 
    10370 VkResult vmaFindMemoryTypeIndexForImageInfo(
    10371  VmaAllocator allocator,
    10372  const VkImageCreateInfo* pImageCreateInfo,
    10373  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10374  uint32_t* pMemoryTypeIndex)
    10375 {
    10376  VMA_ASSERT(allocator != VK_NULL_HANDLE);
    10377  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
    10378  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
    10379  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
    10380 
    10381  const VkDevice hDev = allocator->m_hDevice;
    10382  VkImage hImage = VK_NULL_HANDLE;
    10383  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
    10384  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
    10385  if(res == VK_SUCCESS)
    10386  {
    10387  VkMemoryRequirements memReq = {};
    10388  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
    10389  hDev, hImage, &memReq);
    10390 
    10391  res = vmaFindMemoryTypeIndex(
    10392  allocator,
    10393  memReq.memoryTypeBits,
    10394  pAllocationCreateInfo,
    10395  pMemoryTypeIndex);
    10396 
    10397  allocator->GetVulkanFunctions().vkDestroyImage(
    10398  hDev, hImage, allocator->GetAllocationCallbacks());
    10399  }
    10400  return res;
    10401 }
    10402 
    10403 VkResult vmaCreatePool(
    10404  VmaAllocator allocator,
    10405  const VmaPoolCreateInfo* pCreateInfo,
    10406  VmaPool* pPool)
    10407 {
    10408  VMA_ASSERT(allocator && pCreateInfo && pPool);
    10409 
    10410  VMA_DEBUG_LOG("vmaCreatePool");
    10411 
    10412  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10413 
    10414  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
    10415 
    10416 #if VMA_RECORDING_ENABLED
    10417  if(allocator->GetRecorder() != VMA_NULL)
    10418  {
    10419  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
    10420  }
    10421 #endif
    10422 
    10423  return res;
    10424 }
    10425 
    10426 void vmaDestroyPool(
    10427  VmaAllocator allocator,
    10428  VmaPool pool)
    10429 {
    10430  VMA_ASSERT(allocator);
    10431 
    10432  if(pool == VK_NULL_HANDLE)
    10433  {
    10434  return;
    10435  }
    10436 
    10437  VMA_DEBUG_LOG("vmaDestroyPool");
    10438 
    10439  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10440 
    10441 #if VMA_RECORDING_ENABLED
    10442  if(allocator->GetRecorder() != VMA_NULL)
    10443  {
    10444  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
    10445  }
    10446 #endif
    10447 
    10448  allocator->DestroyPool(pool);
    10449 }
    10450 
    10451 void vmaGetPoolStats(
    10452  VmaAllocator allocator,
    10453  VmaPool pool,
    10454  VmaPoolStats* pPoolStats)
    10455 {
    10456  VMA_ASSERT(allocator && pool && pPoolStats);
    10457 
    10458  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10459 
    10460  allocator->GetPoolStats(pool, pPoolStats);
    10461 }
    10462 
    10463 void vmaMakePoolAllocationsLost(
    10464  VmaAllocator allocator,
    10465  VmaPool pool,
    10466  size_t* pLostAllocationCount)
    10467 {
    10468  VMA_ASSERT(allocator && pool);
    10469 
    10470  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10471 
    10472 #if VMA_RECORDING_ENABLED
    10473  if(allocator->GetRecorder() != VMA_NULL)
    10474  {
    10475  allocator->GetRecorder()->RecordMakePoolAllocationsLost(allocator->GetCurrentFrameIndex(), pool);
    10476  }
    10477 #endif
    10478 
    10479  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
    10480 }
    10481 
    10482 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
    10483 {
    10484  VMA_ASSERT(allocator && pool);
    10485 
    10486  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10487 
    10488  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
    10489 
    10490  return allocator->CheckPoolCorruption(pool);
    10491 }
    10492 
    10493 VkResult vmaAllocateMemory(
    10494  VmaAllocator allocator,
    10495  const VkMemoryRequirements* pVkMemoryRequirements,
    10496  const VmaAllocationCreateInfo* pCreateInfo,
    10497  VmaAllocation* pAllocation,
    10498  VmaAllocationInfo* pAllocationInfo)
    10499 {
    10500  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
    10501 
    10502  VMA_DEBUG_LOG("vmaAllocateMemory");
    10503 
    10504  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10505 
    10506  VkResult result = allocator->AllocateMemory(
    10507  *pVkMemoryRequirements,
    10508  false, // requiresDedicatedAllocation
    10509  false, // prefersDedicatedAllocation
    10510  VK_NULL_HANDLE, // dedicatedBuffer
    10511  VK_NULL_HANDLE, // dedicatedImage
    10512  *pCreateInfo,
    10513  VMA_SUBALLOCATION_TYPE_UNKNOWN,
    10514  pAllocation);
    10515 
    10516 #if VMA_RECORDING_ENABLED
    10517  if(allocator->GetRecorder() != VMA_NULL)
    10518  {
    10519  allocator->GetRecorder()->RecordAllocateMemory(
    10520  allocator->GetCurrentFrameIndex(),
    10521  *pVkMemoryRequirements,
    10522  *pCreateInfo,
    10523  *pAllocation);
    10524  }
    10525 #endif
    10526 
    10527  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
    10528  {
    10529  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10530  }
    10531 
    10532  return result;
    10533 }
    10534 
    10535 VkResult vmaAllocateMemoryForBuffer(
    10536  VmaAllocator allocator,
    10537  VkBuffer buffer,
    10538  const VmaAllocationCreateInfo* pCreateInfo,
    10539  VmaAllocation* pAllocation,
    10540  VmaAllocationInfo* pAllocationInfo)
    10541 {
    10542  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10543 
    10544  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
    10545 
    10546  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10547 
    10548  VkMemoryRequirements vkMemReq = {};
    10549  bool requiresDedicatedAllocation = false;
    10550  bool prefersDedicatedAllocation = false;
    10551  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
    10552  requiresDedicatedAllocation,
    10553  prefersDedicatedAllocation);
    10554 
    10555  VkResult result = allocator->AllocateMemory(
    10556  vkMemReq,
    10557  requiresDedicatedAllocation,
    10558  prefersDedicatedAllocation,
    10559  buffer, // dedicatedBuffer
    10560  VK_NULL_HANDLE, // dedicatedImage
    10561  *pCreateInfo,
    10562  VMA_SUBALLOCATION_TYPE_BUFFER,
    10563  pAllocation);
    10564 
    10565 #if VMA_RECORDING_ENABLED
    10566  if(allocator->GetRecorder() != VMA_NULL)
    10567  {
    10568  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
    10569  allocator->GetCurrentFrameIndex(),
    10570  vkMemReq,
    10571  requiresDedicatedAllocation,
    10572  prefersDedicatedAllocation,
    10573  *pCreateInfo,
    10574  *pAllocation);
    10575  }
    10576 #endif
    10577 
    10578  if(pAllocationInfo && result == VK_SUCCESS)
    10579  {
    10580  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10581  }
    10582 
    10583  return result;
    10584 }
    10585 
    10586 VkResult vmaAllocateMemoryForImage(
    10587  VmaAllocator allocator,
    10588  VkImage image,
    10589  const VmaAllocationCreateInfo* pCreateInfo,
    10590  VmaAllocation* pAllocation,
    10591  VmaAllocationInfo* pAllocationInfo)
    10592 {
    10593  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
    10594 
    10595  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
    10596 
    10597  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10598 
    10599  VkMemoryRequirements vkMemReq = {};
    10600  bool requiresDedicatedAllocation = false;
    10601  bool prefersDedicatedAllocation = false;
    10602  allocator->GetImageMemoryRequirements(image, vkMemReq,
    10603  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10604 
    10605  VkResult result = allocator->AllocateMemory(
    10606  vkMemReq,
    10607  requiresDedicatedAllocation,
    10608  prefersDedicatedAllocation,
    10609  VK_NULL_HANDLE, // dedicatedBuffer
    10610  image, // dedicatedImage
    10611  *pCreateInfo,
    10612  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
    10613  pAllocation);
    10614 
    10615 #if VMA_RECORDING_ENABLED
    10616  if(allocator->GetRecorder() != VMA_NULL)
    10617  {
    10618  allocator->GetRecorder()->RecordAllocateMemoryForImage(
    10619  allocator->GetCurrentFrameIndex(),
    10620  vkMemReq,
    10621  requiresDedicatedAllocation,
    10622  prefersDedicatedAllocation,
    10623  *pCreateInfo,
    10624  *pAllocation);
    10625  }
    10626 #endif
    10627 
    10628  if(pAllocationInfo && result == VK_SUCCESS)
    10629  {
    10630  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10631  }
    10632 
    10633  return result;
    10634 }
    10635 
    10636 void vmaFreeMemory(
    10637  VmaAllocator allocator,
    10638  VmaAllocation allocation)
    10639 {
    10640  VMA_ASSERT(allocator);
    10641 
    10642  if(allocation == VK_NULL_HANDLE)
    10643  {
    10644  return;
    10645  }
    10646 
    10647  VMA_DEBUG_LOG("vmaFreeMemory");
    10648 
    10649  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10650 
    10651 #if VMA_RECORDING_ENABLED
    10652  if(allocator->GetRecorder() != VMA_NULL)
    10653  {
    10654  allocator->GetRecorder()->RecordFreeMemory(
    10655  allocator->GetCurrentFrameIndex(),
    10656  allocation);
    10657  }
    10658 #endif
    10659 
    10660  allocator->FreeMemory(allocation);
    10661 }
    10662 
    10663 void vmaGetAllocationInfo(
    10664  VmaAllocator allocator,
    10665  VmaAllocation allocation,
    10666  VmaAllocationInfo* pAllocationInfo)
    10667 {
    10668  VMA_ASSERT(allocator && allocation && pAllocationInfo);
    10669 
    10670  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10671 
    10672 #if VMA_RECORDING_ENABLED
    10673  if(allocator->GetRecorder() != VMA_NULL)
    10674  {
    10675  allocator->GetRecorder()->RecordGetAllocationInfo(
    10676  allocator->GetCurrentFrameIndex(),
    10677  allocation);
    10678  }
    10679 #endif
    10680 
    10681  allocator->GetAllocationInfo(allocation, pAllocationInfo);
    10682 }
    10683 
    10684 VkBool32 vmaTouchAllocation(
    10685  VmaAllocator allocator,
    10686  VmaAllocation allocation)
    10687 {
    10688  VMA_ASSERT(allocator && allocation);
    10689 
    10690  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10691 
    10692 #if VMA_RECORDING_ENABLED
    10693  if(allocator->GetRecorder() != VMA_NULL)
    10694  {
    10695  allocator->GetRecorder()->RecordTouchAllocation(
    10696  allocator->GetCurrentFrameIndex(),
    10697  allocation);
    10698  }
    10699 #endif
    10700 
    10701  return allocator->TouchAllocation(allocation);
    10702 }
    10703 
    10704 void vmaSetAllocationUserData(
    10705  VmaAllocator allocator,
    10706  VmaAllocation allocation,
    10707  void* pUserData)
    10708 {
    10709  VMA_ASSERT(allocator && allocation);
    10710 
    10711  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10712 
    10713  allocation->SetUserData(allocator, pUserData);
    10714 
    10715 #if VMA_RECORDING_ENABLED
    10716  if(allocator->GetRecorder() != VMA_NULL)
    10717  {
    10718  allocator->GetRecorder()->RecordSetAllocationUserData(
    10719  allocator->GetCurrentFrameIndex(),
    10720  allocation,
    10721  pUserData);
    10722  }
    10723 #endif
    10724 }
    10725 
    10726 void vmaCreateLostAllocation(
    10727  VmaAllocator allocator,
    10728  VmaAllocation* pAllocation)
    10729 {
    10730  VMA_ASSERT(allocator && pAllocation);
    10731 
    10732  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
    10733 
    10734  allocator->CreateLostAllocation(pAllocation);
    10735 
    10736 #if VMA_RECORDING_ENABLED
    10737  if(allocator->GetRecorder() != VMA_NULL)
    10738  {
    10739  allocator->GetRecorder()->RecordCreateLostAllocation(
    10740  allocator->GetCurrentFrameIndex(),
    10741  *pAllocation);
    10742  }
    10743 #endif
    10744 }
    10745 
    10746 VkResult vmaMapMemory(
    10747  VmaAllocator allocator,
    10748  VmaAllocation allocation,
    10749  void** ppData)
    10750 {
    10751  VMA_ASSERT(allocator && allocation && ppData);
    10752 
    10753  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10754 
    10755  VkResult res = allocator->Map(allocation, ppData);
    10756 
    10757 #if VMA_RECORDING_ENABLED
    10758  if(allocator->GetRecorder() != VMA_NULL)
    10759  {
    10760  allocator->GetRecorder()->RecordMapMemory(
    10761  allocator->GetCurrentFrameIndex(),
    10762  allocation);
    10763  }
    10764 #endif
    10765 
    10766  return res;
    10767 }
    10768 
    10769 void vmaUnmapMemory(
    10770  VmaAllocator allocator,
    10771  VmaAllocation allocation)
    10772 {
    10773  VMA_ASSERT(allocator && allocation);
    10774 
    10775  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10776 
    10777 #if VMA_RECORDING_ENABLED
    10778  if(allocator->GetRecorder() != VMA_NULL)
    10779  {
    10780  allocator->GetRecorder()->RecordUnmapMemory(
    10781  allocator->GetCurrentFrameIndex(),
    10782  allocation);
    10783  }
    10784 #endif
    10785 
    10786  allocator->Unmap(allocation);
    10787 }
    10788 
    10789 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10790 {
    10791  VMA_ASSERT(allocator && allocation);
    10792 
    10793  VMA_DEBUG_LOG("vmaFlushAllocation");
    10794 
    10795  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10796 
    10797  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
    10798 
    10799 #if VMA_RECORDING_ENABLED
    10800  if(allocator->GetRecorder() != VMA_NULL)
    10801  {
    10802  allocator->GetRecorder()->RecordFlushAllocation(
    10803  allocator->GetCurrentFrameIndex(),
    10804  allocation, offset, size);
    10805  }
    10806 #endif
    10807 }
    10808 
    10809 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
    10810 {
    10811  VMA_ASSERT(allocator && allocation);
    10812 
    10813  VMA_DEBUG_LOG("vmaInvalidateAllocation");
    10814 
    10815  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10816 
    10817  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
    10818 
    10819 #if VMA_RECORDING_ENABLED
    10820  if(allocator->GetRecorder() != VMA_NULL)
    10821  {
    10822  allocator->GetRecorder()->RecordInvalidateAllocation(
    10823  allocator->GetCurrentFrameIndex(),
    10824  allocation, offset, size);
    10825  }
    10826 #endif
    10827 }
    10828 
    10829 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
    10830 {
    10831  VMA_ASSERT(allocator);
    10832 
    10833  VMA_DEBUG_LOG("vmaCheckCorruption");
    10834 
    10835  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10836 
    10837  return allocator->CheckCorruption(memoryTypeBits);
    10838 }
    10839 
    10840 VkResult vmaDefragment(
    10841  VmaAllocator allocator,
    10842  VmaAllocation* pAllocations,
    10843  size_t allocationCount,
    10844  VkBool32* pAllocationsChanged,
    10845  const VmaDefragmentationInfo *pDefragmentationInfo,
    10846  VmaDefragmentationStats* pDefragmentationStats)
    10847 {
    10848  VMA_ASSERT(allocator && pAllocations);
    10849 
    10850  VMA_DEBUG_LOG("vmaDefragment");
    10851 
    10852  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10853 
    10854  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
    10855 }
    10856 
    10857 VkResult vmaBindBufferMemory(
    10858  VmaAllocator allocator,
    10859  VmaAllocation allocation,
    10860  VkBuffer buffer)
    10861 {
    10862  VMA_ASSERT(allocator && allocation && buffer);
    10863 
    10864  VMA_DEBUG_LOG("vmaBindBufferMemory");
    10865 
    10866  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10867 
    10868  return allocator->BindBufferMemory(allocation, buffer);
    10869 }
    10870 
    10871 VkResult vmaBindImageMemory(
    10872  VmaAllocator allocator,
    10873  VmaAllocation allocation,
    10874  VkImage image)
    10875 {
    10876  VMA_ASSERT(allocator && allocation && image);
    10877 
    10878  VMA_DEBUG_LOG("vmaBindImageMemory");
    10879 
    10880  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10881 
    10882  return allocator->BindImageMemory(allocation, image);
    10883 }
    10884 
    10885 VkResult vmaCreateBuffer(
    10886  VmaAllocator allocator,
    10887  const VkBufferCreateInfo* pBufferCreateInfo,
    10888  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    10889  VkBuffer* pBuffer,
    10890  VmaAllocation* pAllocation,
    10891  VmaAllocationInfo* pAllocationInfo)
    10892 {
    10893  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
    10894 
    10895  VMA_DEBUG_LOG("vmaCreateBuffer");
    10896 
    10897  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    10898 
    10899  *pBuffer = VK_NULL_HANDLE;
    10900  *pAllocation = VK_NULL_HANDLE;
    10901 
    10902  // 1. Create VkBuffer.
    10903  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
    10904  allocator->m_hDevice,
    10905  pBufferCreateInfo,
    10906  allocator->GetAllocationCallbacks(),
    10907  pBuffer);
    10908  if(res >= 0)
    10909  {
    10910  // 2. vkGetBufferMemoryRequirements.
    10911  VkMemoryRequirements vkMemReq = {};
    10912  bool requiresDedicatedAllocation = false;
    10913  bool prefersDedicatedAllocation = false;
    10914  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
    10915  requiresDedicatedAllocation, prefersDedicatedAllocation);
    10916 
    10917  // Make sure alignment requirements for specific buffer usages reported
    10918  // in Physical Device Properties are included in alignment reported by memory requirements.
    10919  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
    10920  {
    10921  VMA_ASSERT(vkMemReq.alignment %
    10922  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
    10923  }
    10924  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
    10925  {
    10926  VMA_ASSERT(vkMemReq.alignment %
    10927  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
    10928  }
    10929  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
    10930  {
    10931  VMA_ASSERT(vkMemReq.alignment %
    10932  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
    10933  }
    10934 
    10935  // 3. Allocate memory using allocator.
    10936  res = allocator->AllocateMemory(
    10937  vkMemReq,
    10938  requiresDedicatedAllocation,
    10939  prefersDedicatedAllocation,
    10940  *pBuffer, // dedicatedBuffer
    10941  VK_NULL_HANDLE, // dedicatedImage
    10942  *pAllocationCreateInfo,
    10943  VMA_SUBALLOCATION_TYPE_BUFFER,
    10944  pAllocation);
    10945 
    10946 #if VMA_RECORDING_ENABLED
    10947  if(allocator->GetRecorder() != VMA_NULL)
    10948  {
    10949  allocator->GetRecorder()->RecordCreateBuffer(
    10950  allocator->GetCurrentFrameIndex(),
    10951  *pBufferCreateInfo,
    10952  *pAllocationCreateInfo,
    10953  *pAllocation);
    10954  }
    10955 #endif
    10956 
    10957  if(res >= 0)
    10958  {
    10959  // 3. Bind buffer with memory.
    10960  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
    10961  if(res >= 0)
    10962  {
    10963  // All steps succeeded.
    10964  #if VMA_STATS_STRING_ENABLED
    10965  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
    10966  #endif
    10967  if(pAllocationInfo != VMA_NULL)
    10968  {
    10969  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    10970  }
    10971 
    10972  return VK_SUCCESS;
    10973  }
    10974  allocator->FreeMemory(*pAllocation);
    10975  *pAllocation = VK_NULL_HANDLE;
    10976  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10977  *pBuffer = VK_NULL_HANDLE;
    10978  return res;
    10979  }
    10980  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
    10981  *pBuffer = VK_NULL_HANDLE;
    10982  return res;
    10983  }
    10984  return res;
    10985 }
    10986 
    10987 void vmaDestroyBuffer(
    10988  VmaAllocator allocator,
    10989  VkBuffer buffer,
    10990  VmaAllocation allocation)
    10991 {
    10992  VMA_ASSERT(allocator);
    10993 
    10994  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    10995  {
    10996  return;
    10997  }
    10998 
    10999  VMA_DEBUG_LOG("vmaDestroyBuffer");
    11000 
    11001  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    11002 
    11003 #if VMA_RECORDING_ENABLED
    11004  if(allocator->GetRecorder() != VMA_NULL)
    11005  {
    11006  allocator->GetRecorder()->RecordDestroyBuffer(
    11007  allocator->GetCurrentFrameIndex(),
    11008  allocation);
    11009  }
    11010 #endif
    11011 
    11012  if(buffer != VK_NULL_HANDLE)
    11013  {
    11014  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
    11015  }
    11016 
    11017  if(allocation != VK_NULL_HANDLE)
    11018  {
    11019  allocator->FreeMemory(allocation);
    11020  }
    11021 }
    11022 
    11023 VkResult vmaCreateImage(
    11024  VmaAllocator allocator,
    11025  const VkImageCreateInfo* pImageCreateInfo,
    11026  const VmaAllocationCreateInfo* pAllocationCreateInfo,
    11027  VkImage* pImage,
    11028  VmaAllocation* pAllocation,
    11029  VmaAllocationInfo* pAllocationInfo)
    11030 {
    11031  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
    11032 
    11033  VMA_DEBUG_LOG("vmaCreateImage");
    11034 
    11035  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    11036 
    11037  *pImage = VK_NULL_HANDLE;
    11038  *pAllocation = VK_NULL_HANDLE;
    11039 
    11040  // 1. Create VkImage.
    11041  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
    11042  allocator->m_hDevice,
    11043  pImageCreateInfo,
    11044  allocator->GetAllocationCallbacks(),
    11045  pImage);
    11046  if(res >= 0)
    11047  {
    11048  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
    11049  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
    11050  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
    11051 
    11052  // 2. Allocate memory using allocator.
    11053  VkMemoryRequirements vkMemReq = {};
    11054  bool requiresDedicatedAllocation = false;
    11055  bool prefersDedicatedAllocation = false;
    11056  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
    11057  requiresDedicatedAllocation, prefersDedicatedAllocation);
    11058 
    11059  res = allocator->AllocateMemory(
    11060  vkMemReq,
    11061  requiresDedicatedAllocation,
    11062  prefersDedicatedAllocation,
    11063  VK_NULL_HANDLE, // dedicatedBuffer
    11064  *pImage, // dedicatedImage
    11065  *pAllocationCreateInfo,
    11066  suballocType,
    11067  pAllocation);
    11068 
    11069 #if VMA_RECORDING_ENABLED
    11070  if(allocator->GetRecorder() != VMA_NULL)
    11071  {
    11072  allocator->GetRecorder()->RecordCreateImage(
    11073  allocator->GetCurrentFrameIndex(),
    11074  *pImageCreateInfo,
    11075  *pAllocationCreateInfo,
    11076  *pAllocation);
    11077  }
    11078 #endif
    11079 
    11080  if(res >= 0)
    11081  {
    11082  // 3. Bind image with memory.
    11083  res = allocator->BindImageMemory(*pAllocation, *pImage);
    11084  if(res >= 0)
    11085  {
    11086  // All steps succeeded.
    11087  #if VMA_STATS_STRING_ENABLED
    11088  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
    11089  #endif
    11090  if(pAllocationInfo != VMA_NULL)
    11091  {
    11092  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
    11093  }
    11094 
    11095  return VK_SUCCESS;
    11096  }
    11097  allocator->FreeMemory(*pAllocation);
    11098  *pAllocation = VK_NULL_HANDLE;
    11099  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    11100  *pImage = VK_NULL_HANDLE;
    11101  return res;
    11102  }
    11103  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
    11104  *pImage = VK_NULL_HANDLE;
    11105  return res;
    11106  }
    11107  return res;
    11108 }
    11109 
    11110 void vmaDestroyImage(
    11111  VmaAllocator allocator,
    11112  VkImage image,
    11113  VmaAllocation allocation)
    11114 {
    11115  VMA_ASSERT(allocator);
    11116 
    11117  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
    11118  {
    11119  return;
    11120  }
    11121 
    11122  VMA_DEBUG_LOG("vmaDestroyImage");
    11123 
    11124  VMA_DEBUG_GLOBAL_MUTEX_LOCK
    11125 
    11126 #if VMA_RECORDING_ENABLED
    11127  if(allocator->GetRecorder() != VMA_NULL)
    11128  {
    11129  allocator->GetRecorder()->RecordDestroyImage(
    11130  allocator->GetCurrentFrameIndex(),
    11131  allocation);
    11132  }
    11133 #endif
    11134 
    11135  if(image != VK_NULL_HANDLE)
    11136  {
    11137  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
    11138  }
    11139  if(allocation != VK_NULL_HANDLE)
    11140  {
    11141  allocator->FreeMemory(allocation);
    11142  }
    11143 }
    11144 
    11145 #endif // #ifdef VMA_IMPLEMENTATION
    VmaVulkanFunctions::vkGetPhysicalDeviceProperties
    PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
    Definition: vk_mem_alloc.h:1345
    VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
    Set this flag if the allocation should have its own memory block.
    Definition: vk_mem_alloc.h:1658
    vmaUnmapMemory
    void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
    Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
    VmaAllocatorCreateInfo::physicalDevice
    VkPhysicalDevice physicalDevice
    Vulkan physical device.
    Definition: vk_mem_alloc.h:1414
    diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index a908c90..5b8fcc0 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -29,7 +29,7 @@ extern "C" { /** \mainpage Vulkan Memory Allocator -Version 2.1.0-alpha.3 (2018-06-14) +Version 2.1.0-alpha.4 (2018-08-22) Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved. \n License: MIT