From f6db18473f5036c02032c338775bba8294de35bd Mon Sep 17 00:00:00 2001 From: Michael Mueller Date: Mon, 27 Mar 2023 16:29:39 +0200 Subject: [PATCH] Adapted MxTasking memory manager to Genode. --- src/mx/memory/dynamic_size_allocator.cpp | 2 +- src/mx/memory/fixed_size_allocator.h | 21 +++++-- src/mx/memory/global_heap.h | 72 ++++++++++++++++++++++-- src/mx/memory/task_allocator_interface.h | 2 +- src/mx/util/vector.h | 2 +- 5 files changed, 86 insertions(+), 13 deletions(-) diff --git a/src/mx/memory/dynamic_size_allocator.cpp b/src/mx/memory/dynamic_size_allocator.cpp index 03ac0ce..7f46ea6 100644 --- a/src/mx/memory/dynamic_size_allocator.cpp +++ b/src/mx/memory/dynamic_size_allocator.cpp @@ -35,7 +35,7 @@ AllocationBlock::~AllocationBlock() { if (this->_allocated_block != nullptr) { - GlobalHeap::free(this->_allocated_block, this->_size); + GlobalHeap::free(this->_allocated_block, this->_size, this->_numa_node_id); } } diff --git a/src/mx/memory/fixed_size_allocator.h b/src/mx/memory/fixed_size_allocator.h index b412bee..e4cfff1 100644 --- a/src/mx/memory/fixed_size_allocator.h +++ b/src/mx/memory/fixed_size_allocator.h @@ -16,6 +16,7 @@ #include #include #include +#include namespace mx::memory::fixed { /** @@ -45,17 +46,18 @@ class Chunk { public: Chunk() noexcept = default; - explicit Chunk(void *memory) noexcept : _memory(memory) {} + explicit Chunk(void *memory, bool is_allocated) noexcept : _memory(memory), _is_allocated(is_allocated) {} ~Chunk() noexcept = default; static constexpr auto size() { return 4096 * 4096; /* 16mb */ } explicit operator void *() const noexcept { return _memory; } explicit operator std::uintptr_t() const noexcept { return reinterpret_cast(_memory); } - explicit operator bool() const noexcept { return _memory != nullptr; } + explicit operator bool() const noexcept { return _memory != nullptr && _is_allocated; } private: void *_memory{nullptr}; + bool _is_allocated{true}; }; /** @@ -77,16 +79,20 @@ public: ~ProcessorHeap() noexcept { + if (_numa_node_id > mx::system::topology::max_node_id()) + return; + for (const auto allocated_chunk : _allocated_chunks) { - GlobalHeap::free(static_cast(allocated_chunk), Chunk::size()); + if (static_cast(allocated_chunk)) + GlobalHeap::free(static_cast(allocated_chunk), Chunk::size(), _numa_node_id); } for (const auto free_chunk : _free_chunk_buffer) { if (static_cast(free_chunk)) { - GlobalHeap::free(static_cast(free_chunk), Chunk::size()); + GlobalHeap::free(static_cast(free_chunk), Chunk::size(), _numa_node_id); } } } @@ -178,7 +184,7 @@ private: auto heap_memory_address = reinterpret_cast(heap_memory); for (auto i = 0U; i < _free_chunk_buffer.size(); ++i) { - _free_chunk_buffer[i] = Chunk(reinterpret_cast(heap_memory_address + (i * Chunk::size()))); + _free_chunk_buffer[i] = Chunk(reinterpret_cast(heap_memory_address + (i * Chunk::size())), i==0); } _next_free_chunk.store(0U); @@ -238,7 +244,9 @@ public: void fill_buffer() { auto chunk = _processor_heap->allocate(); + const auto chunk_address = static_cast(chunk); + //assert((chunk_address != 0) && "Failed to allocate chunk from processor heap"); constexpr auto object_size = S; constexpr auto count_objects = std::uint64_t{Chunk::size() / object_size}; @@ -279,7 +287,8 @@ template class Allocator final : public TaskAllocatorInterface public: explicit Allocator(const util::core_set &core_set) { - for (auto node_id = std::uint8_t(0U); node_id < config::max_numa_nodes(); ++node_id) + //assert((system::topology::max_node_id() <= config::max_numa_nodes()) && "More NUMA nodes detected than supported. Increase value for max_numa_nodes() in mx/memory/config.h."); + for (auto node_id = std::uint8_t(0U); node_id <= system::topology::max_node_id(); ++node_id) { if (core_set.has_core_of_numa_node(node_id)) { diff --git a/src/mx/memory/global_heap.h b/src/mx/memory/global_heap.h index 5f3a939..9a0cc93 100644 --- a/src/mx/memory/global_heap.h +++ b/src/mx/memory/global_heap.h @@ -2,15 +2,76 @@ #include "alignment_helper.h" #include #include -#include +#include +#include +#include +#include +#include +#include + +#include namespace mx::memory { /** * The global heap represents the heap, provided by the OS. */ class GlobalHeap { +private: + alignas(64) std::array heaps{nullptr}; + public: + + GlobalHeap() + { + for (unsigned numa_id = 0; numa_id <= system::topology::max_node_id(); numa_id++) { + Topology::Numa_region const &node = system::Environment::node(numa_id); + heaps[numa_id] = new Genode::Regional_heap(system::Environment::ram(), system::Environment::rm(), const_cast(node)); + } + } + + ~GlobalHeap() + { + for (auto heap : heaps) + delete heap; + } + + inline void *local_allocate(const std::uint8_t numa_node_id, const std::size_t size) + { + return heaps[numa_node_id]->alloc(size); + } + + inline void local_free(void *ptr, const std::size_t size, std::uint8_t node_id) + { + bool success = false; + std::uint8_t next_try = 0; + + do { + try + { + heaps[node_id]->free(ptr, size); + success = true; + } + catch (Genode::Region_map::Invalid_dataspace) + { + success = false; + node_id = (node_id == next_try) ? (next_try + 1) % (mx::system::topology::max_node_id()+1) : next_try; + next_try = (next_try + 1) % (mx::system::topology::max_node_id()+1); + } + } while (next_try != 0); + + if (!success) + std::free(ptr); + } + + static GlobalHeap &myself() + { + static GlobalHeap gheap; + return gheap; + } + + + /** * Allocates the given size on the given NUMA node. * @@ -20,7 +81,7 @@ public: */ static void *allocate(const std::uint8_t numa_node_id, const std::size_t size) { - return numa_alloc_onnode(size, numa_node_id); + return myself().local_allocate(numa_node_id, size); } /** @@ -32,15 +93,18 @@ public: */ static void *allocate_cache_line_aligned(const std::size_t size) { - return std::aligned_alloc(64U, alignment_helper::next_multiple(size, 64UL)); + void *ptr = nullptr; + posix_memalign(&ptr, 64, alignment_helper::next_multiple(size, 64UL)); + return ptr; } + /** * Frees the given memory. * * @param memory Pointer to memory. * @param size Size of the allocated object. */ - static void free(void *memory, const std::size_t size) { numa_free(memory, size); } + static void free(void *memory, const std::size_t size, std::uint8_t node_id) { myself().local_free(memory, size, node_id); } }; } // namespace mx::memory \ No newline at end of file diff --git a/src/mx/memory/task_allocator_interface.h b/src/mx/memory/task_allocator_interface.h index e92a910..d44fd07 100644 --- a/src/mx/memory/task_allocator_interface.h +++ b/src/mx/memory/task_allocator_interface.h @@ -41,7 +41,7 @@ public: /** * @return Allocated memory using systems malloc (but aligned). */ - [[nodiscard]] void *allocate(const std::uint16_t /*core_id*/) override { return std::aligned_alloc(64U, S); } + [[nodiscard]] void *allocate(const std::uint16_t /*core_id*/) override { return std::malloc(S); } /** * Frees the given memory using systems free. diff --git a/src/mx/util/vector.h b/src/mx/util/vector.h index 9a403ed..0485ea5 100644 --- a/src/mx/util/vector.h +++ b/src/mx/util/vector.h @@ -168,7 +168,7 @@ private: static void release(pointer_type data, const std::size_t capacity) noexcept { const auto size = sizeof(value_type) * capacity; - memory::GlobalHeap::free(static_cast(data), size); + memory::GlobalHeap::free(static_cast(data), size, _numa_node_id); } }; } // namespace mx::util \ No newline at end of file