fs: implement system heap

libraries/libstratosphere/include/stratosphere/fssystem.hpp

@@ -25,3 +25,4 @@
 #include <stratosphere/fssystem/fssystem_directory_savedata_filesystem.hpp>
 #include <stratosphere/fssystem/fssystem_romfs_file_system.hpp>
 #include <stratosphere/fssystem/buffers/fssystem_buffer_manager_utils.hpp>
+#include <stratosphere/fssystem/buffers/fssystem_file_system_buddy_heap.hpp>

libraries/libstratosphere/include/stratosphere/fssystem/buffers/fssystem_file_system_buddy_heap.hpp

@@ -0,0 +1,189 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+#include <stratosphere/fs/impl/fs_newable.hpp>
+
+namespace ams::fssystem {
+
+    class FileSystemBuddyHeap {
+        NON_COPYABLE(FileSystemBuddyHeap);
+        NON_MOVEABLE(FileSystemBuddyHeap);
+        public:
+            static constexpr size_t BufferAlignment = sizeof(void *);
+            static constexpr size_t BlockSizeMin    = 2 * sizeof(void *);
+            static constexpr s32 OrderUpperLimit    = BITSIZEOF(s32) - 1;
+        private:
+            class PageList;
+
+            struct PageEntry { PageEntry *next; };
+            static_assert(std::is_pod<PageEntry>::value);
+            static_assert(sizeof(PageEntry) <= BlockSizeMin);
+
+            class PageList : public ::ams::fs::impl::Newable {
+                NON_COPYABLE(PageList);
+                NON_MOVEABLE(PageList);
+                private:
+                    PageEntry *first_page_entry;
+                    PageEntry *last_page_entry;
+                    s32 entry_count;
+                public:
+                    constexpr  PageList() : first_page_entry(), last_page_entry(), entry_count() { /* ... */ }
+
+                    constexpr bool IsEmpty() const { return this->entry_count == 0; }
+                    constexpr s32  GetSize() const { return this->entry_count; }
+
+                    constexpr const PageEntry *GetFront() const { return this->first_page_entry; }
+                public:
+                    PageEntry *PopFront();
+                    void PushBack(PageEntry *page_entry);
+                    bool Remove(PageEntry *page_entry);
+            };
+        private:
+            size_t block_size;
+            s32    order_max;
+            uintptr_t heap_start;
+            size_t    heap_size;
+
+            PageList *free_lists;
+            size_t total_free_size;
+            PageList *external_free_lists;
+            std::unique_ptr<PageList[]> internal_free_lists;
+        public:
+            static constexpr s32 GetBlockCountFromOrder(s32 order) {
+                AMS_ASSERT(0 <= order);
+                AMS_ASSERT(order < OrderUpperLimit);
+                return (1 << order);
+            }
+
+            static constexpr size_t QueryWorkBufferSize(s32 order_max) {
+                AMS_ASSERT(0 < order_max && order_max < OrderUpperLimit);
+                return util::AlignUp(sizeof(PageList) * (order_max + 1) + alignof(PageList), alignof(u64));
+            }
+
+            static constexpr s32 QueryOrderMax(size_t size, size_t block_size) {
+                AMS_ASSERT(size >= block_size);
+                AMS_ASSERT(block_size >= BlockSizeMin);
+                AMS_ASSERT(util::IsPowerOfTwo(block_size));
+                const auto block_count = static_cast<s32>(util::AlignUp(size, block_size) / block_size);
+                for (auto order = 1; true; order++) {
+                    if (block_count <= GetBlockCountFromOrder(order)) {
+                        return order;
+                    }
+                }
+            }
+
+        public:
+            constexpr FileSystemBuddyHeap() : block_size(), order_max(), heap_start(), heap_size(), free_lists(), total_free_size(), external_free_lists(), internal_free_lists() { /* ... */ }
+
+            Result Initialize(uintptr_t address, size_t size, size_t block_size, s32 order_max);
+
+            Result Initialize(uintptr_t address, size_t size, size_t block_size) {
+                return this->Initialize(address, size, block_size, QueryOrderMax(size, block_size));
+            }
+
+            Result Initialize(uintptr_t address, size_t size, size_t block_size, s32 order_max, void *work, size_t work_size) {
+                AMS_ASSERT(work_size >= QueryWorkBufferSize(order_max));
+                const auto aligned_work = util::AlignUp(reinterpret_cast<uintptr_t>(work), alignof(PageList));
+                this->external_free_lists = reinterpret_cast<PageList *>(aligned_work);
+                return this->Initialize(address, size, block_size, order_max);
+            }
+
+            Result Initialize(uintptr_t address, size_t size, size_t block_size, void *work, size_t work_size) {
+                return this->Initialize(address, size, block_size, QueryOrderMax(size, block_size), work, work_size);
+            }
+
+            void Finalize();
+
+            void *AllocateByOrder(s32 order);
+            void Free(void *ptr, s32 order);
+
+            size_t GetTotalFreeSize() const;
+            size_t GetAllocatableSizeMax() const;
+            void Dump() const;
+
+            s32 GetOrderFromBytes(size_t size) const {
+                AMS_ASSERT(this->free_lists != nullptr);
+                return this->GetOrderFromBlockCount(this->GetBlockCountFromSize(size));
+            }
+
+            size_t GetBytesFromOrder(s32 order) const {
+                AMS_ASSERT(this->free_lists != nullptr);
+                AMS_ASSERT(0 <= order);
+                AMS_ASSERT(order < this->GetOrderMax());
+                return (this->GetBlockSize() << order);
+            }
+
+            s32 GetOrderMax() const {
+                AMS_ASSERT(this->free_lists != nullptr);
+                return this->order_max;
+            }
+
+            size_t GetBlockSize() const {
+                AMS_ASSERT(this->free_lists != nullptr);
+                return this->block_size;
+            }
+
+            s32 GetPageBlockCountMax() const {
+                AMS_ASSERT(this->free_lists != nullptr);
+                return 1 << this->GetOrderMax();
+            }
+
+            size_t GetPageSizeMax() const {
+                AMS_ASSERT(this->free_lists != nullptr);
+                return this->GetPageBlockCountMax() * this->GetBlockSize();
+            }
+        private:
+            void DivideBuddies(PageEntry *page_entry, s32 required_order, s32 chosen_order);
+            void JoinBuddies(PageEntry *page_entry, s32 order);
+            PageEntry *GetBuddy(PageEntry *page_entry, s32 order);
+            PageEntry *GetFreePageEntry(s32 order);
+
+            s32 GetOrderFromBlockCount(s32 block_count) const;
+
+            s32 GetBlockCountFromSize(size_t size) const {
+                const size_t bsize = this->GetBlockSize();
+                return static_cast<s32>(util::AlignUp(size, bsize) / bsize);
+            }
+
+            uintptr_t GetAddressFromPageEntry(const PageEntry &page_entry) const {
+                const uintptr_t address = reinterpret_cast<uintptr_t>(std::addressof(page_entry));
+                AMS_ASSERT(this->heap_start <= address);
+                AMS_ASSERT(address < this->heap_start + this->heap_size);
+                AMS_ASSERT(util::IsAligned(address - this->heap_start, this->GetBlockSize()));
+                return address;
+            }
+
+            PageEntry *GetPageEntryFromAddress(uintptr_t address) const {
+                AMS_ASSERT(this->heap_start <= address);
+                AMS_ASSERT(address < this->heap_start + this->heap_size);
+                return reinterpret_cast<PageEntry *>(this->heap_start + util::AlignDown(address - this->heap_start, this->GetBlockSize()));
+            }
+
+            s32 GetIndexFromPageEntry(const PageEntry &page_entry) const {
+                const uintptr_t address = reinterpret_cast<uintptr_t>(std::addressof(page_entry));
+                AMS_ASSERT(this->heap_start <= address);
+                AMS_ASSERT(address < this->heap_start + this->heap_size);
+                AMS_ASSERT(util::IsAligned(address - this->heap_start, this->GetBlockSize()));
+                return static_cast<s32>((address - this->heap_start) / this->GetBlockSize());
+            }
+
+            bool IsAlignedToOrder(const PageEntry *page_entry, s32 order) const {
+                return util::IsAligned(GetIndexFromPageEntry(*page_entry), GetBlockCountFromOrder(order));
+            }
+    };
+
+}

libraries/libstratosphere/source/fssystem/buffers/fssystem_file_system_buddy_heap.cpp

@@ -0,0 +1,360 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <stratosphere.hpp>
+
+namespace ams::fssystem {
+
+    FileSystemBuddyHeap::PageEntry *FileSystemBuddyHeap::PageList::PopFront() {
+        AMS_ASSERT(this->entry_count > 0);
+
+        /* Get the first entry. */
+        auto page_entry = this->first_page_entry;
+
+        /* Advance our list. */
+        this->first_page_entry = page_entry->next;
+        page_entry->next = nullptr;
+
+        /* Decrement our count. */
+        --this->entry_count;
+        AMS_ASSERT(this->entry_count >= 0);
+
+        /* If this was our last page, clear our last entry. */
+        if (this->entry_count == 0) {
+            this->last_page_entry = nullptr;
+        }
+
+        return page_entry;
+    }
+
+    void FileSystemBuddyHeap::PageList::PushBack(PageEntry *page_entry) {
+        AMS_ASSERT(page_entry != nullptr);
+
+        /* If we're empty, we want to set the first page entry. */
+        if (this->IsEmpty()) {
+            this->first_page_entry = page_entry;
+        } else {
+            /* We're not empty, so push the page to the back. */
+            AMS_ASSERT(this->last_page_entry != page_entry);
+            this->last_page_entry->next = page_entry;
+        }
+
+        /* Set our last page entry to be this one, and link it to the list. */
+        this->last_page_entry = page_entry;
+        this->last_page_entry->next = nullptr;
+
+        /* Increment our entry count. */
+        ++this->entry_count;
+        AMS_ASSERT(this->entry_count > 0);
+    }
+
+    bool FileSystemBuddyHeap::PageList::Remove(PageEntry *page_entry) {
+        AMS_ASSERT(page_entry != nullptr);
+
+        /* If we're empty, we can't remove the page list. */
+        if (this->IsEmpty()) {
+            return false;
+        }
+
+        /* We're going to loop over all pages to find this one, then unlink it. */
+        PageEntry *prev_entry = nullptr;
+        PageEntry *cur_entry  = this->first_page_entry;
+
+        while (true) {
+            /* Check if we found the page. */
+            if (cur_entry == page_entry) {
+                if (cur_entry == this->first_page_entry) {
+                    /* If it's the first page, we just set our first. */
+                    this->first_page_entry = cur_entry->next;
+                } else if (cur_entry == this->last_page_entry) {
+                    /* If it's the last page, we set our last. */
+                    this->last_page_entry = prev_entry;
+                    this->last_page_entry->next = nullptr;
+                } else {
+                    /* If it's in the middle, we just unlink. */
+                    prev_entry->next = cur_entry->next;
+                }
+
+                /* Unlink this entry's next. */
+                cur_entry->next = nullptr;
+
+                /* Update our entry count. */
+                --this->entry_count;
+                AMS_ASSERT(this->entry_count >= 0);
+
+                return true;
+            }
+
+            /* If we have no next page, we can't remove. */
+            if (cur_entry->next == nullptr) {
+                return false;
+            }
+
+            /* Advance to the next item in the list. */
+            prev_entry = cur_entry;
+            cur_entry  = cur_entry->next;
+        }
+    }
+
+    Result FileSystemBuddyHeap::Initialize(uintptr_t address, size_t size, size_t block_size, s32 order_max) {
+        /* Ensure our preconditions. */
+        AMS_ASSERT(this->free_lists == nullptr);
+        AMS_ASSERT(address != 0);
+        AMS_ASSERT(util::IsAligned(address, BufferAlignment));
+        AMS_ASSERT(block_size >= BlockSizeMin);
+        AMS_ASSERT(util::IsPowerOfTwo(block_size));
+        AMS_ASSERT(size >= block_size);
+        AMS_ASSERT(order_max > 0);
+        AMS_ASSERT(order_max < OrderUpperLimit);
+
+        /* Set up our basic member variables */
+        this->block_size = block_size;
+        this->order_max  = order_max;
+        this->heap_start = address;
+        this->heap_size  = (size / this->block_size) * this->block_size;
+
+        this->total_free_size = 0;
+
+        /* Determine page sizes. */
+        const auto max_page_size  = this->block_size << this->order_max;
+        const auto max_page_count = util::AlignUp(this->heap_size, max_page_size) / max_page_size;
+        AMS_ASSERT(max_page_count > 0);
+
+        /* Setup the free lists. */
+        if (this->external_free_lists != nullptr) {
+            AMS_ASSERT(this->internal_free_lists == nullptr);
+            this->free_lists = this->external_free_lists;
+        } else {
+            this->internal_free_lists.reset(new PageList[this->order_max + 1]);
+            this->free_lists = this->internal_free_lists.get();
+            R_UNLESS(this->free_lists != nullptr, fs::ResultAllocationFailureInFileSystemBuddyHeapA());
+        }
+
+        /* All but the last page region should go to the max order. */
+        for (size_t i = 0; i < max_page_count - 1; i++) {
+            auto page_entry = this->GetPageEntryFromAddress(this->heap_start + i * max_page_size);
+            this->free_lists[this->order_max].PushBack(page_entry);
+        }
+        this->total_free_size += this->free_lists[this->order_max].GetSize() * this->GetBytesFromOrder(this->order_max);
+
+        /* Allocate remaining space to smaller orders as possible. */
+        {
+            auto remaining   = this->heap_size  - (max_page_count - 1) * max_page_size;
+            auto cur_address = this->heap_start + (max_page_count - 1) * max_page_size;
+            AMS_ASSERT(util::IsAligned(remaining, this->block_size));
+
+            do {
+                /* Determine what order we can use. */
+                auto order = GetOrderFromBytes(remaining + 1);
+                if (order < 0) {
+                    AMS_ASSERT(GetOrderFromBytes(remaining) == this->order_max);
+                    order = this->order_max + 1;
+                }
+                AMS_ASSERT(0 < order);
+                AMS_ASSERT(order <= this->order_max + 1);
+
+                /* Add to the correct free list. */
+                this->free_lists[order - 1].PushBack(GetPageEntryFromAddress(cur_address));
+                this->total_free_size += GetBytesFromOrder(order - 1);
+
+                /* Move on to the next order. */
+                const auto page_size = GetBytesFromOrder(order - 1);
+                cur_address += page_size;
+                remaining   -= page_size;
+            } while (this->block_size <= remaining);
+        }
+
+        return ResultSuccess();
+    }
+
+    void FileSystemBuddyHeap::Finalize() {
+        AMS_ASSERT(this->free_lists != nullptr);
+        this->free_lists = nullptr;
+        this->external_free_lists = nullptr;
+        this->internal_free_lists.reset();
+    }
+
+    void *FileSystemBuddyHeap::AllocateByOrder(s32 order) {
+        AMS_ASSERT(this->free_lists != nullptr);
+        AMS_ASSERT(order >= 0);
+        AMS_ASSERT(order <= this->GetOrderMax());
+
+        /* Get the page entry. */
+        if (const auto page_entry = this->GetFreePageEntry(order); page_entry != nullptr) {
+            /* Ensure we're allocating an unlinked page. */
+            AMS_ASSERT(page_entry->next == nullptr);
+
+            /* Return the address for this entry. */
+            return reinterpret_cast<void *>(this->GetAddressFromPageEntry(*page_entry));
+        } else {
+            return nullptr;
+        }
+    }
+
+    void FileSystemBuddyHeap::Free(void *ptr, s32 order) {
+        AMS_ASSERT(this->free_lists != nullptr);
+        AMS_ASSERT(order >= 0);
+        AMS_ASSERT(order <= this->GetOrderMax());
+
+        /* Allow free(nullptr) */
+        if (ptr == nullptr) {
+            return;
+        }
+
+        /* Ensure the pointer is block aligned. */
+        AMS_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(ptr) - this->heap_start, this->GetBlockSize()));
+
+        /* Get the page entry. */
+        auto page_entry = this->GetPageEntryFromAddress(reinterpret_cast<uintptr_t>(ptr));
+        AMS_ASSERT(this->IsAlignedToOrder(page_entry, order));
+
+        /* Reinsert into the free lists. */
+        this->JoinBuddies(page_entry, order);
+    }
+
+    size_t FileSystemBuddyHeap::GetTotalFreeSize() const {
+        AMS_ASSERT(this->free_lists != nullptr);
+        return this->total_free_size;
+    }
+
+    size_t FileSystemBuddyHeap::GetAllocatableSizeMax() const {
+        AMS_ASSERT(this->free_lists != nullptr);
+
+        /* The maximum allocatable size is a chunk from the biggest non-empty order. */
+        for (s32 order = this->GetOrderMax(); order >= 0; --order) {
+            if (!this->free_lists[order].IsEmpty()) {
+                return this->GetBytesFromOrder(order);
+            }
+        }
+
+        /* If all orders are empty, then we can't allocate anything. */
+        return 0;
+    }
+
+    void FileSystemBuddyHeap::Dump() const {
+        AMS_ASSERT(this->free_lists != nullptr);
+        /* TODO: Support logging metrics. */
+    }
+
+    void FileSystemBuddyHeap::DivideBuddies(PageEntry *page_entry, s32 required_order, s32 chosen_order) {
+        AMS_ASSERT(page_entry != nullptr);
+        AMS_ASSERT(required_order >= 0);
+        AMS_ASSERT(chosen_order >= required_order);
+        AMS_ASSERT(chosen_order <= this->GetOrderMax());
+
+        /* Start at the end of the entry. */
+        auto address = this->GetAddressFromPageEntry(*page_entry) + this->GetBytesFromOrder(chosen_order);
+
+        for (auto order = chosen_order; order > required_order; --order) {
+            /* For each order, subtract that order's size from the address to get the start of a new block. */
+            address -= this->GetBytesFromOrder(order - 1);
+            auto divided_entry = this->GetPageEntryFromAddress(address);
+
+            /* Push back to the list. */
+            this->free_lists[order - 1].PushBack(divided_entry);
+            this->total_free_size += this->GetBytesFromOrder(order - 1);
+        }
+    }
+
+    void FileSystemBuddyHeap::JoinBuddies(PageEntry *page_entry, s32 order) {
+        AMS_ASSERT(page_entry != nullptr);
+        AMS_ASSERT(order >= 0);
+        AMS_ASSERT(order <= this->GetOrderMax());
+
+        auto cur_entry = page_entry;
+        auto cur_order = order;
+
+        while (cur_order < this->GetOrderMax()) {
+            /* Get the buddy page. */
+            const auto buddy_entry = this->GetBuddy(cur_entry, cur_order);
+
+            /* Check whether the buddy is in the relevant free list. */
+            if (buddy_entry != nullptr && this->free_lists[cur_order].Remove(buddy_entry)) {
+                this->total_free_size -= GetBytesFromOrder(cur_order);
+
+                /* Ensure we coalesce with the correct buddy when page is aligned */
+                if (!this->IsAlignedToOrder(cur_entry, cur_order + 1)) {
+                    cur_entry = buddy_entry;
+                }
+
+                ++cur_order;
+            } else {
+                /* Buddy isn't in the free list, so we can't coalesce. */
+                break;
+            }
+        }
+
+        /* Insert the coalesced entry into the free list. */
+        this->free_lists[cur_order].PushBack(cur_entry);
+        this->total_free_size += this->GetBytesFromOrder(cur_order);
+    }
+
+    FileSystemBuddyHeap::PageEntry *FileSystemBuddyHeap::GetBuddy(PageEntry *page_entry, s32 order) {
+        AMS_ASSERT(page_entry != nullptr);
+        AMS_ASSERT(order >= 0);
+        AMS_ASSERT(order <= this->GetOrderMax());
+
+        const auto address = this->GetAddressFromPageEntry(*page_entry);
+        const auto offset  = this->GetBlockCountFromOrder(order) * this->GetBlockSize();
+
+        if (this->IsAlignedToOrder(page_entry, order + 1)) {
+            /* If the page entry is aligned to the next order, return the buddy block to the right of the current entry. */
+            return (address + offset < this->heap_start + this->heap_size) ? GetPageEntryFromAddress(address + offset) : nullptr;
+        } else {
+            /* If the page entry isn't aligned, return the buddy block to the left of the current entry. */
+            return (this->heap_start <= address - offset) ? GetPageEntryFromAddress(address - offset) : nullptr;
+        }
+    }
+
+    FileSystemBuddyHeap::PageEntry *FileSystemBuddyHeap::GetFreePageEntry(s32 order) {
+        AMS_ASSERT(order >= 0);
+        AMS_ASSERT(order <= this->GetOrderMax());
+
+        /* Try orders from low to high until we find a free page entry. */
+        for (auto cur_order = order; cur_order <= this->GetOrderMax(); cur_order++) {
+            if (auto &free_list = this->free_lists[cur_order]; !free_list.IsEmpty()) {
+                /* The current list isn't empty, so grab an entry from it. */
+                PageEntry *page_entry = free_list.PopFront();
+                AMS_ASSERT(page_entry != nullptr);
+
+                /* Update size bookkeeping. */
+                this->total_free_size -= GetBytesFromOrder(cur_order);
+
+                /* If we allocated more memory than needed, free the unneeded portion. */
+                this->DivideBuddies(page_entry, order, cur_order);
+                AMS_ASSERT(page_entry->next == nullptr);
+
+                /* Return the newly-divided entry. */
+                return page_entry;
+            }
+        }
+
+        /* We failed to find a free page. */
+        return nullptr;
+    }
+
+    s32 FileSystemBuddyHeap::GetOrderFromBlockCount(s32 block_count) const {
+        AMS_ASSERT(block_count >= 0);
+
+        /* Return the first order with a big enough block count. */
+        for (s32 order = 0; order <= this->GetOrderMax(); ++order) {
+            if (block_count <= this->GetBlockCountFromOrder(order)) {
+                return order;
+            }
+        }
+        return -1;
+    }
+
+}

libraries/libvapours/include/vapours/results/fs_results.hpp

@@ -76,8 +76,8 @@ namespace ams::fs {
         R_DEFINE_ERROR_RESULT(AllocationFailureInRomFsFileSystemB,            3248);
         R_DEFINE_ERROR_RESULT(AllocationFailureInRomFsFileSystemC,            3249);
         R_DEFINE_ERROR_RESULT(AllocationFailureInPartitionFileSystemCreatorA, 3280);
+        R_DEFINE_ERROR_RESULT(AllocationFailureInFileSystemBuddyHeapA,        3294);
         R_DEFINE_ERROR_RESULT(AllocationFailureInDirectorySaveDataFileSystem, 3321);
-
         R_DEFINE_ERROR_RESULT(AllocationFailureInPartitionFileSystemA,        3347);
         R_DEFINE_ERROR_RESULT(AllocationFailureInPartitionFileSystemB,        3348);
         R_DEFINE_ERROR_RESULT(AllocationFailureInPartitionFileSystemC,        3349);