From 12cd9dc23420e80526d5f298f96421fe2db0e064 Mon Sep 17 00:00:00 2001 From: Yuekai Jia Date: Sat, 29 Jul 2023 00:51:58 +0800 Subject: [PATCH] alloc: add TLSF allocator as the default --- Cargo.lock | 44 ++++++++++++++ apps/c/memtest/expect_trace.out | 1 + apps/memtest/expect_trace.out | 1 + apps/memtest/src/main.rs | 4 +- crates/allocator/Cargo.toml | 18 ++++-- crates/allocator/benches/collections.rs | 7 ++- crates/allocator/src/lib.rs | 18 +++++- crates/allocator/src/tlsf.rs | 81 +++++++++++++++++++++++++ crates/allocator/tests/allocator.rs | 15 ++++- modules/axalloc/Cargo.toml | 8 ++- modules/axalloc/src/lib.rs | 31 ++++++++-- ulib/axlibc/Cargo.toml | 3 + ulib/axstd/Cargo.toml | 3 + 13 files changed, 215 insertions(+), 19 deletions(-) create mode 100644 crates/allocator/src/tlsf.rs diff --git a/Cargo.lock b/Cargo.lock index a9b4471b1c..dfec50b078 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,7 @@ dependencies = [ "buddy_system_allocator", "criterion", "rand", + "rlsf", "slab_allocator", ] @@ -474,6 +475,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bindgen" version = "0.66.1" @@ -709,6 +716,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "const-default" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" + [[package]] name = "core-foundation-sys" version = "0.8.4" @@ -1590,6 +1603,18 @@ dependencies = [ "embedded-hal", ] +[[package]] +name = "rlsf" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222fb240c3286247ecdee6fa5341e7cdad0ffdf8e7e401d9937f2d58482a20bf" +dependencies = [ + "cfg-if", + "const-default", + "libc", + "svgbobdoc", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1775,6 +1800,19 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "svgbobdoc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2c04b93fc15d79b39c63218f15e3fdffaa4c227830686e3b7c5f41244eb3e50" +dependencies = [ + "base64", + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-width", +] + [[package]] name = "syn" version = "1.0.109" @@ -1917,6 +1955,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "version_check" version = "0.9.4" diff --git a/apps/c/memtest/expect_trace.out b/apps/c/memtest/expect_trace.out index aeebb3fae9..cdea6b34db 100644 --- a/apps/c/memtest/expect_trace.out +++ b/apps/c/memtest/expect_trace.out @@ -12,6 +12,7 @@ Found physcial memory regions: .bss (READ | WRITE | RESERVED) free memory (READ | WRITE | FREE) Initialize global memory allocator... + use TLSF allocator. initialize global allocator at: \[0x[0-9a-f]\+, 0x[0-9a-f]\+) Initialize kernel page table... Initialize platform devices... diff --git a/apps/memtest/expect_trace.out b/apps/memtest/expect_trace.out index f70c388dde..57fe228a34 100644 --- a/apps/memtest/expect_trace.out +++ b/apps/memtest/expect_trace.out @@ -12,6 +12,7 @@ Found physcial memory regions: .bss (READ | WRITE | RESERVED) free memory (READ | WRITE | FREE) Initialize global memory allocator... + use TLSF allocator. initialize global allocator at: \[0x[0-9a-f]\+, 0x[0-9a-f]\+) Initialize kernel page table... Initialize platform devices... diff --git a/apps/memtest/src/main.rs b/apps/memtest/src/main.rs index 1d3a081c25..e23e95e723 100644 --- a/apps/memtest/src/main.rs +++ b/apps/memtest/src/main.rs @@ -10,7 +10,7 @@ use std::collections::BTreeMap; use std::vec::Vec; fn test_vec(rng: &mut impl RngCore) { - const N: usize = 1_000_000; + const N: usize = 3_000_000; let mut v = Vec::with_capacity(N); for _ in 0..N { v.push(rng.next_u32()); @@ -23,7 +23,7 @@ fn test_vec(rng: &mut impl RngCore) { } fn test_btree_map(rng: &mut impl RngCore) { - const N: usize = 10_000; + const N: usize = 50_000; let mut m = BTreeMap::new(); for _ in 0..N { let value = rng.next_u32(); diff --git a/crates/allocator/Cargo.toml b/crates/allocator/Cargo.toml index 1eb6b17990..b8770f124d 100644 --- a/crates/allocator/Cargo.toml +++ b/crates/allocator/Cargo.toml @@ -10,15 +10,25 @@ repository = "https://github.com/rcore-os/arceos/tree/main/crates/allocator" documentation = "https://rcore-os.github.io/arceos/allocator/index.html" [features] +default = [] +full = ["bitmap", "tlsf", "slab", "buddy", "allocator_api"] + +bitmap = ["dep:bitmap-allocator"] + +tlsf = ["dep:rlsf"] +slab = ["dep:slab_allocator"] +buddy = ["dep:buddy_system_allocator"] + allocator_api = [] [dependencies] -buddy_system_allocator = { version = "0.9", default-features = false } -bitmap-allocator = { git = "https://github.com/rcore-os/bitmap-allocator.git", rev = "88e871a" } -slab_allocator = { path = "../slab_allocator" } +buddy_system_allocator = { version = "0.9", default-features = false, optional = true } +slab_allocator = { path = "../slab_allocator", optional = true } +rlsf = { version = "0.2", optional = true } +bitmap-allocator = { git = "https://github.com/rcore-os/bitmap-allocator.git", rev = "88e871a", optional = true } [dev-dependencies] -allocator = { path = ".", features = ["allocator_api"] } +allocator = { path = ".", features = ["full"] } rand = { version = "0.8", features = ["small_rng"] } criterion = { version = "0.5", features = ["html_reports"] } diff --git a/crates/allocator/benches/collections.rs b/crates/allocator/benches/collections.rs index 1c3c77a003..c31fd84b44 100644 --- a/crates/allocator/benches/collections.rs +++ b/crates/allocator/benches/collections.rs @@ -7,7 +7,7 @@ use std::alloc::Allocator; use std::collections::BTreeMap; use std::io::Write; -use allocator::{AllocatorRc, BuddyByteAllocator, SlabByteAllocator}; +use allocator::{AllocatorRc, BuddyByteAllocator, SlabByteAllocator, TlsfByteAllocator}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rand::{rngs::SmallRng, seq::SliceRandom, RngCore, SeedableRng}; @@ -80,6 +80,11 @@ fn bench(c: &mut Criterion, alloc_name: &str, alloc: impl Allocator + Clone) { fn criterion_benchmark(c: &mut Criterion) { let mut pool = MemoryPool::new(POOL_SIZE); bench(c, "system", std::alloc::System); + bench( + c, + "tlsf", + AllocatorRc::new(TlsfByteAllocator::new(), pool.as_slice()), + ); bench( c, "slab", diff --git a/crates/allocator/src/lib.rs b/crates/allocator/src/lib.rs index 7e30fe0cf6..f8a62e6b02 100644 --- a/crates/allocator/src/lib.rs +++ b/crates/allocator/src/lib.rs @@ -13,14 +13,26 @@ #![feature(result_option_inspect)] #![cfg_attr(feature = "allocator_api", feature(allocator_api))] +#[cfg(feature = "bitmap")] mod bitmap; -mod buddy; -mod slab; - +#[cfg(feature = "bitmap")] pub use bitmap::BitmapPageAllocator; + +#[cfg(feature = "buddy")] +mod buddy; +#[cfg(feature = "buddy")] pub use buddy::BuddyByteAllocator; + +#[cfg(feature = "slab")] +mod slab; +#[cfg(feature = "slab")] pub use slab::SlabByteAllocator; +#[cfg(feature = "tlsf")] +mod tlsf; +#[cfg(feature = "tlsf")] +pub use tlsf::TlsfByteAllocator; + use core::alloc::Layout; use core::num::NonZeroUsize; diff --git a/crates/allocator/src/tlsf.rs b/crates/allocator/src/tlsf.rs new file mode 100644 index 0000000000..72448a113d --- /dev/null +++ b/crates/allocator/src/tlsf.rs @@ -0,0 +1,81 @@ +//! The TLSF (Two-Level Segregated Fit) dynamic memory allocation algorithm. +//! +//! This module wraps the implementation provided by the [rlsf] crate. + +use super::{AllocError, AllocResult, BaseAllocator, ByteAllocator}; +use core::alloc::Layout; +use core::num::NonZeroUsize; +use core::ptr::NonNull; +use rlsf::Tlsf; + +/// A TLSF (Two-Level Segregated Fit) memory allocator. +/// +/// It's just a wrapper structure of [`rlsf::Tlsf`], with `FLLEN` and `SLLEN` +/// fixed to 28 and 32. +pub struct TlsfByteAllocator { + inner: Tlsf<'static, u32, u32, 28, 32>, // max pool size: 32 * 2^28 = 8G + total_bytes: usize, + used_bytes: usize, +} + +impl TlsfByteAllocator { + /// Creates a new empty [`TlsfByteAllocator`]. + pub const fn new() -> Self { + Self { + inner: Tlsf::new(), + total_bytes: 0, + used_bytes: 0, + } + } +} + +impl BaseAllocator for TlsfByteAllocator { + fn init(&mut self, start: usize, size: usize) { + unsafe { + let pool = core::slice::from_raw_parts_mut(start as *mut u8, size); + self.inner + .insert_free_block_ptr(NonNull::new(pool).unwrap()) + .unwrap(); + } + self.total_bytes = size; + } + + fn add_memory(&mut self, start: usize, size: usize) -> AllocResult { + unsafe { + let pool = core::slice::from_raw_parts_mut(start as *mut u8, size); + self.inner + .insert_free_block_ptr(NonNull::new(pool).unwrap()) + .ok_or(AllocError::InvalidParam)?; + } + self.total_bytes += size; + Ok(()) + } +} + +impl ByteAllocator for TlsfByteAllocator { + fn alloc(&mut self, layout: Layout) -> AllocResult { + let ptr = self.inner.allocate(layout).ok_or(AllocError::NoMemory)?; + self.used_bytes += layout.size(); + Ok(ptr.addr()) + } + + fn dealloc(&mut self, pos: NonZeroUsize, layout: Layout) { + unsafe { + self.inner + .deallocate(NonNull::new_unchecked(pos.get() as _), layout.align()) + } + self.used_bytes -= layout.size(); + } + + fn total_bytes(&self) -> usize { + self.total_bytes + } + + fn used_bytes(&self) -> usize { + self.used_bytes + } + + fn available_bytes(&self) -> usize { + self.total_bytes - self.used_bytes + } +} diff --git a/crates/allocator/tests/allocator.rs b/crates/allocator/tests/allocator.rs index f689f3dba4..d1f8ce66b4 100644 --- a/crates/allocator/tests/allocator.rs +++ b/crates/allocator/tests/allocator.rs @@ -6,7 +6,7 @@ use std::alloc::{Allocator, Layout}; use std::collections::BTreeMap; use std::io::Write; -use allocator::{AllocatorRc, BuddyByteAllocator, SlabByteAllocator}; +use allocator::{AllocatorRc, BuddyByteAllocator, SlabByteAllocator, TlsfByteAllocator}; use rand::{prelude::SliceRandom, Rng}; const POOL_SIZE: usize = 1024 * 1024 * 128; @@ -72,7 +72,6 @@ pub fn test_alignment(n: usize, alloc: &(impl Allocator + Clone)) { let align = 1 << rng.gen_range(0..8); let layout = Layout::from_size_align(size, align).unwrap(); let ptr = alloc.allocate(layout).unwrap(); - // println!("{:?} {:#x}", layout, ptr.addr()); blocks.push((ptr, layout)); } else { // delete a block @@ -131,3 +130,15 @@ fn slab_alloc() { test_btree_map(50_000, &alloc); }) } + +#[test] +fn tlsf_alloc() { + run_test(|pool| { + let alloc = AllocatorRc::new(TlsfByteAllocator::new(), pool); + test_alignment(50, &alloc); + test_vec(3_000_000, &alloc); + test_vec2(30_000, 64, &alloc); + test_vec2(7_500, 520, &alloc); + test_btree_map(50_000, &alloc); + }) +} diff --git a/modules/axalloc/Cargo.toml b/modules/axalloc/Cargo.toml index 83ef1527ff..40eedf404f 100644 --- a/modules/axalloc/Cargo.toml +++ b/modules/axalloc/Cargo.toml @@ -9,10 +9,16 @@ homepage = "https://github.com/rcore-os/arceos" repository = "https://github.com/rcore-os/arceos/tree/main/modules/axalloc" documentation = "https://rcore-os.github.io/arceos/axalloc/index.html" +[features] +default = ["tlsf"] +tlsf = ["allocator/tlsf"] +slab = ["allocator/slab"] +buddy = ["allocator/buddy"] + [dependencies] log = "0.4" cfg-if = "1.0" spinlock = { path = "../../crates/spinlock" } memory_addr = { path = "../../crates/memory_addr" } -allocator = { path = "../../crates/allocator"} +allocator = { path = "../../crates/allocator", features = ["bitmap"] } axerrno = { path = "../../crates/axerrno" } diff --git a/modules/axalloc/src/lib.rs b/modules/axalloc/src/lib.rs index 9ddcb818fb..64ea1de243 100644 --- a/modules/axalloc/src/lib.rs +++ b/modules/axalloc/src/lib.rs @@ -13,8 +13,7 @@ extern crate alloc; mod page; -use allocator::{AllocResult, BaseAllocator, ByteAllocator, PageAllocator}; -use allocator::{BitmapPageAllocator, SlabByteAllocator}; +use allocator::{AllocResult, BaseAllocator, BitmapPageAllocator, ByteAllocator, PageAllocator}; use core::alloc::{GlobalAlloc, Layout}; use core::num::NonZeroUsize; use spinlock::SpinNoIrq; @@ -24,6 +23,16 @@ const MIN_HEAP_SIZE: usize = 0x8000; // 32 K pub use page::GlobalPage; +cfg_if::cfg_if! { + if #[cfg(feature = "slab")] { + use allocator::SlabByteAllocator as DefaultByteAllocator; + } else if #[cfg(feature = "buddy")] { + use allocator::BuddyByteAllocator as DefaultByteAllocator; + } else if #[cfg(feature = "tlsf")] { + use allocator::TlsfByteAllocator as DefaultByteAllocator; + } +} + /// The global allocator used by ArceOS. /// /// It combines a [`ByteAllocator`] and a [`PageAllocator`] into a simple @@ -31,10 +40,12 @@ pub use page::GlobalPage; /// there is no memory, asks the page allocator for more memory and adds it to /// the byte allocator. /// -/// Currently, [`SlabByteAllocator`] is used as the byte allocator, while +/// Currently, [`TlsfByteAllocator`] is used as the byte allocator, while /// [`BitmapPageAllocator`] is used as the page allocator. +/// +/// [`TlsfByteAllocator`]: allocator::TlsfByteAllocator pub struct GlobalAllocator { - balloc: SpinNoIrq, + balloc: SpinNoIrq, palloc: SpinNoIrq>, } @@ -42,14 +53,22 @@ impl GlobalAllocator { /// Creates an empty [`GlobalAllocator`]. pub const fn new() -> Self { Self { - balloc: SpinNoIrq::new(SlabByteAllocator::new()), + balloc: SpinNoIrq::new(DefaultByteAllocator::new()), palloc: SpinNoIrq::new(BitmapPageAllocator::new()), } } /// Returns the name of the allocator. pub const fn name(&self) -> &'static str { - "slab" + cfg_if::cfg_if! { + if #[cfg(feature = "slab")] { + "slab" + } else if #[cfg(feature = "buddy")] { + "buddy" + } else if #[cfg(feature = "tlsf")] { + "TLSF" + } + } } /// Initializes the allocator with the given region. diff --git a/ulib/axlibc/Cargo.toml b/ulib/axlibc/Cargo.toml index bad87e7dab..4cb5d00a70 100644 --- a/ulib/axlibc/Cargo.toml +++ b/ulib/axlibc/Cargo.toml @@ -27,6 +27,9 @@ fp_simd = ["axstd/fp_simd"] # Memory alloc = ["axstd/alloc", "dep:axalloc"] +alloc_tlsf = ["axalloc/tlsf"] +alloc_slab = ["axalloc/slab"] +alloc_buddy = ["axalloc/buddy"] paging = ["axstd/paging"] # Interrupts diff --git a/ulib/axstd/Cargo.toml b/ulib/axstd/Cargo.toml index 08945a31ad..1c6b0371ee 100644 --- a/ulib/axstd/Cargo.toml +++ b/ulib/axstd/Cargo.toml @@ -27,6 +27,9 @@ fp_simd = ["axhal/fp_simd"] # Memory alloc = ["dep:axalloc", "axruntime/alloc", "axio/alloc"] +alloc_tlsf = ["axalloc/tlsf"] +alloc_slab = ["axalloc/slab"] +alloc_buddy = ["axalloc/buddy"] paging = ["axruntime/paging"] # Interrupts