Merge pull request arceos-org#110 from rcore-os/tlsf-alloc

Add TLSF allocator

README.md

@@ -92,7 +92,7 @@ export PATH=`pwd`/x86_64-linux-musl-cross/bin:`pwd`/aarch64-linux-musl-cross/bin
 
 ```bash
 # in arceos directory
-make A=path/to/app ARCH=<arch> LOG=<log> NET=[y|n] BLK=[y|n]
+make A=path/to/app ARCH=<arch> LOG=<log>
 ```
 
 Where `<arch>` should be one of `riscv64`, `aarch64`，`x86_64`.
@@ -106,9 +106,11 @@ More arguments and targets can be found in [Makefile](Makefile).
 For example, to run the [httpserver](apps/net/httpserver/) on `qemu-system-aarch64` with 4 cores:
 
 ```bash
-make A=apps/net/httpserver ARCH=aarch64 LOG=info NET=y SMP=4 run
+make A=apps/net/httpserver ARCH=aarch64 LOG=info SMP=4 run NET=y
 ```
 
+Note that the `NET=y` argument is required to enable the network device in QEMU. These arguments (`BLK`, `GRAPHIC`, etc.) only take effect at runtime not build time.
+
 ### Your custom apps
 
 #### Rust

apps/c/memtest/expect_trace.out

@@ -12,6 +12,7 @@ Found physcial memory regions:
  .bss (READ | WRITE | RESERVED)
  free memory (READ | WRITE | FREE)
 Initialize global memory allocator...
+  use TLSF allocator.
 initialize global allocator at: \[0x[0-9a-f]\+, 0x[0-9a-f]\+)
 Initialize kernel page table...
 Initialize platform devices...

apps/memtest/expect_trace.out

@@ -12,6 +12,7 @@ Found physcial memory regions:
  .bss (READ | WRITE | RESERVED)
  free memory (READ | WRITE | FREE)
 Initialize global memory allocator...
+  use TLSF allocator.
 initialize global allocator at: \[0x[0-9a-f]\+, 0x[0-9a-f]\+)
 Initialize kernel page table...
 Initialize platform devices...

apps/memtest/src/main.rs

@@ -10,7 +10,7 @@ use std::collections::BTreeMap;
 use std::vec::Vec;
 
 fn test_vec(rng: &mut impl RngCore) {
-    const N: usize = 1_000_000;
+    const N: usize = 3_000_000;
     let mut v = Vec::with_capacity(N);
     for _ in 0..N {
         v.push(rng.next_u32());
@@ -23,7 +23,7 @@ fn test_vec(rng: &mut impl RngCore) {
 }
 
 fn test_btree_map(rng: &mut impl RngCore) {
-    const N: usize = 10_000;
+    const N: usize = 50_000;
     let mut m = BTreeMap::new();
     for _ in 0..N {
         let value = rng.next_u32();

crates/allocator/Cargo.toml

@@ -9,7 +9,29 @@ homepage = "https://github.com/rcore-os/arceos"
 repository = "https://github.com/rcore-os/arceos/tree/main/crates/allocator"
 documentation = "https://rcore-os.github.io/arceos/allocator/index.html"
 
+[features]
+default = []
+full = ["bitmap", "tlsf", "slab", "buddy", "allocator_api"]
+
+bitmap = ["dep:bitmap-allocator"]
+
+tlsf = ["dep:rlsf"]
+slab = ["dep:slab_allocator"]
+buddy = ["dep:buddy_system_allocator"]
+
+allocator_api = []
+
 [dependencies]
-buddy_system_allocator = { version = "0.9", default-features = false }
-bitmap-allocator = { git = "https://github.com/rcore-os/bitmap-allocator.git", rev = "88e871a" }
-slab_allocator = { path = "../slab_allocator" }
+buddy_system_allocator = { version = "0.9", default-features = false, optional = true }
+slab_allocator = { path = "../slab_allocator", optional = true }
+rlsf = { version = "0.2", optional = true }
+bitmap-allocator = { git = "https://github.com/rcore-os/bitmap-allocator.git", rev = "88e871a", optional = true }
+
+[dev-dependencies]
+allocator = { path = ".", features = ["full"] }
+rand = { version = "0.8", features = ["small_rng"] }
+criterion = { version = "0.5", features = ["html_reports"] }
+
+[[bench]]
+name = "collections"
+harness = false

crates/allocator/benches/collections.rs

@@ -0,0 +1,101 @@
+#![feature(allocator_api)]
+#![feature(btreemap_alloc)]
+
+mod utils;
+
+use std::alloc::Allocator;
+use std::collections::BTreeMap;
+use std::io::Write;
+
+use allocator::{AllocatorRc, BuddyByteAllocator, SlabByteAllocator, TlsfByteAllocator};
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use rand::{rngs::SmallRng, seq::SliceRandom, RngCore, SeedableRng};
+
+use self::utils::MemoryPool;
+
+const POOL_SIZE: usize = 1024 * 1024 * 128;
+
+fn vec_push(n: usize, alloc: &(impl Allocator + Clone)) {
+    let mut v: Vec<u32, _> = Vec::new_in(alloc.clone());
+    for _ in 0..n {
+        v.push(0xdead_beef);
+    }
+    drop(v);
+}
+
+fn vec_rand_free(n: usize, blk_size: usize, alloc: &(impl Allocator + Clone)) {
+    let mut v = Vec::new_in(alloc.clone());
+    for _ in 0..n {
+        let block = Vec::<u64, _>::with_capacity_in(blk_size, alloc.clone());
+        v.push(block);
+    }
+
+    let mut rng = SmallRng::seed_from_u64(0xdead_beef);
+    let mut index = Vec::with_capacity_in(n, alloc.clone());
+    for i in 0..n {
+        index.push(i);
+    }
+    index.shuffle(&mut rng);
+
+    for i in index {
+        v[i] = Vec::new_in(alloc.clone());
+    }
+    drop(v);
+}
+
+fn btree_map(n: usize, alloc: &(impl Allocator + Clone)) {
+    let mut rng = SmallRng::seed_from_u64(0xdead_beef);
+    let mut m = BTreeMap::new_in(alloc.clone());
+    for _ in 0..n {
+        if rng.next_u32() % 5 == 0 && !m.is_empty() {
+            m.pop_first();
+        } else {
+            let value = rng.next_u32();
+            let mut key = Vec::new_in(alloc.clone());
+            write!(&mut key, "key_{value}").unwrap();
+            m.insert(key, value);
+        }
+    }
+    m.clear();
+    drop(m);
+}
+
+fn bench(c: &mut Criterion, alloc_name: &str, alloc: impl Allocator + Clone) {
+    let mut g = c.benchmark_group(alloc_name);
+    g.bench_function("vec_push_3M", |b| {
+        b.iter(|| vec_push(black_box(3_000_000), &alloc));
+    });
+    g.sample_size(10);
+    g.bench_function("vec_rand_free_25K_64", |b| {
+        b.iter(|| vec_rand_free(black_box(25_000), black_box(64), &alloc));
+    });
+    g.bench_function("vec_rand_free_7500_520", |b| {
+        b.iter(|| vec_rand_free(black_box(7_500), black_box(520), &alloc));
+    });
+    g.bench_function("btree_map_50K", |b| {
+        b.iter(|| btree_map(black_box(50_000), &alloc));
+    });
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let mut pool = MemoryPool::new(POOL_SIZE);
+    bench(c, "system", std::alloc::System);
+    bench(
+        c,
+        "tlsf",
+        AllocatorRc::new(TlsfByteAllocator::new(), pool.as_slice()),
+    );
+    bench(
+        c,
+        "slab",
+        AllocatorRc::new(SlabByteAllocator::new(), pool.as_slice()),
+    );
+    bench(
+        c,
+        "buddy",
+        AllocatorRc::new(BuddyByteAllocator::new(), pool.as_slice()),
+    );
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

crates/allocator/benches/utils/mod.rs

@@ -0,0 +1,25 @@
+use std::alloc::Layout;
+use std::ptr::NonNull;
+
+pub struct MemoryPool {
+    ptr: NonNull<u8>,
+    layout: Layout,
+}
+
+impl MemoryPool {
+    pub fn new(size: usize) -> Self {
+        let layout = Layout::from_size_align(size, 4096).unwrap();
+        let ptr = NonNull::new(unsafe { std::alloc::alloc_zeroed(layout) }).unwrap();
+        Self { ptr, layout }
+    }
+
+    pub fn as_slice(&mut self) -> &mut [u8] {
+        unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.layout.size()) }
+    }
+}
+
+impl Drop for MemoryPool {
+    fn drop(&mut self) {
+        unsafe { std::alloc::dealloc(self.ptr.as_ptr(), self.layout) };
+    }
+}

crates/allocator/src/buddy.rs

@@ -4,6 +4,7 @@
 
 use buddy_system_allocator::Heap;
 use core::alloc::Layout;
+use core::num::NonZeroUsize;
 
 use crate::{AllocError, AllocResult, BaseAllocator, ByteAllocator};
 
@@ -35,17 +36,17 @@ impl BaseAllocator for BuddyByteAllocator {
 }
 
 impl ByteAllocator for BuddyByteAllocator {
-    fn alloc(&mut self, size: usize, align_pow2: usize) -> AllocResult<usize> {
+    fn alloc(&mut self, layout: Layout) -> AllocResult<NonZeroUsize> {
         self.inner
-            .alloc(Layout::from_size_align(size, align_pow2).unwrap())
-            .map(|ptr| ptr.as_ptr() as usize)
+            .alloc(layout)
+            .map(|ptr| ptr.addr())
             .map_err(|_| AllocError::NoMemory)
     }
 
-    fn dealloc(&mut self, pos: usize, size: usize, align_pow2: usize) {
+    fn dealloc(&mut self, pos: NonZeroUsize, layout: Layout) {
         self.inner.dealloc(
-            unsafe { core::ptr::NonNull::new_unchecked(pos as *mut u8) },
-            Layout::from_size_align(size, align_pow2).unwrap(),
+            unsafe { core::ptr::NonNull::new_unchecked(pos.get() as _) },
+            layout,
         )
     }
 

crates/allocator/src/lib.rs

@@ -9,16 +9,33 @@
 //! - [`IdAllocator`]: Used to allocate unique IDs.
 
 #![no_std]
+#![feature(strict_provenance)]
 #![feature(result_option_inspect)]
+#![cfg_attr(feature = "allocator_api", feature(allocator_api))]
 
+#[cfg(feature = "bitmap")]
 mod bitmap;
-mod buddy;
-mod slab;
-
+#[cfg(feature = "bitmap")]
 pub use bitmap::BitmapPageAllocator;
+
+#[cfg(feature = "buddy")]
+mod buddy;
+#[cfg(feature = "buddy")]
 pub use buddy::BuddyByteAllocator;
+
+#[cfg(feature = "slab")]
+mod slab;
+#[cfg(feature = "slab")]
 pub use slab::SlabByteAllocator;
 
+#[cfg(feature = "tlsf")]
+mod tlsf;
+#[cfg(feature = "tlsf")]
+pub use tlsf::TlsfByteAllocator;
+
+use core::alloc::Layout;
+use core::num::NonZeroUsize;
+
 /// The error type used for allocation.
 #[derive(Debug)]
 pub enum AllocError {
@@ -47,10 +64,10 @@ pub trait BaseAllocator {
 /// Byte-granularity allocator.
 pub trait ByteAllocator: BaseAllocator {
     /// Allocate memory with the given size (in bytes) and alignment.
-    fn alloc(&mut self, size: usize, align_pow2: usize) -> AllocResult<usize>;
+    fn alloc(&mut self, layout: Layout) -> AllocResult<NonZeroUsize>;
 
     /// Deallocate memory at the given position, size, and alignment.
-    fn dealloc(&mut self, pos: usize, size: usize, align_pow2: usize);
+    fn dealloc(&mut self, pos: NonZeroUsize, layout: Layout);
 
     /// Returns total memory size in bytes.
     fn total_bytes(&self) -> usize;
@@ -116,3 +133,51 @@ const fn align_down(pos: usize, align: usize) -> usize {
 const fn align_up(pos: usize, align: usize) -> usize {
     (pos + align - 1) & !(align - 1)
 }
+
+#[cfg(feature = "allocator_api")]
+mod allocator_api {
+    extern crate alloc;
+
+    use super::ByteAllocator;
+    use alloc::rc::Rc;
+    use core::alloc::{AllocError, Allocator, Layout};
+    use core::cell::RefCell;
+    use core::ptr::NonNull;
+
+    /// A byte-allocator wrapped in [`Rc<RefCell>`] that implements [`core::alloc::Allocator`].
+    pub struct AllocatorRc<A: ByteAllocator>(Rc<RefCell<A>>);
+
+    impl<A: ByteAllocator> AllocatorRc<A> {
+        /// Creates a new allocator with the given memory pool.
+        pub fn new(mut inner: A, pool: &mut [u8]) -> Self {
+            inner.init(pool.as_mut_ptr() as usize, pool.len());
+            Self(Rc::new(RefCell::new(inner)))
+        }
+    }
+
+    unsafe impl<A: ByteAllocator> Allocator for AllocatorRc<A> {
+        fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
+            match layout.size() {
+                0 => Ok(NonNull::slice_from_raw_parts(NonNull::dangling(), 0)),
+                size => {
+                    let raw_addr = self.0.borrow_mut().alloc(layout).map_err(|_| AllocError)?;
+                    let ptr = unsafe { NonNull::new_unchecked(raw_addr.get() as _) };
+                    Ok(NonNull::slice_from_raw_parts(ptr, size))
+                }
+            }
+        }
+
+        unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
+            self.0.borrow_mut().dealloc(ptr.addr(), layout)
+        }
+    }
+
+    impl<A: ByteAllocator> Clone for AllocatorRc<A> {
+        fn clone(&self) -> Self {
+            Self(self.0.clone())
+        }
+    }
+}
+
+#[cfg(feature = "allocator_api")]
+pub use allocator_api::AllocatorRc;

crates/allocator/src/slab.rs

@@ -4,6 +4,7 @@
 
 use super::{AllocError, AllocResult, BaseAllocator, ByteAllocator};
 use core::alloc::Layout;
+use core::num::NonZeroUsize;
 use slab_allocator::Heap;
 
 /// A byte-granularity memory allocator based on the [slab allocator].
@@ -42,17 +43,15 @@ impl BaseAllocator for SlabByteAllocator {
 }
 
 impl ByteAllocator for SlabByteAllocator {
-    fn alloc(&mut self, size: usize, align_pow2: usize) -> AllocResult<usize> {
+    fn alloc(&mut self, layout: Layout) -> AllocResult<NonZeroUsize> {
         self.inner_mut()
-            .allocate(Layout::from_size_align(size, align_pow2).unwrap())
+            .allocate(layout)
+            .map(|addr| NonZeroUsize::new(addr).unwrap())
             .map_err(|_| AllocError::NoMemory)
     }
 
-    fn dealloc(&mut self, pos: usize, size: usize, align_pow2: usize) {
-        unsafe {
-            self.inner_mut()
-                .deallocate(pos, Layout::from_size_align(size, align_pow2).unwrap())
-        }
+    fn dealloc(&mut self, pos: NonZeroUsize, layout: Layout) {
+        unsafe { self.inner_mut().deallocate(pos.get(), layout) }
     }
 
     fn total_bytes(&self) -> usize {