Implement r0 crate in assembly

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Use inline assembly instead of pre-compiled blobs
 - Removed bors in favor of GitHub Merge Queue
 - `start_trap_rust` is now marked as `unsafe`
+- Implement `r0` as inline assembly
 
 ## [v0.11.0] - 2023-01-18
 

Cargo.toml

@@ -15,7 +15,6 @@ s-mode = []
 single-hart = []
 
 [dependencies]
-r0 = "1.0.0"
 riscv = "0.10"
 riscv-rt-macros = { path = "macros", version = "0.2.0" }
 

link.x

@@ -84,7 +84,7 @@ SECTIONS
     _edata = .;
   } > REGION_DATA AT > REGION_RODATA
 
-  .bss (NOLOAD) :
+  .bss (NOLOAD) : ALIGN(4)
   {
     _sbss = .;
     *(.sbss .sbss.* .bss .bss.*);
@@ -150,6 +150,10 @@ BUG(riscv-rt): .data is not 4-byte aligned");
 ASSERT(_sidata % 4 == 0, "
 BUG(riscv-rt): the LMA of .data is not 4-byte aligned");
 
+/* Make sure that we can safely perform .data initialization on RV64 */ 
+ASSERT(_sidata % 8 == _sdata % 8, "
+BUG(riscv-rt): .data is not similarly 8-byte aligned to the LMA of .data");
+
 ASSERT(_sbss % 4 == 0 && _ebss % 4 == 0, "
 BUG(riscv-rt): .bss is not 4-byte aligned");
 

src/lib.rs

@@ -366,6 +366,8 @@
 #[cfg(riscv)]
 mod asm;
 
+use core::sync::atomic::{compiler_fence, Ordering};
+
 #[cfg(feature = "s-mode")]
 use riscv::register::{scause as xcause, stvec as xtvec, stvec::TrapMode as xTrapMode};
 
@@ -378,19 +380,6 @@ pub use riscv_rt_macros::{entry, pre_init};
 #[doc(hidden)]
 pub static __ONCE__: () = ();
 
-extern "C" {
-    // Boundaries of the .bss section
-    static mut _ebss: u32;
-    static mut _sbss: u32;
-
-    // Boundaries of the .data section
-    static mut _edata: u32;
-    static mut _sdata: u32;
-
-    // Initial values of the .data section (stored in Flash)
-    static _sidata: u32;
-}
-
 /// Rust entry point (_start_rust)
 ///
 /// Zeros bss section, initializes data section and calls main. This function never returns.
@@ -424,8 +413,145 @@ pub unsafe extern "C" fn start_rust(a0: usize, a1: usize, a2: usize) -> ! {
     if _mp_hook(hartid) {
         __pre_init();
 
-        r0::zero_bss(&mut _sbss, &mut _ebss);
-        r0::init_data(&mut _sdata, &mut _edata, &_sidata);
+        // Initialize RAM
+        // 1. Copy over .data from flash to RAM
+        // 2. Zero out .bss
+
+        #[cfg(target_arch = "riscv32")]
+        core::arch::asm!(
+            "
+                // Copy over .data
+                la      {start},_sdata
+                la      {end},_edata
+                la      {input},_sidata
+
+            	bgeu    {start},{end},2f
+            1:
+            	lw      {a},0({input})
+            	addi    {input},{input},4
+            	sw      {a},0({start})
+            	addi    {start},{start},4
+            	bltu    {start},{end},1b
+
+            2:
+                li      {a},0
+
+                // Zero out .bss
+            	la      {start},_sbss
+            	la      {end},_ebss
+
+            	bgeu    {start},{end},3f
+            2:
+            	sw      zero,0({start})
+            	addi    {start},{start},4
+            	bltu    {start},{end},2b
+
+            3:
+                li      {start},0
+                li      {end},0
+                li      {input},0
+            ",
+            start = out(reg) _,
+            end = out(reg) _,
+            input = out(reg) _,
+            a = out(reg) _,
+        );
+
+        #[cfg(target_arch = "riscv64")]
+        core::arch::asm!(
+            "
+                // Copy over .data
+                la      {start},_sdata
+                la      {end},_edata
+                la      {input},_sidata
+
+                bgeu    {start},{end},3f
+
+                //    If _sdata and _sidata are not 8-byte aligned, we copy one word before the main loop. This way, in
+                //    the main loop, we are sure `start` and `input` are 8-byte aligned. This is needed to safely
+                //    perform load and store double instructions.
+                //
+                //    NOTE: We assert in the `link.x` file that _sdata and _sidata are similarly 8-byte aligned. This is
+                //          needed for the main loop here.
+                andi    {b},{start},4
+                beqz    {b},0f
+                lw      {a},0({input})
+                addi    {input},{input},4
+                sw      {a},0({start})
+                addi    {start},{start},4
+
+            0: // .data Main Loop Initialization
+                //    b = FLOOR_ALIGN(_edata, 4)
+                andi    {b},{end},4
+                sub     {b},{end},{b}
+
+            	bgeu    {start},{b},2f
+            1: // .data Main Loop
+            	ld      {a},0({input})
+            	addi    {input},{input},8
+            	sd      {a},0({start})
+            	addi    {start},{start},8
+            	bltu    {start},{b},1b
+
+            2: // .data end align
+                //    If _edata is not 8-byte aligned, we copy one word after the main loop. This way we are sure we
+                //    copied all the data even if _edata is 4-byte aligned.
+                andi    {b},{end},4
+                beqz    {b},3f
+                lw      {a},0({input})
+                addi    {input},{input},4
+                sw      {a},0({start})
+                addi    {start},{start},4
+
+            3: // .data zero registers
+                li      {a},0
+                li      {input},0
+
+            4: // zero out .bss start
+            	la      {start},_sbss
+            	la      {end},_ebss
+
+                bgeu    {start},{end},8f
+
+                //    If _sbss is not 8-byte aligned, we zero one word before the main loop. This way, in the main
+                //    loop, we are sure `start` is 8-byte aligned. This is needed to safely perform store double
+                //    instruction.
+                andi    {b},{start},4
+                beqz    {b},5f
+            	sw      zero,0({start})
+            	addi    {start},{start},4
+
+            5: // .bss main loop initialization
+                //    b = FLOOR_ALIGN(_ebss, 4)
+                andi    {b},{end},4
+                sub     {b},{end},{b}
+
+            	bgeu    {start},{b},7f
+            6: // .bss main loop
+            	sd      zero,0({start})
+            	addi    {start},{start},8
+            	bltu    {start},{b},6b
+
+            7: // .bss end align
+                //    If _ebss is not 8-byte aligned, we need to zero more one word after the main loop.
+                andi    {b},{end},4
+                beqz    {b},8f
+            	sw      zero,0({start})
+
+            8: // .bss zero registers
+                //    Zero out used registers
+                li      {b},0
+                li      {start},0
+                li      {end},0
+        ",
+            start = out(reg) _,
+            end = out(reg) _,
+            input = out(reg) _,
+            a = out(reg) _,
+            b = out(reg) _,
+        );
+
+        compiler_fence(Ordering::SeqCst);
     }
 
     // TODO: Enable FPU when available