[HW/SW] Cheshire integration - Linux on FPGA

.gitmodules

@@ -17,3 +17,6 @@
 	path = toolchain/riscv-llvm
 	url = https://github.com/llvm/llvm-project.git
 	ignore = dirty
+[submodule "cheshire/sw/cva6-sdk"]
+	path = cheshire/sw/cva6-sdk
+	url = [email protected]:moimfeld/cva6-sdk.git

Bender.lock

@@ -30,7 +30,7 @@ packages:
       Git: https://github.com/pulp-platform/common_verification.git
     dependencies: []
   cva6:
-    revision: f9ebe5006fefe97fc1d2a8b1515fa2fd321fec6d
+    revision: 5086bff7a039f02a2d9604bca7b1b3e9d961713e
     version: null
     source:
       Git: https://github.com/pulp-platform/cva6.git

Bender.yml

@@ -10,7 +10,7 @@ package:
 dependencies:
   axi:                { git: "https://github.com/pulp-platform/axi.git",                version: 0.39.1                               }
   common_cells:       { git: "https://github.com/pulp-platform/common_cells.git",       version: 1.22.1                               }
-  cva6:               { git: "https://github.com/pulp-platform/cva6.git",               rev: f9ebe5006fefe97fc1d2a8b1515fa2fd321fec6d } # mp/pulp-v1-araOS
+  cva6:               { git: "https://github.com/pulp-platform/cva6.git",               rev: 5086bff7a039f02a2d9604bca7b1b3e9d961713e } # mp/pulp-v1-araOS
   tech_cells_generic: { git: "https://github.com/pulp-platform/tech_cells_generic.git", version: 0.2.13                               }
   apb:                { git: "https://github.com/pulp-platform/apb.git",                version: 0.2.4                                }
 

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
  - Fix `acc_dispatcher` CVA6 bug for instructions with side effects
  - Fix NaN/subnormal floating-point handling in opqueues
  - Stall vfdiv/vfsqrt instructions following/preceding other fp instructions
+ - Fix vector slicing in the operand requesters
 
 ### Added
 
@@ -34,6 +35,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
  - Add virtual->physical address translation for Ara by sharing CVA6 MMU
  - Add Ara VLSU support for MMU exceptions
  - Add multi-precision conv3d
+ - Add Cheshire bare-metal FPGA flow for vcu128 and vcu118
+ - Add cva6-sdk submodule
+ - Add Cheshire Linux FPGA flow for vcu128 and vcu118
 
 ### Changed
 
@@ -56,6 +60,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
  - Memory size is now constant with NrLanes
  - Enable hierarchical verilation
  - Bump AXI and common cells to solve verilation warnings
+ - Update all Github Actions for CI
+ - Update READMEs with FPGA implementation instructions
 
 ## 3.0.0 - 2023-09-08
 

README.md

@@ -211,6 +211,12 @@ Currently, the following kernels support automatic VCD dumping: `fmatmul`, `fcon
 
 We also provide Synopsys Spyglass linting scripts in the hardware/spyglass. Run make lint in the hardware folder, with a specific MemPool configuration, to run the tests associated with the lint_rtl target.
 
+## FPGA implementation and Linux flow
+
+Ara supports Cheshire's FPGA flow and can be currently implemented on VCU128 and VCU118 in bare-metal and with Linux. The tested configuration is with 2 lanes.
+
+For information about the FPGA bare-metal and Linux flows, please refer to `cheshire/README.md`.
+
 ## Publications
 
 If you want to use Ara, you can cite us:

apps/Makefile

@@ -37,6 +37,13 @@ CVA6_BINARIES := $(addprefix bin/, $(cva6_tests))
 ARA_EXTENSIONS := rv64uv
 ARA_BINARIES := $(addprefix bin/, $(ara_tests))
 
+# Suffix for binaries
+ifeq ($(LINUX),1)
+BIN_SUFFIX := -linux
+else
+BIN_SUFFIX :=
+endif
+
 # FFT requires special treatment because of its header files
 ifeq ($(ENV_DEFINES),)
 bin/fft:  ENV_DEFINES += -DFFT_SAMPLES=$(subst ",,$(firstword $(def_args_fft)))
@@ -98,9 +105,9 @@ endef
 $(foreach app,$(APPS),$(eval $(call app_compile_template_spike,$(app))))
 
 define app_compile_template
-bin/$1: $1/data.S.o $(addsuffix .o, $(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) linker_script
+bin/$1$(BIN_SUFFIX): $1/data.S.o $(addsuffix .o, $(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC) -Iinclude $(RISCV_CCFLAGS) -o $$@ $$(addsuffix .o, $$(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) $$(RISCV_LDFLAGS) -T$$(CURDIR)/common/link.ld
+	$$(RISCV_CC) -Iinclude $(RISCV_CCFLAGS) -o $$@ $$(addsuffix .o, $$(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) $$(RISCV_LDFLAGS)
 	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
 	$$(RISCV_STRIP) $$@ -S --strip-unneeded
 endef
@@ -114,7 +121,7 @@ TESTS_$(1) := $(addprefix bin/, $($(addsuffix _ara_tests, $1)))
 
 bin/$(1)-ara-%: $(TESTS_DIR)/$(1)/%.$(2) $(RUNTIME_GCC) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC_GCC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS_GCC) $$(RISCV_LDFLAGS_GCC) -o $$@ $$< $(RUNTIME_GCC) -T$$(CURDIR)/common/link.ld
+	$$(RISCV_CC_GCC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS_GCC) $$(RISCV_LDFLAGS_GCC) -o $$@ $$< $(RUNTIME_GCC)
 	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
 	$$(RISCV_STRIP) $$@ -S --strip-unneeded
 endef
@@ -124,7 +131,7 @@ TESTS_$(1) := $(addprefix bin/, $($(addsuffix _ara_tests, $1)))
 
 bin/$(1)-ara-%: $(TESTS_DIR)/$(1)/%.$(2) $(RUNTIME_LLVM) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS) $$(RISCV_LDFLAGS) -o $$@ $$< $(RUNTIME_LLVM) -T$$(CURDIR)/common/link.ld
+	$$(RISCV_CC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS) $$(RISCV_LDFLAGS) -o $$@ $$< $(RUNTIME_LLVM)
 	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
 	$$(RISCV_STRIP) $$@ -S --strip-unneeded
 endef

apps/common/runtime.mk

@@ -44,7 +44,18 @@ RISCV_ARCH    ?= rv$(RISCV_XLEN)gcv
 RISCV_ABI     ?= lp64d
 RISCV_TARGET  ?= riscv$(RISCV_XLEN)-unknown-elf
 
-# Use LLVM
+# Use LLVM for bare-metal RVV and GCC for Linux RVV
+ifeq ($(LINUX),1)
+RISCV_PREFIX  ?= $(ARA_DIR)/cheshire/sw/cva6-sdk/buildroot/output/host/bin/riscv64-buildroot-linux-gnu-
+RISCV_CC      ?= $(RISCV_PREFIX)gcc
+RISCV_CXX     ?= $(RISCV_PREFIX)g++
+RISCV_OBJDUMP ?= $(RISCV_PREFIX)objdump
+RISCV_OBJCOPY ?= $(RISCV_PREFIX)objcopy
+RISCV_AS      ?= $(RISCV_PREFIX)as
+RISCV_AR      ?= $(RISCV_PREFIX)ar
+RISCV_LD      ?= $(RISCV_PREFIX)ld.lld
+RISCV_STRIP   ?= $(RISCV_PREFIX)strip
+else
 RISCV_PREFIX  ?= $(LLVM_INSTALL_DIR)/bin/
 RISCV_CC      ?= $(RISCV_PREFIX)clang
 RISCV_CXX     ?= $(RISCV_PREFIX)clang++
@@ -54,6 +65,7 @@ RISCV_AS      ?= $(RISCV_PREFIX)llvm-as
 RISCV_AR      ?= $(RISCV_PREFIX)llvm-ar
 RISCV_LD      ?= $(RISCV_PREFIX)ld.lld
 RISCV_STRIP   ?= $(RISCV_PREFIX)llvm-strip
+endif
 
 # Use gcc to compile scalar riscv-tests
 RISCV_CC_GCC  ?= $(GCC_INSTALL_DIR)/bin/$(RISCV_TARGET)-gcc
@@ -75,6 +87,9 @@ PYTHON ?= python3
 
 # Defines
 ENV_DEFINES ?=
+ifeq ($(LINUX),1)
+ENV_DEFINES += -DARA_LINUX=1
+endif
 ifeq ($(vcd_dump),1)
 ENV_DEFINES += -DVCD_DUMP=1
 endif
@@ -88,27 +103,38 @@ RISCV_WARNINGS += -Wunused-variable -Wall -Wextra -Wno-unused-command-line-argum
 LLVM_FLAGS     ?= -march=rv64gcv_zfh_zvfh0p1 -menable-experimental-extensions -mabi=$(RISCV_ABI) -mno-relax -fuse-ld=lld
 LLVM_V_FLAGS   ?= -fno-vectorize -mllvm -scalable-vectorization=off -mllvm -riscv-v-vector-bits-min=0 -Xclang -target-feature -Xclang +no-optimized-zero-stride-load
 RISCV_FLAGS    ?= $(LLVM_FLAGS) $(LLVM_V_FLAGS) -mcmodel=medany -I$(CURDIR)/common -std=gnu99 -O3 -ffast-math -fno-common -fno-builtin-printf $(DEFINES) $(RISCV_WARNINGS)
+ifeq ($(LINUX),1)
+RISCV_CCFLAGS  ?= -march=rv64gcv -mabi=$(RISCV_ABI) -I$(CURDIR)/common -O3 $(DEFINES)
+RISCV_LDFLAGS  ?=
+else
 RISCV_CCFLAGS  ?= $(RISCV_FLAGS) -ffunction-sections -fdata-sections
+RISCV_LDFLAGS  ?= -static -nostartfiles -lm -Wl,--gc-sections -T$(CURDIR)/common/link.ld
+endif
 RISCV_CCFLAGS_SPIKE  ?= $(RISCV_FLAGS) $(SPIKE_CCFLAGS) -ffunction-sections -fdata-sections
 RISCV_CXXFLAGS ?= $(RISCV_FLAGS) -ffunction-sections -fdata-sections
-RISCV_LDFLAGS  ?= -static -nostartfiles -lm -Wl,--gc-sections
 RISCV_LDFLAGS_SPIKE  ?= $(RISCV_LDFLAGS) $(SPIKE_LDFLAGS) -Wl,--gc-sections
 
 # GCC Flags
 RISCV_FLAGS_GCC    ?= -mcmodel=medany -march=$(RISCV_ARCH) -mabi=$(RISCV_ABI) -I$(CURDIR)/common -static -std=gnu99 -O3 -ffast-math -fno-common -fno-builtin-printf $(DEFINES) $(RISCV_WARNINGS)
 RISCV_CCFLAGS_GCC  ?= $(RISCV_FLAGS_GCC)
 RISCV_CXXFLAGS_GCC ?= $(RISCV_FLAGS_GCC)
-RISCV_LDFLAGS_GCC  ?= -static -nostartfiles -lm -lgcc $(RISCV_FLAGS_GCC)
+RISCV_LDFLAGS_GCC  ?= -static -nostartfiles -lm -lgcc $(RISCV_FLAGS_GCC) -T$(CURDIR)/common/link.ld
 
 ifeq ($(COMPILER),gcc)
 	RISCV_OBJDUMP_FLAGS ?=
 else
+ifneq ($(LINUX),1)
 	RISCV_OBJDUMP_FLAGS ?= --mattr=v
 endif
+endif
 
 # Compile two different versions of the runtime, since we cannot link code compiled with two different toolchains
 RUNTIME_GCC   ?= common/crt0-gcc.S.o common/printf-gcc.c.o common/string-gcc.c.o common/serial-gcc.c.o common/util-gcc.c.o
+ifeq ($(LINUX),1)
+RUNTIME_LLVM  ?= common/util-llvm.c.o
+else
 RUNTIME_LLVM  ?= common/crt0-llvm.S.o common/printf-llvm.c.o common/string-llvm.c.o common/serial-llvm.c.o common/util-llvm.c.o
+endif
 RUNTIME_SPIKE ?= $(spike_env_dir)/benchmarks/common/crt.S.o.spike $(spike_env_dir)/benchmarks/common/syscalls.c.o.spike common/util.c.o.spike
 
 .INTERMEDIATE: $(RUNTIME_GCC) $(RUNTIME_LLVM)

apps/common/util.c

@@ -22,6 +22,8 @@
 
 int *__dummy__errno__ptr__;
 
+unsigned long int timer;
+
 // Floating-point similarity check with threshold
 int similarity_check(double a, double b, double threshold) {
   double diff = a - b;

apps/conjugate_gradient/main.c

@@ -25,10 +25,13 @@
 #include "shared_kernel/spmv.h"
 #include "util.h"
 
-#ifndef SPIKE
-#include "printf.h"
-#else
+#ifdef SPIKE
+#include "util.h"
 #include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
+#include "printf.h"
 #endif
 
 #define USE_SPMV 1

apps/cos/main.c

@@ -20,10 +20,18 @@
 #include <string.h>
 
 #include "kernel/cos.h"
-#include "printf.h"
 #include "runtime.h"
+
 #include "util.h"
 
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
+#include "printf.h"
+#endif
+
 extern size_t N_f64;
 extern double angles_f64[] __attribute__((aligned(4 * NR_LANES)));
 extern double results_f64[] __attribute__((aligned(4 * NR_LANES)));

apps/dotproduct/main.c

@@ -23,10 +23,14 @@
 
 #include "kernel/dotproduct.h"
 
-#ifndef SPIKE
-#include "printf.h"
-#else
+#include "util.h"
+
+#ifdef SPIKE
 #include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
+#include "printf.h"
 #endif
 
 // Run also the scalar benchmark

apps/dropout/kernel/dropout.h

@@ -19,7 +19,15 @@
 #ifndef _DROPOUT_H_
 #define _DROPOUT_H_
 
+#include "util.h"
+
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
 #include <stdio.h>
+#else
+#include "printf.h"
+#endif
 
 #include <riscv_vector.h>
 
@@ -29,10 +37,6 @@
 
 #include "runtime.h"
 
-#ifndef SPIKE
-#include "printf.h"
-#endif
-
 void dropout_gold(const unsigned int n, const float *i, const float scale,
                   const uint8_t *sel_ptr, float *o);
 void dropout_vec(const unsigned int n, const float *i, const float scale,

apps/dtype-conv3d/main.c

@@ -22,7 +22,11 @@
 
 #include "runtime.h"
 
-#ifndef SPIKE
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
 #include "printf.h"
 #endif
 

apps/dtype-matmul/main.c

@@ -18,13 +18,16 @@
 //         Samuel Riedel, ETH Zurich
 
 #include <stdint.h>
-#include <stdio.h>
 #include <string.h>
 
 #include "runtime.h"
 #include "util.h"
 
-#ifndef SPIKE
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
 #include "printf.h"
 #endif
 

apps/dwt/main.c

@@ -23,10 +23,12 @@
 #include "runtime.h"
 #include "util.h"
 
-#ifndef SPIKE
-#include "printf.h"
-#else
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
 #include <stdio.h>
+#else
+#include "printf.h"
 #endif
 
 #define CHECK

apps/exp/main.c

@@ -23,10 +23,12 @@
 #include "runtime.h"
 #include "util.h"
 
-#ifndef SPIKE
-#include "printf.h"
-#else
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
 #include <stdio.h>
+#else
+#include "printf.h"
 #endif
 
 extern size_t N_f64;

apps/fconv2d/main.c

@@ -17,14 +17,17 @@
 // Author: Matteo Perotti
 
 #include <stdint.h>
-#include <stdio.h>
 #include <string.h>
 
 #include "fconv2d.h"
 #include "runtime.h"
 #include "util.h"
 
-#ifndef SPIKE
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
 #include "printf.h"
 #endif
 

apps/fconv3d/main.c

@@ -17,14 +17,17 @@
 // Author: Matteo Perotti
 
 #include <stdint.h>
-#include <stdio.h>
 #include <string.h>
 
 #include "fconv3d.h"
 #include "runtime.h"
 #include "util.h"
 
-#ifndef SPIKE
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
 #include "printf.h"
 #endif
 

apps/fdotproduct/main.c

@@ -24,10 +24,12 @@
 
 #include "kernel/fdotproduct.h"
 
-#ifndef SPIKE
-#include "printf.h"
-#else
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
 #include <stdio.h>
+#else
+#include "printf.h"
 #endif
 
 // Threshold for FP comparisons