From fa6140b8930b916419a23aa935b9a984a9cbe312 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Wed, 15 Feb 2023 11:00:25 +0100
Subject: [PATCH 01/34] Added a way of injecting user module for clock gate(s)

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 configs/veer.config     | 16 ++++++++++++++--
 design/lib/beh_lib.sv   | 13 +++++++++++--
 testbench/flist         |  1 +
 testbench/user_cells.sv | 35 +++++++++++++++++++++++++++++++++++
 4 files changed, 61 insertions(+), 4 deletions(-)
 create mode 100644 testbench/user_cells.sv

diff --git a/configs/veer.config b/configs/veer.config
index 48a9b82203e..f705eb1cf73 100755
--- a/configs/veer.config
+++ b/configs/veer.config
@@ -33,7 +33,7 @@ my @argv_orig = @ARGV;
 my $defines_case = "U";
 
 # Include these macros in verilog (pattern matched)
-my @verilog_vars = qw (xlen config_key reset_vec tec_rv_icg numiregs nmi_vec target protection.* testbench.* dccm.* retstack core.* iccm.* btb.* bht.* icache.* pic.* regwidth memmap bus.*);
+my @verilog_vars = qw (xlen config_key reset_vec tec_rv_icg numiregs nmi_vec target protection.* testbench.* dccm.* retstack core.* iccm.* btb.* bht.* icache.* pic.* regwidth memmap bus.* tech_specific_.* user_.*);
 
 # Include these macros in assembly (pattern matched)
 my @asm_vars = qw (xlen reset_vec nmi_vec target dccm.* iccm.* pic.* memmap  testbench.* protection.* core.*);
@@ -48,7 +48,7 @@ my @dvars = qw(retstack btb bht core dccm iccm icache pic protection memmap bus)
 # Prefix all macros with
 my $prefix = "RV_";
 # No prefix if keyword has
-my $no_prefix = 'RV|TOP|tec_rv_icg|regwidth|clock_period|^datawidth|verilator|SDVT_AHB';
+my $no_prefix = 'RV|TOP|tec_rv_icg|regwidth|clock_period|^datawidth|verilator|SDVT_AHB|tech_specific_.*|user_.*';
 
 my $vlog_use__wh = 1;
 
@@ -1111,6 +1111,10 @@ our %config = (#{{{
     "csr" => \%csr,                                                     # Whisper only
     "perf_events" => \@perf_events,                                     # Whisper only
     "even_odd_trigger_chains" => "true",                                # Whisper only
+
+    "tech_specific_ec_rv_icg" => '0',
+
+    "user_ec_rv_icg" => 'user_clock_gate',
 );
 
 
@@ -1890,6 +1894,14 @@ if (defined($config{"testbench"}{"build_axi_native"}) && ($config{"testbench"}{"
 
 delete $config{core}{fpga_optimize} if ($config{core}{fpga_optimize} == 0);
 
+# Remove TECH_SPECIFIC_* defines if they are set to 0
+foreach my $key (keys(%config)) {
+    if (grep(/tech_specific_/, $key)) {
+        if ($config{$key} == 0) {
+            delete $config{$key};
+        }
+    }
+}
 
 print "$self: Writing $tdfile\n";
 print "$self: Writing $paramfile\n";
diff --git a/design/lib/beh_lib.sv b/design/lib/beh_lib.sv
index a36e86fff8f..b92ddbf9b48 100644
--- a/design/lib/beh_lib.sv
+++ b/design/lib/beh_lib.sv
@@ -748,7 +748,7 @@ module rvecc_decode_64  (
 
  endmodule // rvecc_decode_64
 
-
+`ifndef TECH_SPECIFIC_EC_RV_ICG
 module `TEC_RV_ICG
   (
    input logic SE, EN, CK,
@@ -773,6 +773,7 @@ module `TEC_RV_ICG
    assign Q = CK & en_ff;
 
 endmodule
+`endif
 
 `ifndef RV_FPGA_OPTIMIZE
 module rvclkhdr
@@ -786,7 +787,11 @@ module rvclkhdr
    logic   SE;
    assign       SE = 0;
 
+`ifdef TECH_SPECIFIC_EC_RV_ICG
+   `USER_EC_RV_ICG clkhdr ( .*, .EN(en), .CK(clk), .Q(l1clk));
+`else
    `TEC_RV_ICG clkhdr ( .*, .EN(en), .CK(clk), .Q(l1clk));
+`endif
 
 endmodule // rvclkhdr
 `endif
@@ -805,7 +810,11 @@ module rvoclkhdr
 `ifdef RV_FPGA_OPTIMIZE
    assign l1clk = clk;
 `else
-   `TEC_RV_ICG clkhdr ( .*, .EN(en), .CK(clk), .Q(l1clk));
+   `ifdef TECH_SPECIFIC_EC_RV_ICG
+      `USER_EC_RV_ICG clkhdr ( .*, .EN(en), .CK(clk), .Q(l1clk));
+   `else
+      `TEC_RV_ICG clkhdr ( .*, .EN(en), .CK(clk), .Q(l1clk));
+    `endif
 `endif
 
 endmodule
diff --git a/testbench/flist b/testbench/flist
index d1c28f09fbd..402cd6f9676 100644
--- a/testbench/flist
+++ b/testbench/flist
@@ -45,3 +45,4 @@ $RV_ROOT/design/lib/el2_lib.sv
 -v $RV_ROOT/design/lib/mem_lib.sv
 -y $RV_ROOT/design/lib
 -v $RV_ROOT/testbench/axi_lsu_dma_bridge.sv
+-v $RV_ROOT/testbench/user_cells.sv
diff --git a/testbench/user_cells.sv b/testbench/user_cells.sv
new file mode 100644
index 00000000000..f61e624f5f3
--- /dev/null
+++ b/testbench/user_cells.sv
@@ -0,0 +1,35 @@
+
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2020 Western Digital Corporation or its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file contains examples of user (technology specific) cells that can
+// be used thruought the core
+
+// Clock gate example
+module user_clock_gate (
+    input  logic CK,
+    output logic Q,
+    input  logic EN
+);
+
+    logic gate;
+
+    initial gate = 0;
+    always @(negedge CK)
+        gate <= EN;
+
+    assign Q = CK & gate;
+
+endmodule

From 7f9cd4e8d0a11535aa6fda7235b3e6c03fcecdec Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 3 Apr 2023 16:17:41 +0200
Subject: [PATCH 02/34] Corrected default WIDTH param value of rvdffppe

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 design/lib/beh_lib.sv | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/design/lib/beh_lib.sv b/design/lib/beh_lib.sv
index a36e86fff8f..662de73c6e6 100644
--- a/design/lib/beh_lib.sv
+++ b/design/lib/beh_lib.sv
@@ -291,7 +291,7 @@ endmodule
 // special power flop for predict packet
 // format: { LEFT, RIGHT==31 }
 // LEFT # of bits will be done with rvdffe; RIGHT is enabled by LEFT[LSB] & en
-module rvdffppe #( parameter WIDTH=32 )
+module rvdffppe #( parameter integer WIDTH = 39 )
    (
      input  logic [WIDTH-1:0] din,
      input  logic             clk,
@@ -301,13 +301,13 @@ module rvdffppe #( parameter WIDTH=32 )
      output logic [WIDTH-1:0] dout
      );
 
-   localparam RIGHT = 31;
-   localparam LEFT = WIDTH - RIGHT;
+   localparam integer RIGHT = 31;
+   localparam integer LEFT  = WIDTH - RIGHT;
 
-   localparam LMSB = WIDTH-1;
-   localparam LLSB = LMSB-LEFT+1;
-   localparam RMSB = LLSB-1;
-   localparam RLSB = LLSB-RIGHT;
+   localparam integer LMSB  = WIDTH-1;
+   localparam integer LLSB  = LMSB-LEFT+1;
+   localparam integer RMSB  = LLSB-1;
+   localparam integer RLSB  = LLSB-RIGHT;
 
 
 `ifndef RV_PHYSICAL

From 0a778d6eaf9bf00734c3af6bfb9f1a93ef109866 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Tue, 4 Apr 2023 15:36:04 +0200
Subject: [PATCH 03/34] Fixed swapped mnemonics in testbench/dasm.svi

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 testbench/dasm.svi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testbench/dasm.svi b/testbench/dasm.svi
index aef232c1b89..1c03aea42e1 100644
--- a/testbench/dasm.svi
+++ b/testbench/dasm.svi
@@ -305,7 +305,7 @@ string mn;
     imm = opcode[24:20];
     case(opcode[14:12])
     1: mn = "slli";
-    5: mn = opcode[30] ? "srli": "srai";
+    5: mn = opcode[30] ? "srai": "srli";
     endcase
 
     return $sformatf("%s    %s,%s,%0d", mn, abi_reg[opcode[11:7]], abi_reg[opcode[19:15]], imm);

From 7992955ec038c7ba508e0c0d255cbc269b5094e1 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Thu, 6 Apr 2023 14:37:34 +0200
Subject: [PATCH 04/34] Initial makefile able to run RISC-V DV code in
 Verilated simulation and in spike ISS

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile | 85 +++++++++++++++++++++++++++++++++++++++++
 tools/riscv-dv/link.ld  | 18 +++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 tools/riscv-dv/Makefile
 create mode 100644 tools/riscv-dv/link.ld

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
new file mode 100644
index 00000000000..32884ca4acc
--- /dev/null
+++ b/tools/riscv-dv/Makefile
@@ -0,0 +1,85 @@
+RISCV_DV_PATH   = $(RV_ROOT)/third_party/riscv-dv
+RISCV_DV_TEST  ?= instr_base_test
+
+WORK_DIR       ?= work
+TEST_DIR        = $(WORK_DIR)/test_$(RISCV_DV_TEST)
+
+UVM_VERBOSITY   = NONE
+SEED            = 10
+INSTR_COUNT     = 100
+
+VEER_TARGET     = default
+VEER_CONF       = -set build_axi4 \
+                  -set reset_vec=0x2000
+
+VERILATOR       = verilator
+VERILATOR_CFLAGS= "-std=c++11"
+VERILATOR_INC   = -I$(WORK_DIR) -I$(RV_ROOT)/testbench 
+VERILATOR_EXE   = $(RV_ROOT)/testbench/test_tb_top.cpp
+
+GCC_PREFIX      = riscv64-unknown-elf
+ARCH            = rv32imc
+ABI             = ilp32
+#LDSCRIPT        = $(RV_ROOT)/testbench/link.ld
+LDSCRIPT        = link.ld
+
+HDL_FILES = $(WORK_DIR)/common_defines.vh \
+            $(WORK_DIR)/el2_pdef.vh \
+            $(RV_ROOT)/testbench/tb_top.sv \
+            $(RV_ROOT)/testbench/ahb_sif.sv \
+            $(RV_ROOT)/design/include/el2_def.sv
+
+# Directory rules
+$(WORK_DIR):
+	mkdir -p $@
+
+$(TEST_DIR):
+	mkdir -p $@
+
+# VeeR config
+$(WORK_DIR)/defines.h: | $(WORK_DIR)
+	BUILD_PATH=$(WORK_DIR) $(RV_ROOT)/configs/veer.config -target=$(VEER_TARGET) $(VEER_CONF)
+	echo '`undef RV_ASSERT_ON' >> $(WORK_DIR)/common_defines.vh
+
+# Verilated testbench rules
+$(WORK_DIR)/verilator/Vtb_top.mk: $(WORK_DIR)/defines.h
+	$(VERILATOR) --cc -CFLAGS $(VERILATOR_CFLAGS) $(VERILATOR_INC) \
+        $(HDL_FILES) -f $(RV_ROOT)/testbench/flist --top-module tb_top \
+		-exe $(VERILATOR_EXE) -Wno-WIDTH -Wno-UNOPTFLAT --autoflush \
+		-Mdir $(WORK_DIR)/verilator
+
+$(WORK_DIR)/verilator/Vtb_top: $(WORK_DIR)/verilator/Vtb_top.mk
+	$(MAKE) -C $(WORK_DIR)/verilator -f Vtb_top.mk OPT_FAST="-O3"
+
+# Generation rules
+$(TEST_DIR)/program.s: | $(TEST_DIR)
+	LD_LIBRARY_PATH=$(RISCV_DV_PATH)/euvm/build $(RISCV_DV_PATH)/euvm/build/riscv_instr_gen \
+		+UVM_VERBOSITY=$(UVM_VERBOSITY) +random_seed=$(SEED) +instr_cnt=$(INSTR_COUNT) \
+		+asm_file_name=$@ \
+		+UVM_TESTNAME=riscv.test.riscv_instr_base_test.riscv_instr_base_test
+	mv $@_0.S $@
+	touch $(TEST_DIR)/user_define.h
+	touch $(TEST_DIR)/user_init.s
+
+# Compilation rules
+$(TEST_DIR)/program.elf: $(TEST_DIR)/program.s $(LDSCRIPT)
+	$(GCC_PREFIX)-gcc -march=$(ARCH) -mabi=$(ABI) -O0 -T$(LDSCRIPT) -I$(TEST_DIR) -static -nostdlib -nostartfiles $< -o $@
+	$(GCC_PREFIX)-objdump -S $@ > $(basename $@).lst
+
+$(TEST_DIR)/program.hex: $(TEST_DIR)/program.elf
+	$(GCC_PREFIX)-objcopy -O verilog $< $@
+
+# Run the ISS simulation
+$(TEST_DIR)/iss_run.log: $(TEST_DIR)/program.elf
+	spike -m0x2000:0x100000 --isa=$(ARCH) --log=$@ -l $<
+
+# Run the HDL simulation
+$(TEST_DIR)/hdl_run.log: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
+	cd $(TEST_DIR) && $(abspath $(WORK_DIR)/verilator/Vtb_top) 2>&1 | tee hdl_run.log
+
+run: $(TEST_DIR)/run.log
+
+clean:
+	rm -rf $(WORK_DIR)
+
+.PHONY: clean run
diff --git a/tools/riscv-dv/link.ld b/tools/riscv-dv/link.ld
new file mode 100644
index 00000000000..b179f117996
--- /dev/null
+++ b/tools/riscv-dv/link.ld
@@ -0,0 +1,18 @@
+OUTPUT_ARCH( "riscv" )
+ENTRY(_start)
+
+SECTIONS
+{
+  . = 0x2000;
+  .text : { *(.text) }
+  . = ALIGN(0x1000);
+  .tohost : { *(.tohost) }
+  . = ALIGN(0x1000);
+  .page_table : { *(.page_table) }
+  .data : { *(.data) }
+  .user_stack : { *(.user_stack) }
+  .kernel_data : { *(.kernel_data) }
+  .kernel_stack : { *(.kernel_stack) }
+  .bss : { *(.bss) }
+  _end = .;
+}

From 39730deb38c95e248987d2e78fd7e43b5e455700 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Thu, 6 Apr 2023 15:46:50 +0200
Subject: [PATCH 05/34] End-to-end RISC-V DV flow, currently reports execution
 mismatch.

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile                 | 19 ++++--
 tools/riscv-dv/veer_log_to_trace_csv.py | 79 +++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 6 deletions(-)
 create mode 100644 tools/riscv-dv/veer_log_to_trace_csv.py

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index 32884ca4acc..cceb97246a9 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -20,7 +20,6 @@ VERILATOR_EXE   = $(RV_ROOT)/testbench/test_tb_top.cpp
 GCC_PREFIX      = riscv64-unknown-elf
 ARCH            = rv32imc
 ABI             = ilp32
-#LDSCRIPT        = $(RV_ROOT)/testbench/link.ld
 LDSCRIPT        = link.ld
 
 HDL_FILES = $(WORK_DIR)/common_defines.vh \
@@ -70,16 +69,24 @@ $(TEST_DIR)/program.hex: $(TEST_DIR)/program.elf
 	$(GCC_PREFIX)-objcopy -O verilog $< $@
 
 # Run the ISS simulation
-$(TEST_DIR)/iss_run.log: $(TEST_DIR)/program.elf
-	spike -m0x2000:0x100000 --isa=$(ARCH) --log=$@ -l $<
+$(TEST_DIR)/iss_trace.csv: $(TEST_DIR)/program.elf
+	spike -m0x2000:0x100000 --isa=$(ARCH) --log=$(TEST_DIR)/iss_run.log --log-commits -l $<
+	python3 $(RISCV_DV_PATH)/scripts/spike_log_to_trace_csv.py --log $(TEST_DIR)/iss_run.log --csv $@
 
 # Run the HDL simulation
-$(TEST_DIR)/hdl_run.log: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
+$(TEST_DIR)/hdl_trace.csv: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
 	cd $(TEST_DIR) && $(abspath $(WORK_DIR)/verilator/Vtb_top) 2>&1 | tee hdl_run.log
+	python3 veer_log_to_trace_csv.py --log $(TEST_DIR)/exec.log --csv $@
 
-run: $(TEST_DIR)/run.log
+compare: $(TEST_DIR)/iss_trace.csv $(TEST_DIR)/hdl_trace.csv
+	rm -rf compare.log
+	python3 $(RISCV_DV_PATH)/scripts/instr_trace_compare.py \
+		--csv_file_1 $(TEST_DIR)/iss_trace.csv --csv_name_1 ISS --csv_file_2 $(TEST_DIR)/hdl_trace.csv --csv_name_2 HDL \
+		--in_order_mode 1 --log compare.log --verbose 10 --mismatch_print_limit 20
+
+run: $(TEST_DIR)/iss_trace.csv $(TEST_DIR)/hdl_trace.csv
 
 clean:
 	rm -rf $(WORK_DIR)
 
-.PHONY: clean run
+.PHONY: clean run compare
diff --git a/tools/riscv-dv/veer_log_to_trace_csv.py b/tools/riscv-dv/veer_log_to_trace_csv.py
new file mode 100644
index 00000000000..e7dd91377c1
--- /dev/null
+++ b/tools/riscv-dv/veer_log_to_trace_csv.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+import argparse
+import re
+
+# =============================================================================
+
+INSTR_RE = re.compile(r"^\s*(?P<cyc>[0-9]+)\s+:\s+#(?P<inst>[0-9]+)\s+0\s+(?P<pc>[0-9a-f]+)\s+(?P<opc>[0-9a-f]+)\s+(?P<reg>[^=;]+)=(?P<val>[0-9a-f]+)\s+;\s+(?P<mnemonic>.*)")
+
+def parse_log(file_name):
+
+    # Read the log
+    with open(file_name, "r") as fp:
+        lines = fp.readlines()
+
+    # Parse
+    data = []
+    for line in lines:
+
+        line  = line.strip()
+        match = INSTR_RE.match(line)
+        if match is not None:
+
+            entry = (
+                match.group("pc"),
+                "",
+                "{}:{}".format(match.group("reg"), match.group("val")),
+                "",
+                match.group("opc"),
+                "0", # TODO
+                match.group("mnemonic"),
+                "",
+                "",
+            )
+
+            data.append(entry)
+
+    return data
+
+def write_csv(file_name, data):
+
+    # Add header
+    lines = ["pc,instr,gpr,csr,binary,mode,instr_str,operand,pad"]
+
+    # Add data
+    lines.extend([",".join(fields) for fields in data])
+
+    # Write
+    with open(file_name, "w") as fp:
+        for line in lines:
+            fp.write(line + "\n")
+
+# =============================================================================
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--log",
+        type=str,
+        required=True,
+        help="HDL simulation trace log"
+    )
+    parser.add_argument(
+        "--csv",
+        type=str,
+        required=True,
+        help="Output CSV file"
+    )
+
+    args = parser.parse_args()
+
+    # Parse log
+    data = parse_log(args.log)
+
+    # Write CSV
+    write_csv(args.csv, data)
+
+if __name__ == "__main__":
+    main()

From ce38b5d212cd95be00cfc0acc4f2d6add6748b62 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Fri, 7 Apr 2023 12:18:20 +0200
Subject: [PATCH 06/34] Reworked VeeR execution trace parser

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile                 |   2 +-
 tools/riscv-dv/veer_log_to_trace_csv.py | 160 ++++++++++++++++++++----
 2 files changed, 138 insertions(+), 24 deletions(-)

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index cceb97246a9..2e1d93b7219 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -76,7 +76,7 @@ $(TEST_DIR)/iss_trace.csv: $(TEST_DIR)/program.elf
 # Run the HDL simulation
 $(TEST_DIR)/hdl_trace.csv: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
 	cd $(TEST_DIR) && $(abspath $(WORK_DIR)/verilator/Vtb_top) 2>&1 | tee hdl_run.log
-	python3 veer_log_to_trace_csv.py --log $(TEST_DIR)/exec.log --csv $@
+	PYTHONPATH=$(RISCV_DV_PATH)/scripts python3 veer_log_to_trace_csv.py --log $(TEST_DIR)/exec.log --csv $@
 
 compare: $(TEST_DIR)/iss_trace.csv $(TEST_DIR)/hdl_trace.csv
 	rm -rf compare.log
diff --git a/tools/riscv-dv/veer_log_to_trace_csv.py b/tools/riscv-dv/veer_log_to_trace_csv.py
index e7dd91377c1..9f876796fde 100644
--- a/tools/riscv-dv/veer_log_to_trace_csv.py
+++ b/tools/riscv-dv/veer_log_to_trace_csv.py
@@ -2,52 +2,166 @@
 import argparse
 import re
 
+from riscv_trace_csv import RiscvInstructionTraceEntry, RiscvInstructionTraceCsv
+
 # =============================================================================
 
-INSTR_RE = re.compile(r"^\s*(?P<cyc>[0-9]+)\s+:\s+#(?P<inst>[0-9]+)\s+0\s+(?P<pc>[0-9a-f]+)\s+(?P<opc>[0-9a-f]+)\s+(?P<reg>[^=;]+)=(?P<val>[0-9a-f]+)\s+;\s+(?P<mnemonic>.*)")
+INSTR_RE = re.compile(r"^\s*(?P<cyc>[0-9]+)\s+:\s+#(?P<inst>[0-9]+)\s+0\s+"
+                      r"(?P<pc>[0-9a-f]+)\s+(?P<opc>[0-9a-f]+)\s+"
+                      r"((?P<reg>[^=;]+)=(?P<val>[0-9a-f]+))?"
+                      r"\s+;\s+(?P<mnemonic>.*)")
+
+NB_RE    = re.compile(r"^\s*(?P<cyc>[0-9]+)\s+:\s+"
+                      r"(?P<reg>[^=;]+)=(?P<val>[0-9a-f]+)"
+                      r"\s+;\s+(?P<mnemonic>(nbL|nbD))")
+
+# =============================================================================
 
 def parse_log(file_name):
+    """
+    Parses VeeR-EL2 execution log generated by HDL simulation.
+
+    The core is in-order however, due to pipelined implementation certain
+    instructions may have an effect in different clock cycle than they are
+    executed. The testbench trace handes this by emitting special "nbL" and
+    "nbD" entries which need to be correlated with the actual instruction.
+
+    Most of the logic of this parser does exactly that. Every trace entry is
+    put into a temporary queue. Whenever a "nbL"/"nbD" is encountered, the
+    queue is searched for a matching counterpart. This happens in the opposite
+    way as well eg. when a "div" is encountered the queue is searched for "nbD"
+    Once an entry is found, relevant data is filled in.
+
+    Entires are poped of the queue only when they contain all the information
+    for the complete trace.
+    """
 
     # Read the log
     with open(file_name, "r") as fp:
         lines = fp.readlines()
 
-    # Parse
-    data = []
-    for line in lines:
+    data  = []
+    queue = []
 
+    for line in lines:
         line  = line.strip()
+
+        # Instruction
         match = INSTR_RE.match(line)
         if match is not None:
+            groups = match.groupdict()
+
+            gpr = None
+            if groups["reg"] and groups["val"]:
+                gpr = ("{}:{}".format(groups["reg"], groups["val"]))
+
+            fields   = groups["mnemonic"].split()
+            mnemonic = fields[0]
+            operands = fields[1].split(",") if len(fields) > 1 else []
+
+            entry = None
+
+            # Delayed effect, search the queue
+            if gpr is None and mnemonic in ["lw", "div", "divu", "rem", "remu"]:
+                for ent in reversed(queue):
+
+                    if (ent.operand == "nbL" and mnemonic in ["lw"]) or \
+                       (ent.operand == "nbD" and mnemonic in ["div", "divu", "rem", "remu"]):
+
+                        assert len(operands), line
+                        assert len(ent.gpr),  ent.get_trace_string()
+
+                        reg, val = ent.gpr[0].split(":") # FIXME: Assuming single GPR
+                        if reg == operands[0]:
+                            entry = ent
+                            break
+
+            # Enqueue or not
+            enqueue = entry is None and (gpr is not None or mnemonic in \
+                                         ["div", "divu", "rem", "remu", "lw"])
 
-            entry = (
-                match.group("pc"),
-                "",
-                "{}:{}".format(match.group("reg"), match.group("val")),
-                "",
-                match.group("opc"),
-                "0", # TODO
-                match.group("mnemonic"),
-                "",
-                "",
-            )
+            # Entry not found in the queue, create it
+            if not entry:
+                entry = RiscvInstructionTraceEntry()
 
+            # Fill data
+            entry.pc        = groups["pc"]
+            entry.binary    = groups["opc"]
+            entry.operand   = groups["mnemonic"]
+            entry.mode      = "0" # TODO
+
+            # Append GPR if any
+            if gpr:
+                entry.gpr.append(gpr)
+
+            # Enqueue
+            if enqueue:
+                queue.append(entry)
+
+        # nbL / nbD
+        match = NB_RE.match(line)
+        if match is not None:
+            groups = match.groupdict()
+
+            assert groups["reg"] and groups["val"], line
+            gpr = ("{}:{}".format(groups["reg"], groups["val"]))
+
+            # Find an existing nbL/nbD entry in the queue. Match destination GPR
+            for entry in reversed(queue):
+
+                fields   = entry.operand.split()
+                mnemonic = fields[0]
+                operands = fields[1].split(",") if len(fields) > 1 else []
+
+                if (groups["mnemonic"] == "nbL" and mnemonic in ["lw"]) or \
+                   (groups["mnemonic"] == "nbD" and mnemonic in ["div", "divu", "rem", "remu"]):
+                    assert len(operands), entry
+                    if groups["reg"] == operands[0]:
+                        entry.gpr.append(gpr)
+                        break
+
+            # Add a new entry
+            else:
+                entry = RiscvInstructionTraceEntry()
+                entry.operand = groups["mnemonic"]
+                entry.gpr.append(gpr)
+
+                queue.append(entry)
+
+        # Dequeue entries that have all they need. Stop at the first one which
+        # is missing something.
+        while len(queue):
+            entry = queue[0]
+
+            if not entry.pc or not entry.gpr:
+                break
+
+            # Pop
             data.append(entry)
+            queue = queue[1:]
+
+        # Safeguard
+        if len(queue) >= 10:
+            print("ERROR: Malformed trace, the queue grew too much")
+            for entry in reversed(queue):
+                print("", entry.get_trace_string())
+            assert False
 
     return data
 
+
 def write_csv(file_name, data):
+    """
+    Writes the trace to CSV
+    """
 
-    # Add header
-    lines = ["pc,instr,gpr,csr,binary,mode,instr_str,operand,pad"]
+    with open(file_name, "w") as fp:
 
-    # Add data
-    lines.extend([",".join(fields) for fields in data])
+        writer = RiscvInstructionTraceCsv(fp)
+        writer.start_new_trace()
 
-    # Write
-    with open(file_name, "w") as fp:
-        for line in lines:
-            fp.write(line + "\n")
+        for entry in data:
+            writer.write_trace_entry(entry)
 
 # =============================================================================
 

From 4c216292db71921f3ca9c71aa08649af27c0eb7c Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Tue, 11 Apr 2023 16:00:02 +0200
Subject: [PATCH 07/34] Created a temporary assembly code fixup script for VeeR
 and integrated it with the Makefile flow

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile      |  32 ++++++++---
 tools/riscv-dv/code_fixup.py | 104 +++++++++++++++++++++++++++++++++++
 2 files changed, 127 insertions(+), 9 deletions(-)
 create mode 100644 tools/riscv-dv/code_fixup.py

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index 2e1d93b7219..caec14ebb4c 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -51,12 +51,20 @@ $(WORK_DIR)/verilator/Vtb_top: $(WORK_DIR)/verilator/Vtb_top.mk
 	$(MAKE) -C $(WORK_DIR)/verilator -f Vtb_top.mk OPT_FAST="-O3"
 
 # Generation rules
-$(TEST_DIR)/program.s: | $(TEST_DIR)
+$(TEST_DIR)/program.s: code_fixup.py | $(TEST_DIR)
 	LD_LIBRARY_PATH=$(RISCV_DV_PATH)/euvm/build $(RISCV_DV_PATH)/euvm/build/riscv_instr_gen \
 		+UVM_VERBOSITY=$(UVM_VERBOSITY) +random_seed=$(SEED) +instr_cnt=$(INSTR_COUNT) \
 		+asm_file_name=$@ \
-		+UVM_TESTNAME=riscv.test.riscv_instr_base_test.riscv_instr_base_test
-	mv $@_0.S $@
+		+UVM_TESTNAME=riscv.test.riscv_instr_base_test.riscv_instr_base_test \
+		+num_of_tests=1 \
+		+start_idx=0 \
+		+num_of_sub_programs=0 \
+		+instr_cnt=10 \
+		+no_branch_jump=true \
+		+no_csr_instr=true \
+		+no_fence=true \
+		+no_data_page=true
+	python3 code_fixup.py -i $@_0.S -o $@
 	touch $(TEST_DIR)/user_define.h
 	touch $(TEST_DIR)/user_init.s
 
@@ -64,19 +72,25 @@ $(TEST_DIR)/program.s: | $(TEST_DIR)
 $(TEST_DIR)/program.elf: $(TEST_DIR)/program.s $(LDSCRIPT)
 	$(GCC_PREFIX)-gcc -march=$(ARCH) -mabi=$(ABI) -O0 -T$(LDSCRIPT) -I$(TEST_DIR) -static -nostdlib -nostartfiles $< -o $@
 	$(GCC_PREFIX)-objdump -S $@ > $(basename $@).lst
+	$(GCC_PREFIX)-nm -n $@ > $(basename $@).sym
 
 $(TEST_DIR)/program.hex: $(TEST_DIR)/program.elf
 	$(GCC_PREFIX)-objcopy -O verilog $< $@
 
 # Run the ISS simulation
-$(TEST_DIR)/iss_trace.csv: $(TEST_DIR)/program.elf
-	spike -m0x2000:0x100000 --isa=$(ARCH) --log=$(TEST_DIR)/iss_run.log --log-commits -l $<
-	python3 $(RISCV_DV_PATH)/scripts/spike_log_to_trace_csv.py --log $(TEST_DIR)/iss_run.log --csv $@
+$(TEST_DIR)/iss_trace.log: $(TEST_DIR)/program.elf
+	timeout --foreground 10s spike -m0x2000:0x100000 --isa=$(ARCH) --misaligned --log=$@ --log-commits -l $< || true
+
+$(TEST_DIR)/iss_trace.csv: $(TEST_DIR)/iss_trace.log
+	python3 $(RISCV_DV_PATH)/scripts/spike_log_to_trace_csv.py --log $< --csv $@
 
 # Run the HDL simulation
-$(TEST_DIR)/hdl_trace.csv: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
-	cd $(TEST_DIR) && $(abspath $(WORK_DIR)/verilator/Vtb_top) 2>&1 | tee hdl_run.log
-	PYTHONPATH=$(RISCV_DV_PATH)/scripts python3 veer_log_to_trace_csv.py --log $(TEST_DIR)/exec.log --csv $@
+$(TEST_DIR)/hdl_trace.log: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
+	cd $(TEST_DIR) && $(abspath $(WORK_DIR)/verilator/Vtb_top)
+	mv $(TEST_DIR)/exec.log $@
+
+$(TEST_DIR)/hdl_trace.csv: veer_log_to_trace_csv.py $(TEST_DIR)/hdl_trace.log
+	PYTHONPATH=$(RISCV_DV_PATH)/scripts python3 veer_log_to_trace_csv.py --log $< --csv $@
 
 compare: $(TEST_DIR)/iss_trace.csv $(TEST_DIR)/hdl_trace.csv
 	rm -rf compare.log
diff --git a/tools/riscv-dv/code_fixup.py b/tools/riscv-dv/code_fixup.py
new file mode 100644
index 00000000000..d288ddda143
--- /dev/null
+++ b/tools/riscv-dv/code_fixup.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+import argparse
+import re
+
+# =============================================================================
+
+class AssemblyLine:
+    """
+    Simple assembly line representation
+    """
+
+    RE_INSTR = re.compile(r"(?P<mnemonic>\S+)\s+(?P<operands>.*)")
+
+    def __init__(self, text):
+        self.text       = text
+        self.mnemonic   = None
+        self.operands   = None
+
+        # Strip label if any
+        if ":" in text:
+            text = text.split(":", maxsplit=1)[1]
+
+        # Strip comment if any
+        if "#" in text:
+            text = text.split("#", maxsplit=1)[0]
+
+        # Get instruction and operands
+        m = self.RE_INSTR.match(text.strip())
+        if m is not None:
+
+            if m.group("mnemonic")[0] == ".":
+                return
+
+            self.mnemonic = m.group("mnemonic").lower()
+            self.operands = [op.strip() for op in m.group("operands").split()]
+
+    def __str__(self):
+        return self.text
+
+# =============================================================================
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-i",
+        type=str,
+        required=True,
+        help="Input assembly file"
+    )
+    parser.add_argument(
+        "-o",
+        type=str,
+        required=True,
+        help="Output assembly file"
+    )
+
+    args = parser.parse_args()
+
+    # Read and parse
+    with open(args.i, "r") as fp:
+        inp_lines = [AssemblyLine(l) for l in fp.readlines()]
+
+    # Identify a delayed write instruction followed by another one which writes
+    # to the same register
+    out_lines = []
+    for i in range(len(inp_lines)):
+        line = inp_lines[i]
+        out_lines.append(line)
+
+        # Bypass
+        if not line.mnemonic:
+            continue
+
+        # Check if it is a delayed write. If not then bypass
+        is_delayed = line.mnemonic in ["div", "divu", "rem", "remu", "lw"]
+        if not is_delayed:
+            continue
+
+        # Get next 2 instructions
+        following = []
+        for j in range(i+1, len(inp_lines)):
+            if inp_lines[j].mnemonic is not None:
+                following.append(inp_lines[j])
+                if len(following) >= 2:
+                    break
+
+        # If any of the instructions targets the same register insert NOPs
+        dst = line.operands[0]
+        for j, l in enumerate(following):
+            if l.operands and l.operands[0] == dst:
+                nops = max(0, 2 - j)
+                for _ in range(nops):
+                    out_lines.append(" " * 18 + "nop # FIXME: A fixup not to make VeeR cancel a delayed write\n")
+                break
+
+    # Write
+    with open(args.o, "w") as fp:
+        for l in out_lines:
+            fp.write(str(l))
+
+
+if __name__ == "__main__":
+    main()

From 6776f1e533cb6e64e4e7915ab7d264feb8531f06 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Tue, 11 Apr 2023 16:29:21 +0200
Subject: [PATCH 08/34] Updated ld script

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/link.ld | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/riscv-dv/link.ld b/tools/riscv-dv/link.ld
index b179f117996..1f4a0fb6a32 100644
--- a/tools/riscv-dv/link.ld
+++ b/tools/riscv-dv/link.ld
@@ -9,10 +9,12 @@ SECTIONS
   .tohost : { *(.tohost) }
   . = ALIGN(0x1000);
   .page_table : { *(.page_table) }
+  _end = .;
+
+  . = 0x10000;
   .data : { *(.data) }
   .user_stack : { *(.user_stack) }
   .kernel_data : { *(.kernel_data) }
   .kernel_stack : { *(.kernel_stack) }
   .bss : { *(.bss) }
-  _end = .;
 }

From 25f4fb43ea185546cd51acf6619e32002850c02b Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Tue, 11 Apr 2023 16:29:33 +0200
Subject: [PATCH 09/34] VeeR execution log parser terminates on first
 occurrence of ecall instruction

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/veer_log_to_trace_csv.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tools/riscv-dv/veer_log_to_trace_csv.py b/tools/riscv-dv/veer_log_to_trace_csv.py
index 9f876796fde..a2241b398d7 100644
--- a/tools/riscv-dv/veer_log_to_trace_csv.py
+++ b/tools/riscv-dv/veer_log_to_trace_csv.py
@@ -61,6 +61,10 @@ def parse_log(file_name):
 
             entry = None
 
+            # Stop on ecall
+            if mnemonic == "ecall":
+                break
+
             # Delayed effect, search the queue
             if gpr is None and mnemonic in ["lw", "div", "divu", "rem", "remu"]:
                 for ent in reversed(queue):

From 6e9a3a93f2ea1d3ff312bdd5d10d2f6c8680514a Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Tue, 11 Apr 2023 16:30:22 +0200
Subject: [PATCH 10/34] Finalized the makefile flow

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index caec14ebb4c..1ec2e78164e 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -10,7 +10,7 @@ INSTR_COUNT     = 100
 
 VEER_TARGET     = default
 VEER_CONF       = -set build_axi4 \
-                  -set reset_vec=0x2000
+                  -set reset_vec=0x2000 # Needed for spike
 
 VERILATOR       = verilator
 VERILATOR_CFLAGS= "-std=c++11"
@@ -28,6 +28,8 @@ HDL_FILES = $(WORK_DIR)/common_defines.vh \
             $(RV_ROOT)/testbench/ahb_sif.sv \
             $(RV_ROOT)/design/include/el2_def.sv
 
+all: compare
+
 # Directory rules
 $(WORK_DIR):
 	mkdir -p $@
@@ -89,18 +91,22 @@ $(TEST_DIR)/hdl_trace.log: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
 	cd $(TEST_DIR) && $(abspath $(WORK_DIR)/verilator/Vtb_top)
 	mv $(TEST_DIR)/exec.log $@
 
-$(TEST_DIR)/hdl_trace.csv: veer_log_to_trace_csv.py $(TEST_DIR)/hdl_trace.log
+$(TEST_DIR)/hdl_trace.csv: $(TEST_DIR)/hdl_trace.log veer_log_to_trace_csv.py
 	PYTHONPATH=$(RISCV_DV_PATH)/scripts python3 veer_log_to_trace_csv.py --log $< --csv $@
 
-compare: $(TEST_DIR)/iss_trace.csv $(TEST_DIR)/hdl_trace.csv
-	rm -rf compare.log
+$(TEST_DIR)/comparison.log: $(TEST_DIR)/iss_trace.csv $(TEST_DIR)/hdl_trace.csv
+	rm -rf $@
 	python3 $(RISCV_DV_PATH)/scripts/instr_trace_compare.py \
 		--csv_file_1 $(TEST_DIR)/iss_trace.csv --csv_name_1 ISS --csv_file_2 $(TEST_DIR)/hdl_trace.csv --csv_name_2 HDL \
-		--in_order_mode 1 --log compare.log --verbose 10 --mismatch_print_limit 20
+		--in_order_mode 1 --log $@ --verbose 10 --mismatch_print_limit 20
 
-run: $(TEST_DIR)/iss_trace.csv $(TEST_DIR)/hdl_trace.csv
+compare: $(TEST_DIR)/comparison.log
+	@grep "\[PASSED\]" $<
 
 clean:
+	rm -rf $(TEST_DIR)
+
+fullclean:
 	rm -rf $(WORK_DIR)
 
-.PHONY: clean run compare
+.PHONY: clean fullclean compare

From 2929272b36688de58dc2661eb476768a35b56e9f Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Wed, 12 Apr 2023 09:32:26 +0200
Subject: [PATCH 11/34] Added RISC-V DV as a submodule

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 .gitmodules          | 3 +++
 third_party/riscv-dv | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 third_party/riscv-dv

diff --git a/.gitmodules b/.gitmodules
index bbb77d9c518..bc3f4fcced0 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "third-party/picolibc"]
 	path = third_party/picolibc
 	url = https://github.com/picolibc/picolibc
+[submodule "third_party/riscv-dv"]
+	path = third_party/riscv-dv
+	url = https://github.com/antmicro/riscv-dv
diff --git a/third_party/riscv-dv b/third_party/riscv-dv
new file mode 160000
index 00000000000..1b75af1b123
--- /dev/null
+++ b/third_party/riscv-dv
@@ -0,0 +1 @@
+Subproject commit 1b75af1b12349f90e7c555f5f33e9441616e7651

From 6a64639e7e41ca669f7be0168fc5ea88f5604fed Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Wed, 12 Apr 2023 11:46:05 +0200
Subject: [PATCH 12/34] Changed to use RISC-V DV instruction generator
 available in the current path

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index 1ec2e78164e..fa8475361d4 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -54,7 +54,7 @@ $(WORK_DIR)/verilator/Vtb_top: $(WORK_DIR)/verilator/Vtb_top.mk
 
 # Generation rules
 $(TEST_DIR)/program.s: code_fixup.py | $(TEST_DIR)
-	LD_LIBRARY_PATH=$(RISCV_DV_PATH)/euvm/build $(RISCV_DV_PATH)/euvm/build/riscv_instr_gen \
+	riscv_instr_gen \
 		+UVM_VERBOSITY=$(UVM_VERBOSITY) +random_seed=$(SEED) +instr_cnt=$(INSTR_COUNT) \
 		+asm_file_name=$@ \
 		+UVM_TESTNAME=riscv.test.riscv_instr_base_test.riscv_instr_base_test \

From be92563c6c6173e82c6b6e783a90a52d2a4ccd41 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Wed, 12 Apr 2023 11:02:20 +0200
Subject: [PATCH 13/34] Initial RISC-V DV CI

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 .github/workflows/riscv-dv.yml | 252 +++++++++++++++++++++++++++++++++
 1 file changed, 252 insertions(+)
 create mode 100644 .github/workflows/riscv-dv.yml

diff --git a/.github/workflows/riscv-dv.yml b/.github/workflows/riscv-dv.yml
new file mode 100644
index 00000000000..7e63130df9d
--- /dev/null
+++ b/.github/workflows/riscv-dv.yml
@@ -0,0 +1,252 @@
+name: RISC-V DV tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+
+  verilator:
+    name: Build Verilator
+    runs-on: ubuntu-latest
+    env:
+      CCACHE_DIR: "/opt/veer-el2/.cache/"
+      DEBIAN_FRONTEND: "noninteractive"
+
+    steps:
+      - name: Install prerequisities
+        run: |
+          sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
+            git autoconf automake autotools-dev curl python3 python3-pip \
+            libmpc-dev libmpfr-dev libgmp-dev gawk build-essential bison flex \
+            texinfo gperf libtool patchutils bc zlib1g zlib1g-dev libexpat-dev \
+            ninja-build ccache libfl2 libfl-dev
+
+      - name: Create Cache Timestamp
+        id: cache_timestamp
+        uses: nanzm/get-time-action@v1.1
+        with:
+          format: 'YYYY-MM-DD-HH-mm-ss'
+
+      - name: Setup cache
+        uses: actions/cache@v2
+        timeout-minutes: 3
+        continue-on-error: true
+        with:
+          path: "/opt/veer-el2/.cache/"
+          key: cache_verilator_${{ steps.cache_timestamp.outputs.time }}
+          restore-keys: cache_verilator_
+
+      - name: Build Verilator
+        run: |
+          git clone https://github.com/verilator/verilator
+          pushd verilator
+            git checkout v5.002
+            autoconf
+            ./configure --prefix=/opt/verilator
+            make -j `nproc`
+            make install
+          popd
+          cd /opt && tar -czvf verilator.tar.gz verilator/
+
+      - name: Store Verilator binaries
+        uses: actions/upload-artifact@v3
+        with:
+          name: verilator
+          path: /opt/*.tar.gz
+          retention-days: 1
+
+  spike:
+    name: Build Spike ISS
+    runs-on: ubuntu-latest
+    env:
+      CCACHE_DIR: "/opt/veer-el2/.cache/"
+      DEBIAN_FRONTEND: "noninteractive"
+
+    steps:
+      - name: Install prerequisities
+        run: |
+          sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
+            git build-essential cmake ccache device-tree-compiler
+
+      - name: Create Cache Timestamp
+        id: cache_timestamp
+        uses: nanzm/get-time-action@v1.1
+        with:
+          format: 'YYYY-MM-DD-HH-mm-ss'
+
+      - name: Setup cache
+        uses: actions/cache@v2
+        timeout-minutes: 3
+        continue-on-error: true
+        with:
+          path: "/opt/veer-el2/.cache/"
+          key: cache_spike_${{ steps.cache_timestamp.outputs.time }}
+          restore-keys: cache_spike_
+
+      - name: Build Spike
+        run: |
+          git clone https://github.com/riscv-software-src/riscv-isa-sim spike
+          export CC="ccache gcc"
+          export CXX="ccache g++"
+          pushd spike
+            git checkout d70ea67d
+            mkdir build
+            cd build
+            ../configure --prefix=/opt/spike
+            make -j`nproc`
+            make install
+          popd
+          rm -rf /opt/spike/include # Remove include and lib to save space
+          rm -rf /opt/spike/lib
+          cd /opt && tar -czvf spike.tar.gz spike/
+
+      - name: Store Spike binaries
+        uses: actions/upload-artifact@v3
+        with:
+          name: spike
+          path: /opt/*.tar.gz
+          retention-days: 1
+
+  instrgen:
+    name: Build RISC-V DV instruction generator
+    runs-on: ubuntu-latest
+    env:
+      DEBIAN_FRONTEND: "noninteractive"
+
+    steps:
+      - name: Install prerequisities
+        run: |
+          sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
+            git build-essential
+
+      - name: Create Cache Timestamp
+        id: cache_timestamp
+        uses: nanzm/get-time-action@v1.1
+        with:
+          format: 'YYYY-MM-DD-HH-mm-ss'
+
+      - name: Setup cache
+        uses: actions/cache@v2
+        timeout-minutes: 3
+        continue-on-error: true
+        with:
+          path: "/opt/veer-el2/.cache/"
+          key: cache_instrgen_${{ steps.cache_timestamp.outputs.time }}
+          restore-keys: cache_instrgen_
+
+      - name: Download eUVM
+        run: |
+          mkdir -p /opt
+          pushd /opt
+            wget https://github.com/coverify/euvm/releases/download/v1.0-beta25/euvm-1.0-beta25.tar.xz
+            tar -Jxf euvm-1.0-beta25.tar.xz
+            mv euvm-1.0-beta25 euvm
+          popd
+
+      - name: Build
+        run: |
+          # TODO: Use upstream once all the relevant changes for RV32IMC land
+          git clone https://github.com/antmicro/riscv-dv -b master+fixes
+          export PATH=/opt/euvm/bin:$PATH
+          mkdir -p /opt/instrgen
+          pushd riscv-dv/euvm/build
+            # FIXME: ldc2 fails to create directories when copying cached object files. Do this manually upfront
+            find ../riscv -type d | sed 's/../picobj-ldc/' | xargs mkdir -p
+            make -j`nproc` TARGET=RV32IMC LDC2="ldc2 --cache=/opt/veer-el2/.cache"
+            cp *.so /opt/instrgen
+            cp riscv_instr_gen /opt/instrgen
+          popd
+          # Bundle LDC shard libs with the instruction generator
+          rsync -a /opt/euvm/lib/*-ldc-shared.so* /opt/instrgen/
+          cd /opt && tar -czvf instrgen.tar.gz instrgen/
+
+      - name: Store binaries
+        uses: actions/upload-artifact@v3
+        with:
+          name: instrgen
+          path: /opt/*.tar.gz
+          retention-days: 1
+
+  # TODO: Parallelize this job for multiple tests
+  tests:
+    name: Run RISC-V DV tests
+    runs-on: ubuntu-latest
+    needs: [verilator, spike, instrgen]
+    env:
+      DEBIAN_FRONTEND: "noninteractive"
+
+    steps:
+
+      - name: Install utils
+        run: |
+          sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
+            git cpanminus ccache device-tree-compiler python3-minimal python3-pip \
+            gcc-riscv64-unknown-elf
+          sudo cpanm Bit::Vector
+
+      - name: Download Verilator binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: verilator
+          path: /opt
+
+      - name: Download Spike binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: spike
+          path: /opt
+
+      - name: Download instruction generator binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: instrgen
+          path: /opt
+
+      - name: Unpack binaries
+        run: |
+          pushd /opt
+            tar -zxvf verilator.tar.gz 
+            tar -zxvf spike.tar.gz 
+            tar -zxvf instrgen.tar.gz 
+          popd
+
+      - name: Setup repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          path: veer
+
+      - name: Create Cache Timestamp
+        id: cache_timestamp
+        uses: nanzm/get-time-action@v1.1
+        with:
+          format: 'YYYY-MM-DD-HH-mm-ss'
+
+      - name: Setup cache
+        uses: actions/cache@v2
+        timeout-minutes: 3
+        continue-on-error: true
+        with:
+          path: "/opt/veer-el2/.cache/"
+          key: cache_tests_${{ steps.cache_timestamp.outputs.time }}
+          restore-keys: cache_tests_
+
+      - name: Run test
+        run: |
+          export PATH=/opt/verilator/bin:$PATH
+          export PATH=/opt/spike/bin:$PATH
+          export PATH=/opt/instrgen:$PATH
+          export LD_LIBRARY_PATH=/opt/instrgen:$LD_LIBRARY_PATH
+          export RV_ROOT=`realpath veer`
+
+          pushd ${RV_ROOT}
+            cd tools/riscv-dv && make -j`nproc`
+          popd
+
+      - name: Pack artifacts
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: riscv-dv-results
+          path: veer/tools/riscv-dv/work/test_*

From 2bed74b20f065e50ba60e5821de8b69a72c51801 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Fri, 14 Apr 2023 09:56:26 +0200
Subject: [PATCH 14/34] Bumped RISC-V DV submodule

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 third_party/riscv-dv | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/riscv-dv b/third_party/riscv-dv
index 1b75af1b123..d30ac5eb8c5 160000
--- a/third_party/riscv-dv
+++ b/third_party/riscv-dv
@@ -1 +1 @@
-Subproject commit 1b75af1b12349f90e7c555f5f33e9441616e7651
+Subproject commit d30ac5eb8c5b523c54314716b7f578d6e859a130

From 14fc19b0d30751b04c25fa2883c09ea160a9181c Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Fri, 14 Apr 2023 09:27:34 +0200
Subject: [PATCH 15/34] Added dumping CSR writes to the execution log from
 testbench

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 testbench/tb_top.sv | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/testbench/tb_top.sv b/testbench/tb_top.sv
index b8e31d40a30..a9665282064 100644
--- a/testbench/tb_top.sv
+++ b/testbench/tb_top.sv
@@ -105,6 +105,10 @@ module tb_top ( input bit core_clk );
     logic [4:0]                 wb_dest;
     logic [31:0]                wb_data;
 
+    logic                       wb_csr_valid;
+    logic [11:0]                wb_csr_dest;
+    logic [31:0]                wb_csr_data;
+
 `ifdef RV_BUILD_AXI4
    //-------------------------- LSU AXI signals--------------------------
    // AXI Write Channels
@@ -350,9 +354,12 @@ module tb_top ( input bit core_clk );
 
     // trace monitor
     always @(posedge core_clk) begin
-        wb_valid  <= `DEC.dec_i0_wen_r;
-        wb_dest   <= `DEC.dec_i0_waddr_r;
-        wb_data   <= `DEC.dec_i0_wdata_r;
+        wb_valid      <= `DEC.dec_i0_wen_r;
+        wb_dest       <= `DEC.dec_i0_waddr_r;
+        wb_data       <= `DEC.dec_i0_wdata_r;
+        wb_csr_valid  <= `DEC.dec_csr_wen_r;
+        wb_csr_dest   <= `DEC.dec_csr_wraddr_r;
+        wb_csr_data   <= `DEC.dec_csr_wrdata_r;
         if (trace_rv_i_valid_ip) begin
            $fwrite(tp,"%b,%h,%h,%0h,%0h,3,%b,%h,%h,%b\n", trace_rv_i_valid_ip, 0, trace_rv_i_address_ip,
                   0, trace_rv_i_insn_ip,trace_rv_i_exception_ip,trace_rv_i_ecause_ip,
@@ -360,18 +367,19 @@ module tb_top ( input bit core_clk );
            // Basic trace - no exception register updates
            // #1 0 ee000000 b0201073 c 0b02       00000000
            commit_count++;
-           $fwrite (el, "%10d : %8s 0 %h %h%13s ; %s\n", cycleCnt, $sformatf("#%0d",commit_count),
+           $fwrite (el, "%10d : %8s 0 %h %h%13s %14s ; %s\n", cycleCnt, $sformatf("#%0d",commit_count),
                         trace_rv_i_address_ip, trace_rv_i_insn_ip,
-                        (wb_dest !=0 && wb_valid)?  $sformatf("%s=%h", abi_reg[wb_dest], wb_data) : "             ",
+                        (wb_dest !=0 && wb_valid)?  $sformatf("%s=%h", abi_reg[wb_dest], wb_data) : "            ",
+                        (wb_csr_valid)? $sformatf("c%h=%h", wb_csr_dest, wb_csr_data) : "             ",
                         dasm(trace_rv_i_insn_ip, trace_rv_i_address_ip, wb_dest & {5{wb_valid}}, wb_data)
                    );
         end
         if(`DEC.dec_nonblock_load_wen) begin
-            $fwrite (el, "%10d : %32s=%h ; nbL\n", cycleCnt, abi_reg[`DEC.dec_nonblock_load_waddr], `DEC.lsu_nonblock_load_data);
+            $fwrite (el, "%10d : %32s=%h                ; nbL\n", cycleCnt, abi_reg[`DEC.dec_nonblock_load_waddr], `DEC.lsu_nonblock_load_data);
             tb_top.gpr[0][`DEC.dec_nonblock_load_waddr] = `DEC.lsu_nonblock_load_data;
         end
         if(`DEC.exu_div_wren) begin
-            $fwrite (el, "%10d : %32s=%h ; nbD\n", cycleCnt, abi_reg[`DEC.div_waddr_wb], `DEC.exu_div_result);
+            $fwrite (el, "%10d : %32s=%h                ; nbD\n", cycleCnt, abi_reg[`DEC.div_waddr_wb], `DEC.exu_div_result);
             tb_top.gpr[0][`DEC.div_waddr_wb] = `DEC.exu_div_result;
         end
     end
@@ -422,7 +430,7 @@ module tb_top ( input bit core_clk );
         $readmemh("program.hex",  imem.mem);
         tp = $fopen("trace_port.csv","w");
         el = $fopen("exec.log","w");
-        $fwrite (el, "//   Cycle : #inst    0    pc    opcode    reg=value   ; mnemonic\n");
+        $fwrite (el, "//   Cycle : #inst    0    pc    opcode    reg=value    csr=value     ; mnemonic\n");
         fd = $fopen("console.log","w");
         commit_count = 0;
         preload_dccm();

From b0c6db5f1406a4dc1e0d1accca13679beeb43cf9 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Thu, 13 Apr 2023 15:29:58 +0200
Subject: [PATCH 16/34] Support for parsing CSR write effects from VeeR
 testbench execution logs

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/veer_log_to_trace_csv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tools/riscv-dv/veer_log_to_trace_csv.py b/tools/riscv-dv/veer_log_to_trace_csv.py
index a2241b398d7..2e98a32c61c 100644
--- a/tools/riscv-dv/veer_log_to_trace_csv.py
+++ b/tools/riscv-dv/veer_log_to_trace_csv.py
@@ -8,7 +8,8 @@
 
 INSTR_RE = re.compile(r"^\s*(?P<cyc>[0-9]+)\s+:\s+#(?P<inst>[0-9]+)\s+0\s+"
                       r"(?P<pc>[0-9a-f]+)\s+(?P<opc>[0-9a-f]+)\s+"
-                      r"((?P<reg>[^=;]+)=(?P<val>[0-9a-f]+))?"
+                      r"((?P<reg>[^=;]+)=(?P<val>[0-9a-f]+))?\s+"
+                      r"((?P<csr>[^=;]+)=(?P<csr_val>[0-9a-f]+))?"
                       r"\s+;\s+(?P<mnemonic>.*)")
 
 NB_RE    = re.compile(r"^\s*(?P<cyc>[0-9]+)\s+:\s+"
@@ -52,8 +53,11 @@ def parse_log(file_name):
             groups = match.groupdict()
 
             gpr = None
+            csr = None
             if groups["reg"] and groups["val"]:
                 gpr = ("{}:{}".format(groups["reg"], groups["val"]))
+            if groups["csr"] and groups["csr_val"]:
+                csr = ("{}:{}".format(groups["csr"], groups["csr_val"]))
 
             fields   = groups["mnemonic"].split()
             mnemonic = fields[0]
@@ -97,6 +101,8 @@ def parse_log(file_name):
             # Append GPR if any
             if gpr:
                 entry.gpr.append(gpr)
+            if csr:
+                entry.csr.append(csr)
 
             # Enqueue
             if enqueue:

From f6e8c1f5f28bf8bf4f6d63de045f243a37fe73e7 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Thu, 13 Apr 2023 15:31:07 +0200
Subject: [PATCH 17/34] Added VeeR-ISS (a.k.a. whisper) to the RISC-V DV
 verification flow

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index fa8475361d4..ffca3aba58b 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -79,13 +79,20 @@ $(TEST_DIR)/program.elf: $(TEST_DIR)/program.s $(LDSCRIPT)
 $(TEST_DIR)/program.hex: $(TEST_DIR)/program.elf
 	$(GCC_PREFIX)-objcopy -O verilog $< $@
 
-# Run the ISS simulation
-$(TEST_DIR)/iss_trace.log: $(TEST_DIR)/program.elf
+# Run the Spike ISS simulation
+$(TEST_DIR)/spike_trace.log: $(TEST_DIR)/program.elf
 	timeout --foreground 10s spike -m0x2000:0x100000 --isa=$(ARCH) --misaligned --log=$@ --log-commits -l $< || true
 
-$(TEST_DIR)/iss_trace.csv: $(TEST_DIR)/iss_trace.log
+$(TEST_DIR)/spike_trace.csv: $(TEST_DIR)/spike_trace.log
 	python3 $(RISCV_DV_PATH)/scripts/spike_log_to_trace_csv.py --log $< --csv $@
 
+# Run the Whisper ISS simulation
+$(TEST_DIR)/whisper_trace.log: $(TEST_DIR)/program.elf
+	whisper --isa imc --xlen 32 -l -t $< > $@ || rm -rf $@
+
+$(TEST_DIR)/whisper_trace.csv: $(TEST_DIR)/whisper_trace.log
+	python3 $(RISCV_DV_PATH)/scripts/whisper_log_trace_csv.py --log $< --csv $@
+
 # Run the HDL simulation
 $(TEST_DIR)/hdl_trace.log: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
 	cd $(TEST_DIR) && $(abspath $(WORK_DIR)/verilator/Vtb_top)
@@ -94,14 +101,23 @@ $(TEST_DIR)/hdl_trace.log: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
 $(TEST_DIR)/hdl_trace.csv: $(TEST_DIR)/hdl_trace.log veer_log_to_trace_csv.py
 	PYTHONPATH=$(RISCV_DV_PATH)/scripts python3 veer_log_to_trace_csv.py --log $< --csv $@
 
-$(TEST_DIR)/comparison.log: $(TEST_DIR)/iss_trace.csv $(TEST_DIR)/hdl_trace.csv
+$(TEST_DIR)/spike_comparison.log: $(TEST_DIR)/spike_trace.csv $(TEST_DIR)/hdl_trace.csv
+	rm -rf $@
+	python3 $(RISCV_DV_PATH)/scripts/instr_trace_compare.py \
+		--csv_file_1 $(TEST_DIR)/spike_trace.csv --csv_name_1 ISS --csv_file_2 $(TEST_DIR)/hdl_trace.csv --csv_name_2 HDL \
+		--in_order_mode 1 --log $@ --verbose 10 --mismatch_print_limit 20
+	tail -n 2 $@
+
+$(TEST_DIR)/whisper_comparison.log: $(TEST_DIR)/whisper_trace.csv $(TEST_DIR)/hdl_trace.csv
 	rm -rf $@
 	python3 $(RISCV_DV_PATH)/scripts/instr_trace_compare.py \
-		--csv_file_1 $(TEST_DIR)/iss_trace.csv --csv_name_1 ISS --csv_file_2 $(TEST_DIR)/hdl_trace.csv --csv_name_2 HDL \
+		--csv_file_1 $(TEST_DIR)/whisper_trace.csv --csv_name_1 ISS --csv_file_2 $(TEST_DIR)/hdl_trace.csv --csv_name_2 HDL \
 		--in_order_mode 1 --log $@ --verbose 10 --mismatch_print_limit 20
+	tail -n 2 $@
 
-compare: $(TEST_DIR)/comparison.log
-	@grep "\[PASSED\]" $<
+compare: $(TEST_DIR)/spike_comparison.log $(TEST_DIR)/whisper_comparison.log
+	@grep "\[PASSED\]" $(TEST_DIR)/spike_comparison.log   >/dev/null
+	@grep "\[PASSED\]" $(TEST_DIR)/whisper_comparison.log >/dev/null
 
 clean:
 	rm -rf $(TEST_DIR)

From 60510277b85680f2755ee8d80fe787ff0b40bbff Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Thu, 13 Apr 2023 15:37:50 +0200
Subject: [PATCH 18/34] Added using VeeR-ISS in the CI

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 .github/workflows/riscv-dv.yml | 66 +++++++++++++++++++++++++++++++---
 tools/riscv-dv/Makefile        | 18 ++++++----
 2 files changed, 73 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/riscv-dv.yml b/.github/workflows/riscv-dv.yml
index 7e63130df9d..5b10ec37010 100644
--- a/.github/workflows/riscv-dv.yml
+++ b/.github/workflows/riscv-dv.yml
@@ -108,6 +108,54 @@ jobs:
           path: /opt/*.tar.gz
           retention-days: 1
 
+  veer-iss:
+    name: Build VeeR-ISS
+    runs-on: ubuntu-latest
+    env:
+      CCACHE_DIR: "/opt/veer-el2/.cache/"
+      DEBIAN_FRONTEND: "noninteractive"
+
+    steps:
+      - name: Install prerequisities
+        run: |
+          sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
+            git build-essential ccache libboost-all-dev
+
+      - name: Create Cache Timestamp
+        id: cache_timestamp
+        uses: nanzm/get-time-action@v1.1
+        with:
+          format: 'YYYY-MM-DD-HH-mm-ss'
+
+      - name: Setup cache
+        uses: actions/cache@v2
+        timeout-minutes: 3
+        continue-on-error: true
+        with:
+          path: "/opt/veer-el2/.cache/"
+          key: cache_veer-iss_${{ steps.cache_timestamp.outputs.time }}
+          restore-keys: cache_veer-iss_
+
+      - name: Build VeeR-ISS
+        run: |
+          git clone https://github.com/chipsalliance/VeeR-ISS veer-iss
+          export CC="ccache gcc"
+          export CXX="ccache g++"
+          pushd veer-iss
+            git checkout 666c94e
+            make -j`nproc`
+            mkdir -p /opt/veer-iss
+            cp build-Linux/whisper /opt/veer-iss/
+          popd
+          cd /opt && tar -czvf veer-iss.tar.gz veer-iss/
+
+      - name: Store VeeR-ISS binaries
+        uses: actions/upload-artifact@v3
+        with:
+          name: veer-iss
+          path: /opt/*.tar.gz
+          retention-days: 1
+
   instrgen:
     name: Build RISC-V DV instruction generator
     runs-on: ubuntu-latest
@@ -172,7 +220,7 @@ jobs:
   tests:
     name: Run RISC-V DV tests
     runs-on: ubuntu-latest
-    needs: [verilator, spike, instrgen]
+    needs: [verilator, spike, veer-iss, instrgen]
     env:
       DEBIAN_FRONTEND: "noninteractive"
 
@@ -182,7 +230,7 @@ jobs:
         run: |
           sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
             git cpanminus ccache device-tree-compiler python3-minimal python3-pip \
-            gcc-riscv64-unknown-elf
+            libboost-all-dev gcc-riscv64-unknown-elf
           sudo cpanm Bit::Vector
 
       - name: Download Verilator binaries
@@ -197,6 +245,12 @@ jobs:
           name: spike
           path: /opt
 
+      - name: Download VeeR-ISS binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: veer-iss
+          path: /opt
+
       - name: Download instruction generator binaries
         uses: actions/download-artifact@v3
         with:
@@ -206,9 +260,10 @@ jobs:
       - name: Unpack binaries
         run: |
           pushd /opt
-            tar -zxvf verilator.tar.gz 
-            tar -zxvf spike.tar.gz 
-            tar -zxvf instrgen.tar.gz 
+            tar -zxvf verilator.tar.gz
+            tar -zxvf spike.tar.gz
+            tar -zxvf veer-iss.tar.gz
+            tar -zxvf instrgen.tar.gz
           popd
 
       - name: Setup repository
@@ -236,6 +291,7 @@ jobs:
         run: |
           export PATH=/opt/verilator/bin:$PATH
           export PATH=/opt/spike/bin:$PATH
+          export PATH=/opt/veer-iss:$PATH
           export PATH=/opt/instrgen:$PATH
           export LD_LIBRARY_PATH=/opt/instrgen:$LD_LIBRARY_PATH
           export RV_ROOT=`realpath veer`
diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index ffca3aba58b..49740e47e91 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -10,7 +10,13 @@ INSTR_COUNT     = 100
 
 VEER_TARGET     = default
 VEER_CONF       = -set build_axi4 \
-                  -set reset_vec=0x2000 # Needed for spike
+                  -set reset_vec=0x2000 \
+                  -set inst_access_enable0=1 \
+                  -set inst_access_addr0=0x00000000 \
+                  -set inst_access_mask0=0x001fffff \
+                  -set data_access_enable0=1 \
+                  -set data_access_addr0=0x00000000 \
+                  -set data_access_mask0=0x001fffff
 
 VERILATOR       = verilator
 VERILATOR_CFLAGS= "-std=c++11"
@@ -61,8 +67,8 @@ $(TEST_DIR)/program.s: code_fixup.py | $(TEST_DIR)
 		+num_of_tests=1 \
 		+start_idx=0 \
 		+num_of_sub_programs=0 \
-		+instr_cnt=10 \
-		+no_branch_jump=true \
+		+instr_cnt=1000 \
+		+no_branch_jump=false \
 		+no_csr_instr=true \
 		+no_fence=true \
 		+no_data_page=true
@@ -81,14 +87,14 @@ $(TEST_DIR)/program.hex: $(TEST_DIR)/program.elf
 
 # Run the Spike ISS simulation
 $(TEST_DIR)/spike_trace.log: $(TEST_DIR)/program.elf
-	timeout --foreground 10s spike -m0x2000:0x100000 --isa=$(ARCH) --misaligned --log=$@ --log-commits -l $< || true
+	timeout --foreground 10s spike -m0x2000:0x98000 --isa=$(ARCH) --misaligned --log=$@ --log-commits -l $< || true
 
 $(TEST_DIR)/spike_trace.csv: $(TEST_DIR)/spike_trace.log
 	python3 $(RISCV_DV_PATH)/scripts/spike_log_to_trace_csv.py --log $< --csv $@
 
 # Run the Whisper ISS simulation
-$(TEST_DIR)/whisper_trace.log: $(TEST_DIR)/program.elf
-	whisper --isa imc --xlen 32 -l -t $< > $@ || rm -rf $@
+$(TEST_DIR)/whisper_trace.log: $(TEST_DIR)/program.elf whisper.json
+	whisper -v --configfile whisper.json --isa imc --xlen 32 --memorysize 0x100000 --raw -m 1000000 -l -t $< > $@ || rm -rf $@
 
 $(TEST_DIR)/whisper_trace.csv: $(TEST_DIR)/whisper_trace.log
 	python3 $(RISCV_DV_PATH)/scripts/whisper_log_trace_csv.py --log $< --csv $@

From c291f535a709e6d620fa08ca94029b612bf98a37 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Fri, 14 Apr 2023 18:00:00 +0200
Subject: [PATCH 19/34] Switched to RISC-V DV code generation and ISS
 invocation done by run.py, added matrix CI

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 .github/workflows/riscv-dv.yml |  91 ++++++---------------------
 tools/riscv-dv/Makefile        | 109 +++++++++++++--------------------
 tools/riscv-dv/link.ld         |  20 ------
 3 files changed, 59 insertions(+), 161 deletions(-)
 delete mode 100644 tools/riscv-dv/link.ld

diff --git a/.github/workflows/riscv-dv.yml b/.github/workflows/riscv-dv.yml
index 5b10ec37010..6e170bce3d1 100644
--- a/.github/workflows/riscv-dv.yml
+++ b/.github/workflows/riscv-dv.yml
@@ -156,76 +156,22 @@ jobs:
           path: /opt/*.tar.gz
           retention-days: 1
 
-  instrgen:
-    name: Build RISC-V DV instruction generator
-    runs-on: ubuntu-latest
-    env:
-      DEBIAN_FRONTEND: "noninteractive"
-
-    steps:
-      - name: Install prerequisities
-        run: |
-          sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
-            git build-essential
-
-      - name: Create Cache Timestamp
-        id: cache_timestamp
-        uses: nanzm/get-time-action@v1.1
-        with:
-          format: 'YYYY-MM-DD-HH-mm-ss'
-
-      - name: Setup cache
-        uses: actions/cache@v2
-        timeout-minutes: 3
-        continue-on-error: true
-        with:
-          path: "/opt/veer-el2/.cache/"
-          key: cache_instrgen_${{ steps.cache_timestamp.outputs.time }}
-          restore-keys: cache_instrgen_
-
-      - name: Download eUVM
-        run: |
-          mkdir -p /opt
-          pushd /opt
-            wget https://github.com/coverify/euvm/releases/download/v1.0-beta25/euvm-1.0-beta25.tar.xz
-            tar -Jxf euvm-1.0-beta25.tar.xz
-            mv euvm-1.0-beta25 euvm
-          popd
-
-      - name: Build
-        run: |
-          # TODO: Use upstream once all the relevant changes for RV32IMC land
-          git clone https://github.com/antmicro/riscv-dv -b master+fixes
-          export PATH=/opt/euvm/bin:$PATH
-          mkdir -p /opt/instrgen
-          pushd riscv-dv/euvm/build
-            # FIXME: ldc2 fails to create directories when copying cached object files. Do this manually upfront
-            find ../riscv -type d | sed 's/../picobj-ldc/' | xargs mkdir -p
-            make -j`nproc` TARGET=RV32IMC LDC2="ldc2 --cache=/opt/veer-el2/.cache"
-            cp *.so /opt/instrgen
-            cp riscv_instr_gen /opt/instrgen
-          popd
-          # Bundle LDC shard libs with the instruction generator
-          rsync -a /opt/euvm/lib/*-ldc-shared.so* /opt/instrgen/
-          cd /opt && tar -czvf instrgen.tar.gz instrgen/
-
-      - name: Store binaries
-        uses: actions/upload-artifact@v3
-        with:
-          name: instrgen
-          path: /opt/*.tar.gz
-          retention-days: 1
-
   # TODO: Parallelize this job for multiple tests
   tests:
     name: Run RISC-V DV tests
     runs-on: ubuntu-latest
-    needs: [verilator, spike, veer-iss, instrgen]
+    needs: [verilator, spike, veer-iss]
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - riscv_arithmetic_basic_test
+        iss:
+          - spike
     env:
       DEBIAN_FRONTEND: "noninteractive"
 
     steps:
-
       - name: Install utils
         run: |
           sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
@@ -251,19 +197,12 @@ jobs:
           name: veer-iss
           path: /opt
 
-      - name: Download instruction generator binaries
-        uses: actions/download-artifact@v3
-        with:
-          name: instrgen
-          path: /opt
-
       - name: Unpack binaries
         run: |
           pushd /opt
             tar -zxvf verilator.tar.gz
             tar -zxvf spike.tar.gz
             tar -zxvf veer-iss.tar.gz
-            tar -zxvf instrgen.tar.gz
           popd
 
       - name: Setup repository
@@ -272,6 +211,10 @@ jobs:
           submodules: recursive
           path: veer
 
+      - name: Install Python deps
+        run: |
+          pip install -r veer/third_party/riscv-dv/requirements.txt
+
       - name: Create Cache Timestamp
         id: cache_timestamp
         uses: nanzm/get-time-action@v1.1
@@ -290,14 +233,16 @@ jobs:
       - name: Run test
         run: |
           export PATH=/opt/verilator/bin:$PATH
-          export PATH=/opt/spike/bin:$PATH
           export PATH=/opt/veer-iss:$PATH
-          export PATH=/opt/instrgen:$PATH
-          export LD_LIBRARY_PATH=/opt/instrgen:$LD_LIBRARY_PATH
           export RV_ROOT=`realpath veer`
+          export RISCV_GCC=riscv64-unknown-elf-gcc
+          export RISCV_OBJCOPY=riscv64-unknown-elf-objcopy
+          export SPIKE_PATH=/opt/spike/bin
+
+          ${RISCV_GCC} --version
 
           pushd ${RV_ROOT}
-            cd tools/riscv-dv && make -j`nproc`
+            cd tools/riscv-dv && make -j`nproc` RISCV_DV_TEST=${{ matrix.test }} RISCV_DV_ISS=${{ matrix.iss }} run
           popd
 
       - name: Pack artifacts
diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index 49740e47e91..7241aee8794 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -1,16 +1,16 @@
 RISCV_DV_PATH   = $(RV_ROOT)/third_party/riscv-dv
-RISCV_DV_TEST  ?= instr_base_test
+RISCV_DV_ISS   ?= spike
+RISCV_DV_TEST  ?= riscv_arithmetic_basic_test
+RISCV_DV_ITER  ?= 1
+RISCV_DV_BATCH ?= 1
 
 WORK_DIR       ?= work
 TEST_DIR        = $(WORK_DIR)/test_$(RISCV_DV_TEST)
-
-UVM_VERBOSITY   = NONE
-SEED            = 10
-INSTR_COUNT     = 100
+SIM_DIR         = $(TEST_DIR)/hdl_sim
 
 VEER_TARGET     = default
 VEER_CONF       = -set build_axi4 \
-                  -set reset_vec=0x2000 \
+                  -set reset_vec=0x80000000# \
                   -set inst_access_enable0=1 \
                   -set inst_access_addr0=0x00000000 \
                   -set inst_access_mask0=0x001fffff \
@@ -23,18 +23,14 @@ VERILATOR_CFLAGS= "-std=c++11"
 VERILATOR_INC   = -I$(WORK_DIR) -I$(RV_ROOT)/testbench 
 VERILATOR_EXE   = $(RV_ROOT)/testbench/test_tb_top.cpp
 
-GCC_PREFIX      = riscv64-unknown-elf
-ARCH            = rv32imc
-ABI             = ilp32
-LDSCRIPT        = link.ld
-
 HDL_FILES = $(WORK_DIR)/common_defines.vh \
             $(WORK_DIR)/el2_pdef.vh \
             $(RV_ROOT)/testbench/tb_top.sv \
             $(RV_ROOT)/testbench/ahb_sif.sv \
             $(RV_ROOT)/design/include/el2_def.sv
 
-all: compare
+all:
+	@echo "Use 'make run'"
 
 # Directory rules
 $(WORK_DIR):
@@ -58,72 +54,48 @@ $(WORK_DIR)/verilator/Vtb_top.mk: $(WORK_DIR)/defines.h
 $(WORK_DIR)/verilator/Vtb_top: $(WORK_DIR)/verilator/Vtb_top.mk
 	$(MAKE) -C $(WORK_DIR)/verilator -f Vtb_top.mk OPT_FAST="-O3"
 
-# Generation rules
-$(TEST_DIR)/program.s: code_fixup.py | $(TEST_DIR)
-	riscv_instr_gen \
-		+UVM_VERBOSITY=$(UVM_VERBOSITY) +random_seed=$(SEED) +instr_cnt=$(INSTR_COUNT) \
-		+asm_file_name=$@ \
-		+UVM_TESTNAME=riscv.test.riscv_instr_base_test.riscv_instr_base_test \
-		+num_of_tests=1 \
-		+start_idx=0 \
-		+num_of_sub_programs=0 \
-		+instr_cnt=1000 \
-		+no_branch_jump=false \
-		+no_csr_instr=true \
-		+no_fence=true \
-		+no_data_page=true
-	python3 code_fixup.py -i $@_0.S -o $@
-	touch $(TEST_DIR)/user_define.h
-	touch $(TEST_DIR)/user_init.s
-
-# Compilation rules
-$(TEST_DIR)/program.elf: $(TEST_DIR)/program.s $(LDSCRIPT)
-	$(GCC_PREFIX)-gcc -march=$(ARCH) -mabi=$(ABI) -O0 -T$(LDSCRIPT) -I$(TEST_DIR) -static -nostdlib -nostartfiles $< -o $@
-	$(GCC_PREFIX)-objdump -S $@ > $(basename $@).lst
-	$(GCC_PREFIX)-nm -n $@ > $(basename $@).sym
-
-$(TEST_DIR)/program.hex: $(TEST_DIR)/program.elf
-	$(GCC_PREFIX)-objcopy -O verilog $< $@
-
-# Run the Spike ISS simulation
-$(TEST_DIR)/spike_trace.log: $(TEST_DIR)/program.elf
-	timeout --foreground 10s spike -m0x2000:0x98000 --isa=$(ARCH) --misaligned --log=$@ --log-commits -l $< || true
-
-$(TEST_DIR)/spike_trace.csv: $(TEST_DIR)/spike_trace.log
+# Code generation, compilation and ISS simulation via RISC-V DV flow
+$(TEST_DIR)/generate.log: | $(TEST_DIR)
+	PYTHONPATH=$(RISCV_DV_PATH)/pygen python3 $(RISCV_DV_PATH)/run.py --simulator pyflow \
+		--test $(RISCV_DV_TEST) --iss $(RISCV_DV_ISS) \
+		--iterations $(RISCV_DV_ITER) --batch_size $(RISCV_DV_BATCH) \
+		--isa rv32imc --mabi ilp32 -v -o $(TEST_DIR) 2>&1 | tee $(TEST_DIR)/generate.log
+
+$(TEST_DIR)/asm_test/%.hex: $(TEST_DIR)/asm_test/%.o
+	$(RISCV_OBJCOPY) -O verilog $< $@
+
+# HDL simulation
+$(SIM_DIR)/%.log: $(TEST_DIR)/asm_test/%.hex $(WORK_DIR)/verilator/Vtb_top
+	mkdir -p $(basename $@)
+	cp $< $(basename $@)/program.hex
+	cd $(basename $@) && $(abspath $(WORK_DIR)/verilator/Vtb_top)
+	mv $(basename $@)/exec.log $@
+
+# Log conversion rules
+$(TEST_DIR)/spike_sim/%.csv: $(TEST_DIR)/spike_sim/%.log
 	python3 $(RISCV_DV_PATH)/scripts/spike_log_to_trace_csv.py --log $< --csv $@
 
-# Run the Whisper ISS simulation
-$(TEST_DIR)/whisper_trace.log: $(TEST_DIR)/program.elf whisper.json
-	whisper -v --configfile whisper.json --isa imc --xlen 32 --memorysize 0x100000 --raw -m 1000000 -l -t $< > $@ || rm -rf $@
-
-$(TEST_DIR)/whisper_trace.csv: $(TEST_DIR)/whisper_trace.log
+$(TEST_DIR)/whisper_sim/%.csv: $(TEST_DIR)/whisper_sim/%.log
 	python3 $(RISCV_DV_PATH)/scripts/whisper_log_trace_csv.py --log $< --csv $@
 
-# Run the HDL simulation
-$(TEST_DIR)/hdl_trace.log: $(WORK_DIR)/verilator/Vtb_top $(TEST_DIR)/program.hex
-	cd $(TEST_DIR) && $(abspath $(WORK_DIR)/verilator/Vtb_top)
-	mv $(TEST_DIR)/exec.log $@
-
-$(TEST_DIR)/hdl_trace.csv: $(TEST_DIR)/hdl_trace.log veer_log_to_trace_csv.py
+$(SIM_DIR)/%.csv: $(SIM_DIR)/%.log veer_log_to_trace_csv.py
 	PYTHONPATH=$(RISCV_DV_PATH)/scripts python3 veer_log_to_trace_csv.py --log $< --csv $@
 
-$(TEST_DIR)/spike_comparison.log: $(TEST_DIR)/spike_trace.csv $(TEST_DIR)/hdl_trace.csv
-	rm -rf $@
-	python3 $(RISCV_DV_PATH)/scripts/instr_trace_compare.py \
-		--csv_file_1 $(TEST_DIR)/spike_trace.csv --csv_name_1 ISS --csv_file_2 $(TEST_DIR)/hdl_trace.csv --csv_name_2 HDL \
-		--in_order_mode 1 --log $@ --verbose 10 --mismatch_print_limit 20
-	tail -n 2 $@
-
-$(TEST_DIR)/whisper_comparison.log: $(TEST_DIR)/whisper_trace.csv $(TEST_DIR)/hdl_trace.csv
+# Trace comparison
+$(TEST_DIR)/comp_%.log: $(TEST_DIR)/$(RISCV_DV_ISS)_sim/%.csv $(SIM_DIR)/%.csv
 	rm -rf $@
 	python3 $(RISCV_DV_PATH)/scripts/instr_trace_compare.py \
-		--csv_file_1 $(TEST_DIR)/whisper_trace.csv --csv_name_1 ISS --csv_file_2 $(TEST_DIR)/hdl_trace.csv --csv_name_2 HDL \
+		--csv_file_1 $(word 1, $^) --csv_name_1 ISS --csv_file_2 $(word 2, $^) --csv_name_2 HDL \
 		--in_order_mode 1 --log $@ --verbose 10 --mismatch_print_limit 20
 	tail -n 2 $@
 
-compare: $(TEST_DIR)/spike_comparison.log $(TEST_DIR)/whisper_comparison.log
-	@grep "\[PASSED\]" $(TEST_DIR)/spike_comparison.log   >/dev/null
-	@grep "\[PASSED\]" $(TEST_DIR)/whisper_comparison.log >/dev/null
+run:
+	# Run RISC-V DV
+	$(MAKE) $(TEST_DIR)/generate.log
+	# Run HDL simulation(s) and trace comparison
+	find $(TEST_DIR)/ -name "sim_*.log" | sed 's/sim_/comp_/g' | xargs $(MAKE)
+	# Check for errors
+	for F in $(TEST_DIR)/comp_*.log; do grep "\[PASSED\]" $$F; if [ $$? -ne 0 ]; then exit 255; fi; done
 
 clean:
 	rm -rf $(TEST_DIR)
@@ -131,4 +103,5 @@ clean:
 fullclean:
 	rm -rf $(WORK_DIR)
 
-.PHONY: clean fullclean compare
+.PHONY: all run generate clean fullclean compare
+.SECONDARY:
diff --git a/tools/riscv-dv/link.ld b/tools/riscv-dv/link.ld
deleted file mode 100644
index 1f4a0fb6a32..00000000000
--- a/tools/riscv-dv/link.ld
+++ /dev/null
@@ -1,20 +0,0 @@
-OUTPUT_ARCH( "riscv" )
-ENTRY(_start)
-
-SECTIONS
-{
-  . = 0x2000;
-  .text : { *(.text) }
-  . = ALIGN(0x1000);
-  .tohost : { *(.tohost) }
-  . = ALIGN(0x1000);
-  .page_table : { *(.page_table) }
-  _end = .;
-
-  . = 0x10000;
-  .data : { *(.data) }
-  .user_stack : { *(.user_stack) }
-  .kernel_data : { *(.kernel_data) }
-  .kernel_stack : { *(.kernel_stack) }
-  .bss : { *(.bss) }
-}

From 83cd6c92dd3334fc979691135f6612ece6fdedc5 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 17 Apr 2023 11:48:55 +0200
Subject: [PATCH 20/34] Integrated code fixup needed for VeeR with the RISC-V
 DV flow

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile      | 12 +++++++++++-
 tools/riscv-dv/code_fixup.py |  4 +++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index 7241aee8794..5e31f2606e8 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -56,10 +56,20 @@ $(WORK_DIR)/verilator/Vtb_top: $(WORK_DIR)/verilator/Vtb_top.mk
 
 # Code generation, compilation and ISS simulation via RISC-V DV flow
 $(TEST_DIR)/generate.log: | $(TEST_DIR)
+	# Generate
 	PYTHONPATH=$(RISCV_DV_PATH)/pygen python3 $(RISCV_DV_PATH)/run.py --simulator pyflow \
 		--test $(RISCV_DV_TEST) --iss $(RISCV_DV_ISS) \
 		--iterations $(RISCV_DV_ITER) --batch_size $(RISCV_DV_BATCH) \
-		--isa rv32imc --mabi ilp32 -v -o $(TEST_DIR) 2>&1 | tee $(TEST_DIR)/generate.log
+		--isa rv32imc --mabi ilp32 --steps gen -v -o $(TEST_DIR) 2>&1 | tee $(TEST_DIR)/generate.log
+
+	# Patch the code
+	find $(TEST_DIR)/asm_test -name "*.S" -exec python3 code_fixup.py -i {} -o {} \;
+
+	# Compile, simulate
+	PYTHONPATH=$(RISCV_DV_PATH)/pygen python3 $(RISCV_DV_PATH)/run.py --simulator pyflow \
+		--test $(RISCV_DV_TEST) --iss $(RISCV_DV_ISS) \
+		--iterations $(RISCV_DV_ITER) --batch_size $(RISCV_DV_BATCH) \
+		--isa rv32imc --mabi ilp32 --steps gcc_compile,iss_sim -v -o $(TEST_DIR) 2>&1 | tee -a $(TEST_DIR)/generate.log
 
 $(TEST_DIR)/asm_test/%.hex: $(TEST_DIR)/asm_test/%.o
 	$(RISCV_OBJCOPY) -O verilog $< $@
diff --git a/tools/riscv-dv/code_fixup.py b/tools/riscv-dv/code_fixup.py
index d288ddda143..cd982544dd9 100644
--- a/tools/riscv-dv/code_fixup.py
+++ b/tools/riscv-dv/code_fixup.py
@@ -57,6 +57,8 @@ def main():
 
     args = parser.parse_args()
 
+    max_nops = 10
+
     # Read and parse
     with open(args.i, "r") as fp:
         inp_lines = [AssemblyLine(l) for l in fp.readlines()]
@@ -89,7 +91,7 @@ def main():
         dst = line.operands[0]
         for j, l in enumerate(following):
             if l.operands and l.operands[0] == dst:
-                nops = max(0, 2 - j)
+                nops = max(0, max_nops - j)
                 for _ in range(nops):
                     out_lines.append(" " * 18 + "nop # FIXME: A fixup not to make VeeR cancel a delayed write\n")
                 break

From bf3d8e9e77db6a60434461ab4ae7a27c9eee9fad Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 17 Apr 2023 12:01:28 +0200
Subject: [PATCH 21/34] Fixed handling of execution corner case when a delayed
 effect insruction attempts writing to x0 (zero)

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/veer_log_to_trace_csv.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tools/riscv-dv/veer_log_to_trace_csv.py b/tools/riscv-dv/veer_log_to_trace_csv.py
index 2e98a32c61c..8c905dcb04d 100644
--- a/tools/riscv-dv/veer_log_to_trace_csv.py
+++ b/tools/riscv-dv/veer_log_to_trace_csv.py
@@ -71,6 +71,11 @@ def parse_log(file_name):
 
             # Delayed effect, search the queue
             if gpr is None and mnemonic in ["lw", "div", "divu", "rem", "remu"]:
+
+                # Skip if targets x0 (zero) which makes no sense
+                if operands[0] == "zero":
+                    continue 
+
                 for ent in reversed(queue):
 
                     if (ent.operand == "nbL" and mnemonic in ["lw"]) or \
@@ -143,6 +148,7 @@ def parse_log(file_name):
         while len(queue):
             entry = queue[0]
 
+            # Cannot dequeue, break
             if not entry.pc or not entry.gpr:
                 break
 

From fdefb36ec2835439fc748c983f132266f8dc33c9 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 17 Apr 2023 12:20:58 +0200
Subject: [PATCH 22/34] RISC-V DV workflow tweaks

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 .github/workflows/riscv-dv.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/riscv-dv.yml b/.github/workflows/riscv-dv.yml
index 6e170bce3d1..5c623e5ffac 100644
--- a/.github/workflows/riscv-dv.yml
+++ b/.github/workflows/riscv-dv.yml
@@ -156,7 +156,6 @@ jobs:
           path: /opt/*.tar.gz
           retention-days: 1
 
-  # TODO: Parallelize this job for multiple tests
   tests:
     name: Run RISC-V DV tests
     runs-on: ubuntu-latest
@@ -168,6 +167,7 @@ jobs:
           - riscv_arithmetic_basic_test
         iss:
           - spike
+          - whisper
     env:
       DEBIAN_FRONTEND: "noninteractive"
 
@@ -238,16 +238,17 @@ jobs:
           export RISCV_GCC=riscv64-unknown-elf-gcc
           export RISCV_OBJCOPY=riscv64-unknown-elf-objcopy
           export SPIKE_PATH=/opt/spike/bin
+          export WHISPER_ISS=/opt/veer-iss/whisper
 
           ${RISCV_GCC} --version
 
           pushd ${RV_ROOT}
-            cd tools/riscv-dv && make -j`nproc` RISCV_DV_TEST=${{ matrix.test }} RISCV_DV_ISS=${{ matrix.iss }} run
+            cd tools/riscv-dv && make -j`nproc` RISCV_DV_TEST=${{ matrix.test }} RISCV_DV_ISS=${{ matrix.iss }} RISCV_DV_ITER=3 run
           popd
 
       - name: Pack artifacts
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: riscv-dv-results
+          name: artifacts-${{ matrix.test }}
           path: veer/tools/riscv-dv/work/test_*

From ee21b3ce56d79b2537f4db8abafb908d7fdda7e9 Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Wed, 17 May 2023 16:39:10 +0200
Subject: [PATCH 23/34] Integrated Renode with the RISC-V DV flow

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 tools/riscv-dv/Makefile | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index 5e31f2606e8..fc7955ab10d 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -29,6 +29,8 @@ HDL_FILES = $(WORK_DIR)/common_defines.vh \
             $(RV_ROOT)/testbench/ahb_sif.sv \
             $(RV_ROOT)/design/include/el2_def.sv
 
+MAKEFILE  = $(abspath $(MAKEFILE_LIST))
+
 all:
 	@echo "Use 'make run'"
 
@@ -67,7 +69,7 @@ $(TEST_DIR)/generate.log: | $(TEST_DIR)
 
 	# Compile, simulate
 	PYTHONPATH=$(RISCV_DV_PATH)/pygen python3 $(RISCV_DV_PATH)/run.py --simulator pyflow \
-		--test $(RISCV_DV_TEST) --iss $(RISCV_DV_ISS) \
+		--test $(RISCV_DV_TEST) --iss $(RISCV_DV_ISS) --iss_timeout 60 \
 		--iterations $(RISCV_DV_ITER) --batch_size $(RISCV_DV_BATCH) \
 		--isa rv32imc --mabi ilp32 --steps gcc_compile,iss_sim -v -o $(TEST_DIR) 2>&1 | tee -a $(TEST_DIR)/generate.log
 
@@ -88,6 +90,9 @@ $(TEST_DIR)/spike_sim/%.csv: $(TEST_DIR)/spike_sim/%.log
 $(TEST_DIR)/whisper_sim/%.csv: $(TEST_DIR)/whisper_sim/%.log
 	python3 $(RISCV_DV_PATH)/scripts/whisper_log_trace_csv.py --log $< --csv $@
 
+$(TEST_DIR)/renode_sim/%.csv: $(TEST_DIR)/renode_sim/%.log
+	python3 $(RISCV_DV_PATH)/scripts/renode_log_to_trace_csv.py --log $< --csv $@
+
 $(SIM_DIR)/%.csv: $(SIM_DIR)/%.log veer_log_to_trace_csv.py
 	PYTHONPATH=$(RISCV_DV_PATH)/scripts python3 veer_log_to_trace_csv.py --log $< --csv $@
 
@@ -101,9 +106,9 @@ $(TEST_DIR)/comp_%.log: $(TEST_DIR)/$(RISCV_DV_ISS)_sim/%.csv $(SIM_DIR)/%.csv
 
 run:
 	# Run RISC-V DV
-	$(MAKE) $(TEST_DIR)/generate.log
+	$(MAKE) -f $(MAKEFILE) $(TEST_DIR)/generate.log
 	# Run HDL simulation(s) and trace comparison
-	find $(TEST_DIR)/ -name "sim_*.log" | sed 's/sim_/comp_/g' | xargs $(MAKE)
+	find $(TEST_DIR)/ -name "sim_*.log" | sed 's/sim_/comp_/g' | xargs $(MAKE) -f $(MAKEFILE)
 	# Check for errors
 	for F in $(TEST_DIR)/comp_*.log; do grep "\[PASSED\]" $$F; if [ $$? -ne 0 ]; then exit 255; fi; done
 

From 88e737355bd95ec07af56be3bb923c23f466d7ac Mon Sep 17 00:00:00 2001
From: Michal Czyz <mczyz@antmicro.com>
Date: Thu, 18 May 2023 14:19:53 +0200
Subject: [PATCH 24/34] Add support for renode

---
 .github/workflows/riscv-dv.yml | 57 ++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/riscv-dv.yml b/.github/workflows/riscv-dv.yml
index 5c623e5ffac..fac599c1714 100644
--- a/.github/workflows/riscv-dv.yml
+++ b/.github/workflows/riscv-dv.yml
@@ -56,6 +56,51 @@ jobs:
           path: /opt/*.tar.gz
           retention-days: 1
 
+#--------------#
+#    Renode
+#--------------#
+  renode:
+    name: Download Renode
+    runs-on: ubuntu-latest
+    env:
+      CCACHE_DIR: "/opt/veer-el2/.cache/"
+      DEBIAN_FRONTEND: "noninteractive"
+
+    steps:
+      - name: Create Cache Timestamp
+        id: cache_timestamp
+        uses: nanzm/get-time-action@v1.1
+        with:
+          format: 'YYYY-MM-DD-HH-mm-ss'
+
+      - name: Setup cache
+        uses: actions/cache@v2
+        timeout-minutes: 3
+        continue-on-error: true
+        with:
+          path: "/opt/veer-el2/.cache/"
+          key: cache_renode_${{ steps.cache_timestamp.outputs.time }}
+          restore-keys: cache_renode_
+
+      - name: Get latest release
+        uses: robinraju/release-downloader@v1.8
+        with:
+          repository: "renode/renode"
+          latest: true
+          fileName: "renode-*.linux-portable.tar.gz"
+          extract: false
+
+      - name: Rename the archive
+        run: |
+          mv ${{ github.workspace }}/renode-*.tar.gz ${{ github.workspace }}/renode.tar.gz
+
+      - name: Store Renode binaries
+        uses: actions/upload-artifact@v3
+        with:
+          name: renode
+          path: ${{ github.workspace }}/renode.tar.gz
+          retention-days: 1
+
   spike:
     name: Build Spike ISS
     runs-on: ubuntu-latest
@@ -159,7 +204,7 @@ jobs:
   tests:
     name: Run RISC-V DV tests
     runs-on: ubuntu-latest
-    needs: [verilator, spike, veer-iss]
+    needs: [verilator, spike, veer-iss, renode]
     strategy:
       fail-fast: false
       matrix:
@@ -168,6 +213,7 @@ jobs:
         iss:
           - spike
           - whisper
+          - renode
     env:
       DEBIAN_FRONTEND: "noninteractive"
 
@@ -185,6 +231,12 @@ jobs:
           name: verilator
           path: /opt
 
+      - name: Download Renode binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: renode
+          path: /opt
+
       - name: Download Spike binaries
         uses: actions/download-artifact@v3
         with:
@@ -203,6 +255,7 @@ jobs:
             tar -zxvf verilator.tar.gz
             tar -zxvf spike.tar.gz
             tar -zxvf veer-iss.tar.gz
+            tar -zxvf renode.tar.gz --strip-components=1
           popd
 
       - name: Setup repository
@@ -239,7 +292,7 @@ jobs:
           export RISCV_OBJCOPY=riscv64-unknown-elf-objcopy
           export SPIKE_PATH=/opt/spike/bin
           export WHISPER_ISS=/opt/veer-iss/whisper
-
+          export RENODE_PATH=/opt/renode
           ${RISCV_GCC} --version
 
           pushd ${RV_ROOT}

From bd0ac932e3d399212dd1cfd013a9d0e83259172d Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 22 May 2023 09:58:50 +0200
Subject: [PATCH 25/34] Bumped RISC-V DV submodule

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 .gitmodules          | 2 +-
 third_party/riscv-dv | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index bc3f4fcced0..f6ccdc7bd12 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -3,4 +3,4 @@
 	url = https://github.com/picolibc/picolibc
 [submodule "third_party/riscv-dv"]
 	path = third_party/riscv-dv
-	url = https://github.com/antmicro/riscv-dv
+	url = https://github.com/chipsalliance/riscv-dv
diff --git a/third_party/riscv-dv b/third_party/riscv-dv
index d30ac5eb8c5..c409955d679 160000
--- a/third_party/riscv-dv
+++ b/third_party/riscv-dv
@@ -1 +1 @@
-Subproject commit d30ac5eb8c5b523c54314716b7f578d6e859a130
+Subproject commit c409955d67923afd1ebfc5ff0f9a52e4852b229f

From b7c096b94933c55a02f0f83025dcee837a60a44f Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 22 May 2023 10:34:04 +0200
Subject: [PATCH 26/34] Separated CI artifact packages for different ISS

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 .github/workflows/riscv-dv.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/riscv-dv.yml b/.github/workflows/riscv-dv.yml
index fac599c1714..7ad2b6d9aad 100644
--- a/.github/workflows/riscv-dv.yml
+++ b/.github/workflows/riscv-dv.yml
@@ -303,5 +303,5 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: artifacts-${{ matrix.test }}
+          name: artifacts-${{ matrix.test }}-${{ matrix.iss }}
           path: veer/tools/riscv-dv/work/test_*

From a583f6a207e5936b9b94d90485582b102a1ffa9d Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 22 May 2023 11:10:17 +0200
Subject: [PATCH 27/34] Added seeding rng in RISC-V DV to the CI

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 .github/workflows/riscv-dv.yml | 7 ++++++-
 tools/riscv-dv/Makefile        | 5 +++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/riscv-dv.yml b/.github/workflows/riscv-dv.yml
index 7ad2b6d9aad..a1a03d30c97 100644
--- a/.github/workflows/riscv-dv.yml
+++ b/.github/workflows/riscv-dv.yml
@@ -296,7 +296,12 @@ jobs:
           ${RISCV_GCC} --version
 
           pushd ${RV_ROOT}
-            cd tools/riscv-dv && make -j`nproc` RISCV_DV_TEST=${{ matrix.test }} RISCV_DV_ISS=${{ matrix.iss }} RISCV_DV_ITER=3 run
+            cd tools/riscv-dv && make -j`nproc` \
+              RISCV_DV_TEST=${{ matrix.test }} \
+              RISCV_DV_ISS=${{ matrix.iss }} \
+              RISCV_DV_ITER=3 \
+              RISCV_DV_SEED=999 \
+              run
           popd
 
       - name: Pack artifacts
diff --git a/tools/riscv-dv/Makefile b/tools/riscv-dv/Makefile
index fc7955ab10d..d4d1b8cace8 100644
--- a/tools/riscv-dv/Makefile
+++ b/tools/riscv-dv/Makefile
@@ -1,6 +1,7 @@
 RISCV_DV_PATH   = $(RV_ROOT)/third_party/riscv-dv
 RISCV_DV_ISS   ?= spike
 RISCV_DV_TEST  ?= riscv_arithmetic_basic_test
+RISCV_DV_SEED  ?= 999
 RISCV_DV_ITER  ?= 1
 RISCV_DV_BATCH ?= 1
 
@@ -61,7 +62,7 @@ $(TEST_DIR)/generate.log: | $(TEST_DIR)
 	# Generate
 	PYTHONPATH=$(RISCV_DV_PATH)/pygen python3 $(RISCV_DV_PATH)/run.py --simulator pyflow \
 		--test $(RISCV_DV_TEST) --iss $(RISCV_DV_ISS) \
-		--iterations $(RISCV_DV_ITER) --batch_size $(RISCV_DV_BATCH) \
+		--start_seed $(RISCV_DV_SEED) --iterations $(RISCV_DV_ITER) --batch_size $(RISCV_DV_BATCH) \
 		--isa rv32imc --mabi ilp32 --steps gen -v -o $(TEST_DIR) 2>&1 | tee $(TEST_DIR)/generate.log
 
 	# Patch the code
@@ -70,7 +71,7 @@ $(TEST_DIR)/generate.log: | $(TEST_DIR)
 	# Compile, simulate
 	PYTHONPATH=$(RISCV_DV_PATH)/pygen python3 $(RISCV_DV_PATH)/run.py --simulator pyflow \
 		--test $(RISCV_DV_TEST) --iss $(RISCV_DV_ISS) --iss_timeout 60 \
-		--iterations $(RISCV_DV_ITER) --batch_size $(RISCV_DV_BATCH) \
+		--start_seed $(RISCV_DV_SEED) --iterations $(RISCV_DV_ITER) --batch_size $(RISCV_DV_BATCH) \
 		--isa rv32imc --mabi ilp32 --steps gcc_compile,iss_sim -v -o $(TEST_DIR) 2>&1 | tee -a $(TEST_DIR)/generate.log
 
 $(TEST_DIR)/asm_test/%.hex: $(TEST_DIR)/asm_test/%.o

From 87e5c9cba931faa145c462855c8f932da50815ad Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 29 May 2023 14:47:30 +0200
Subject: [PATCH 28/34] Added missing el2_pkg imports

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 design/dec/el2_dec_tlu_ctl.sv |  4 +++-
 design/el2_dma_ctrl.sv        |  4 +++-
 design/el2_pic_ctrl.sv        |  4 +++-
 design/ifu/el2_ifu.sv         | 39 ++++++++++++++++++-----------------
 design/include/el2_def.sv     |  2 ++
 design/lib/el2_lib.sv         | 16 ++++++++++----
 6 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/design/dec/el2_dec_tlu_ctl.sv b/design/dec/el2_dec_tlu_ctl.sv
index 511c0adaa56..0e753606f85 100644
--- a/design/dec/el2_dec_tlu_ctl.sv
+++ b/design/dec/el2_dec_tlu_ctl.sv
@@ -2788,7 +2788,9 @@ assign dec_csr_rddata_d[31:0] = ( ({32{csr_misa}}      & 32'h40001104) |
 
 endmodule // el2_dec_tlu_ctl
 
-module el2_dec_timer_ctl #(
+module el2_dec_timer_ctl
+import el2_pkg::*;
+#(
 `include "el2_param.vh"
  )
   (
diff --git a/design/el2_dma_ctrl.sv b/design/el2_dma_ctrl.sv
index b415a8846e3..1398a10bdb1 100644
--- a/design/el2_dma_ctrl.sv
+++ b/design/el2_dma_ctrl.sv
@@ -21,7 +21,9 @@
 //
 //********************************************************************************
 
-module el2_dma_ctrl #(
+module el2_dma_ctrl 
+import el2_pkg::*;
+#(
 `include "el2_param.vh"
  )(
    input logic         clk,
diff --git a/design/el2_pic_ctrl.sv b/design/el2_pic_ctrl.sv
index 8f8881f848c..797872e155e 100644
--- a/design/el2_pic_ctrl.sv
+++ b/design/el2_pic_ctrl.sv
@@ -20,7 +20,9 @@
 // Comments:
 //********************************************************************************
 
-module el2_pic_ctrl #(
+module el2_pic_ctrl 
+import el2_pkg::*;
+#(
 `include "el2_param.vh"
  )
                   (
diff --git a/design/ifu/el2_ifu.sv b/design/ifu/el2_ifu.sv
index 837a9c24957..d36cd68bd7f 100644
--- a/design/ifu/el2_ifu.sv
+++ b/design/ifu/el2_ifu.sv
@@ -241,6 +241,26 @@ import el2_pkg::*;
    logic [1:0] [$clog2(pt.BTB_SIZE)-1:0] ifu_bp_fa_index_f;
 
 
+   logic [1:0]   ic_fetch_val_f;
+   logic [31:0] ic_data_f;
+   logic [31:0] ifu_fetch_data_f;
+   logic ifc_fetch_req_f;
+   logic ifc_fetch_req_f_raw;
+   logic [1:0] iccm_rd_ecc_double_err;  // This fetch has an iccm double error.
+
+   logic ifu_async_error_start;
+
+
+   assign ifu_fetch_data_f[31:0] = ic_data_f[31:0];
+   assign ifu_fetch_val[1:0] = ic_fetch_val_f[1:0];
+   assign ifu_fetch_pc[31:1] = ifc_fetch_addr_f[31:1];
+
+ logic                       ifc_fetch_uncacheable_bf;      // The fetch request is uncacheable space. BF stage
+ logic                       ifc_fetch_req_bf;              // Fetch request. Comes with the address.  BF stage
+ logic                       ifc_fetch_req_bf_raw;          // Fetch request without some qualifications. Used for clock-gating. BF stage
+ logic                       ifc_iccm_access_bf;            // This request is to the ICCM. Do not generate misses to the bus.
+ logic                       ifc_region_acc_fault_bf;       // Access fault. in ICCM region but offset is outside defined ICCM.
+
    // fetch control
    el2_ifu_ifc_ctl #(.pt(pt)) ifc (.*
                     );
@@ -262,25 +282,6 @@ import el2_pkg::*;
    end
 
 
-   logic [1:0]   ic_fetch_val_f;
-   logic [31:0] ic_data_f;
-   logic [31:0] ifu_fetch_data_f;
-   logic ifc_fetch_req_f;
-   logic ifc_fetch_req_f_raw;
-   logic [1:0] iccm_rd_ecc_double_err;  // This fetch has an iccm double error.
-
-   logic ifu_async_error_start;
-
-
-   assign ifu_fetch_data_f[31:0] = ic_data_f[31:0];
-   assign ifu_fetch_val[1:0] = ic_fetch_val_f[1:0];
-   assign ifu_fetch_pc[31:1] = ifc_fetch_addr_f[31:1];
-
- logic                       ifc_fetch_uncacheable_bf;      // The fetch request is uncacheable space. BF stage
- logic                       ifc_fetch_req_bf;              // Fetch request. Comes with the address.  BF stage
- logic                       ifc_fetch_req_bf_raw;          // Fetch request without some qualifications. Used for clock-gating. BF stage
- logic                       ifc_iccm_access_bf;            // This request is to the ICCM. Do not generate misses to the bus.
- logic                       ifc_region_acc_fault_bf;       // Access fault. in ICCM region but offset is outside defined ICCM.
 
    // aligner
 
diff --git a/design/include/el2_def.sv b/design/include/el2_def.sv
index cae80f40f94..f4de36e2b5e 100644
--- a/design/include/el2_def.sv
+++ b/design/include/el2_def.sv
@@ -3,6 +3,8 @@
 //`define EL2_DEF_SV
 package el2_pkg;
 
+`include "el2_pdef.vh"
+
 typedef struct packed {
                        logic  trace_rv_i_valid_ip;
                        logic [31:0] trace_rv_i_insn_ip;
diff --git a/design/lib/el2_lib.sv b/design/lib/el2_lib.sv
index 6f71a3c13f2..a1c96246125 100644
--- a/design/lib/el2_lib.sv
+++ b/design/lib/el2_lib.sv
@@ -1,4 +1,6 @@
-module el2_btb_tag_hash #(
+module el2_btb_tag_hash
+import el2_pkg::*;
+#(
 `include "el2_param.vh"
  ) (
                        input logic [pt.BTB_ADDR_HI+pt.BTB_BTAG_SIZE+pt.BTB_BTAG_SIZE+pt.BTB_BTAG_SIZE:pt.BTB_ADDR_HI+1] pc,
@@ -10,7 +12,9 @@ module el2_btb_tag_hash #(
                    pc[pt.BTB_ADDR_HI+pt.BTB_BTAG_SIZE:pt.BTB_ADDR_HI+1])};
 endmodule
 
-module el2_btb_tag_hash_fold  #(
+module el2_btb_tag_hash_fold
+import el2_pkg::*;
+#(
 `include "el2_param.vh"
  )(
                        input logic [pt.BTB_ADDR_HI+pt.BTB_BTAG_SIZE+pt.BTB_BTAG_SIZE:pt.BTB_ADDR_HI+1] pc,
@@ -23,7 +27,9 @@ module el2_btb_tag_hash_fold  #(
 
 endmodule
 
-module el2_btb_addr_hash  #(
+module el2_btb_addr_hash
+import el2_pkg::*;
+#(
 `include "el2_param.vh"
  )(
                         input logic [pt.BTB_INDEX3_HI:pt.BTB_INDEX1_LO] pc,
@@ -43,7 +49,9 @@ end
 
 endmodule
 
-module el2_btb_ghr_hash  #(
+module el2_btb_ghr_hash
+import el2_pkg::*;
+#(
 `include "el2_param.vh"
  )(
                        input logic [pt.BTB_ADDR_HI:pt.BTB_ADDR_LO] hashin,

From 9c9261374d0ae2743640b095d4ed0a4a7f8c459e Mon Sep 17 00:00:00 2001
From: Rafal Kolucki <rkolucki@antmicro.com>
Date: Thu, 30 Mar 2023 15:29:56 +0200
Subject: [PATCH 29/34] lint: waive module-filename rule for libraries

Signed-off-by: Rafal Kolucki <rkolucki@antmicro.com>
---
 .github/workflows/verible-lint.yml | 2 ++
 violations.waiver                  | 6 ++++++
 2 files changed, 8 insertions(+)
 create mode 100644 violations.waiver

diff --git a/.github/workflows/verible-lint.yml b/.github/workflows/verible-lint.yml
index cfe593f3e1b..d023a762eb4 100644
--- a/.github/workflows/verible-lint.yml
+++ b/.github/workflows/verible-lint.yml
@@ -17,5 +17,7 @@ jobs:
       - uses: chipsalliance/verible-linter-action@main
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
+          extra_args: '--waiver_files=./violations.waiver'
           paths: |
             ./design
+
diff --git a/violations.waiver b/violations.waiver
new file mode 100644
index 00000000000..5844b2e4556
--- /dev/null
+++ b/violations.waiver
@@ -0,0 +1,6 @@
+waive --rule=module-filename    --location="design/lib/.*_lib.sv"
+waive --rule=line-length        --location="design/ifu/.*.sv"
+waive --rule=line-length        --location="design/el2_.*_ctrl.sv"
+waive --rule=no-trailing-spaces --location="design/ifu/.*.sv"
+waive --rule=no-trailing-spaces --location="design/el2_.*_ctrl.sv"
+

From 01ba37ce6281fc47661f1d33d67774e261882f01 Mon Sep 17 00:00:00 2001
From: Michal Czyz <mczyz@antmicro.com>
Date: Mon, 5 Jun 2023 11:02:03 +0200
Subject: [PATCH 30/34] Implement minimal PyUVM test environment

---
 verification/README.md                        |  70 +++++++++
 verification/__init__.py                      |   0
 verification/requirements.txt                 |  13 ++
 verification/test_pyuvm/Makefile              | 139 ++++++++++++++++++
 verification/test_pyuvm/__init__.py           |   0
 verification/test_pyuvm/test_irq/irq_utils.py | 102 +++++++++++++
 verification/test_pyuvm/test_irq/irq_uvm.py   | 129 ++++++++++++++++
 verification/test_pyuvm/test_irq/test_irq.py  |  24 +++
 verification/test_pyuvm/test_pyuvm.py         |  27 ++++
 9 files changed, 504 insertions(+)
 create mode 100644 verification/README.md
 create mode 100644 verification/__init__.py
 create mode 100644 verification/requirements.txt
 create mode 100755 verification/test_pyuvm/Makefile
 create mode 100644 verification/test_pyuvm/__init__.py
 create mode 100644 verification/test_pyuvm/test_irq/irq_utils.py
 create mode 100644 verification/test_pyuvm/test_irq/irq_uvm.py
 create mode 100644 verification/test_pyuvm/test_irq/test_irq.py
 create mode 100644 verification/test_pyuvm/test_pyuvm.py

diff --git a/verification/README.md b/verification/README.md
new file mode 100644
index 00000000000..00266622312
--- /dev/null
+++ b/verification/README.md
@@ -0,0 +1,70 @@
+# Verification
+
+The verification directory contains [cocotb](https://github.com/cocotb/cocotb) tests and [pyuvm](https://github.com/pyuvm/pyuvm) tests
+
+## Setup
+
+In order to run the tests, one must clone the repository, create a python virtual environment and patch cocotb's verilator support file. Verilator is used as a backend simulator and must be present in the system.
+
+### Clone repository
+
+Remember to set the `RV_ROOT` environment variable, which is required to generate a VeeR-EL2 Core configuration files.
+
+    git clone git@github.com:chipsalliance/Cores-VeeR-EL2.git
+    cd Cores-Veer-EL2
+    export RV_ROOT=$(pwd)
+
+### Prepare python virtual environment
+
+    cd $RV_ROOT/verification
+    python -m venv venv
+    source venv/bin/activate
+    pip install -r requirements.txt
+
+### Patch cocotb
+
+Due to issues with Verilator's `--timing` options, the `cocotb/share/lib/verilator.cpp` must be patched. The `--timing` option is only required if the HDL code contains timing structures.
+
+TODO: update link to upstream when patch is merged
+
+    cd $RV_ROOT/third_party
+    git clone https://github.com/antmicro/cocotb
+    git checkout mczyz/verilator-patch-timing
+    pip install -e $RV_ROOT/third_party/cocotb
+
+### Install Verilator
+
+Verification tests were run with Verilator-5.0.10. Installation instruction is avaialable in the Verilator's User Guide:
+
+    https://veripool.org/guide/latest/install.html
+
+## Tests
+
+All tests are wrapped in a pytest, which can be executed with:
+
+    python -m pytest -sv <test_name>
+
+If you want to generate html reports, it is recommended to use:
+
+    python -m pytest -v <test_name> --html=index.html
+
+If you want to generate a mardkown report, it is recommended to use:
+
+    python -m pytest -v <test_name> --md=test.md
+
+### PyUVM
+
+The PyUVM test uses Makefile located in the verification directory:
+
+    cd $RV_ROOT/verification
+    UVM_TEST=<valid_test_name> make all
+
+Also available targets are:
+
+    make clean
+    make verilator-pyuvm
+
+Run all tests:
+
+    python -m pytest -sv test.py
+
diff --git a/verification/__init__.py b/verification/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/verification/requirements.txt b/verification/requirements.txt
new file mode 100644
index 00000000000..e02d6b69ede
--- /dev/null
+++ b/verification/requirements.txt
@@ -0,0 +1,13 @@
+# Installing custom cocotb due to:
+# cocotb scheduler incorrectly handles simulation time
+# when Verilator is used with flag --timing
+${RV_ROOT}/third_party/cocotb
+# cocotb==1.7.2
+cocotb-bus==0.2.1
+cocotb-coverage==1.1.0
+cocotb-test==0.2.4
+pytest
+pytest-html
+pytest-timeout
+pytest-md
+pyuvm
diff --git a/verification/test_pyuvm/Makefile b/verification/test_pyuvm/Makefile
new file mode 100755
index 00000000000..ad637f49837
--- /dev/null
+++ b/verification/test_pyuvm/Makefile
@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2020 Western Digital Corporation or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+NPROC = $$((`nproc`-1))
+
+# -------------------------------------
+# VeeR Configuration
+# -------------------------------------
+CONF_PARAMS = -set build_axi4
+
+# Allow snapshot override
+target = default
+snapshot = $(target)
+
+# Check for RV_ROOT
+ifeq (,$(wildcard ${RV_ROOT}/configs/veer.config))
+$(error env var RV_ROOT does not point to a valid dir! Exiting!)
+endif
+
+# Allow tool override
+VEER_CONFIG = ${RV_ROOT}/configs/veer.config
+BUILD_DIR = snapshots/${snapshot}
+
+# If define files do not exist, then run veer.config.
+${BUILD_DIR}/defines.h:
+	BUILD_PATH=${BUILD_DIR} ${VEER_CONFIG} -target=$(target) $(CONF_PARAMS)
+	echo '`undef RV_ASSERT_ON' >> ${BUILD_DIR}/common_defines.vh
+
+# -------------------------------------
+# cocotb setup
+# -------------------------------------
+COCOTB_DIR = ${RV_ROOT}/third_party/cocotb
+COCOTB_SHARE_DIR = ${COCOTB_DIR}/cocotb/share
+
+COCOTB_HDL_TIMEUNIT ?= 1ns
+COCOTB_HDL_TIMEPRECISION ?= 1ps
+
+# -------------------------------------
+# Testbench setup
+# -------------------------------------
+UVM_TEST ?= test_irq.test_irq
+SIM_DIR ?= sim
+
+TOP_MODULE = el2_veer_wrapper
+
+VERILOG_DEFINE_FILES = $(BUILD_DIR)/common_defines.vh
+VERILOG_DEFINE_FILES += ${RV_ROOT}/design/include/el2_def.sv
+VERILOG_DEFINE_FILES += $(BUILD_DIR)/el2_pdef.vh
+
+VERILOG_INCLUDE_DIRS = ${BUILD_DIR}
+
+# -------------------------------------
+# Compilation/simulation configuration
+# -------------------------------------
+VERILATOR = verilator
+VERILATOR_SIM_DEBUG ?= 0
+VERILATOR_TRACE ?= 0
+
+EXTRA_ARGS += --timescale $(COCOTB_HDL_TIMEUNIT)/$(COCOTB_HDL_TIMEPRECISION) --timing
+COMPILE_ARGS += -CFLAGS -std=c++20 --vpi --public-flat-rw --prefix Vtop -o Vtop
+COMPILE_ARGS += -LDFLAGS "-Wl,-rpath,$(shell cocotb-config --lib-dir) -L$(shell cocotb-config --lib-dir) -lcocotbvpi_verilator"
+COMPILE_ARGS += $(addprefix +incdir+, $(VERILOG_INCLUDE_DIRS))
+WARNING_ARGS += -Wno-WIDTH -Wno-UNOPTFLAT
+
+FLIST_FILE = ${RV_ROOT}/testbench/flist
+
+ifeq ($(VERILATOR_SIM_DEBUG), 1)
+  COMPILE_ARGS += --debug -CFLAGS "-DVL_DEBUG -DVERILATOR_SIM_DEBUG -g"
+  PLUSARGS += +verilator+debug
+  BUILD_ARGS += OPT_FAST=-Og OPT_SLOW=-Og OPT_GLOBAL=-Og
+endif
+
+ifeq ($(VERILATOR_TRACE),1)
+  EXTRA_ARGS += --trace --trace-structs
+endif
+
+# Coverage reporting
+COVERAGE ?= all
+ifeq ("$(COVERAGE)", "all")
+    VERILATOR_COVERAGE = --coverage
+else ifeq ("$(COVERAGE)", "branch")
+    VERILATOR_COVERAGE = --coverage-line
+else ifeq ("$(COVERAGE)", "toggle")
+    VERILATOR_COVERAGE = --coverage-toggle
+else ifeq ("$(COVERAGE)", "functional")
+    VERILATOR_COVERAGE = --coverage-user
+else ifneq ("$(COVERAGE)", "")
+    $(error Unknown COVERAGE value '$(COVERAGE)')
+endif
+
+# -------------------------------------
+# Make PyUVM test with Verilator
+# -------------------------------------
+$(SIM_DIR):
+	mkdir -p $@
+
+$(SIM_DIR)/Vtop.mk: $(COCOTB_SHARE_DIR)/lib/verilator/verilator.cpp | $(SIM_DIR)
+	$(VERILATOR) --cc --exe -Mdir $(SIM_DIR) \
+	${COMPILE_ARGS} ${EXTRA_ARGS} \
+	${VERILOG_DEFINE_FILES} -f ${FLIST_FILE} \
+	${WARNING_ARGS} \
+	--top-module $(TOP_MODULE) \
+	$(VERILATOR_COVERAGE) \
+	$(COCOTB_SHARE_DIR)/lib/verilator/verilator.cpp
+
+$(SIM_DIR)/Vtop: $(SIM_DIR)/Vtop.mk
+	$(MAKE) -j${NPROC} -C $(SIM_DIR) $(BUILD_ARGS) -f Vtop.mk
+
+verilator-pyuvm: ${BUILD_DIR}/defines.h $(SIM_DIR)/Vtop.mk $(SIM_DIR)/Vtop
+	MODULE=$(UVM_TEST) $(SIM_DIR)/Vtop
+
+# -------------------------------------
+# Standard recipes
+# -------------------------------------
+
+all: clean verilator-pyuvm
+
+clean:
+	rm -rf obj_dir assets sim snapshots coverage.dat results.xml console.log exec.log *.cpp.s *.dis *.exe *.o *.map *.hex trace_port.csv verilator-cocotb-build
+
+help:
+	@echo Make sure the environment variable RV_ROOT is set.
+	@echo Expected usage: make verilator-pyuvm
+
+.PHONY: help clean
+
diff --git a/verification/test_pyuvm/__init__.py b/verification/test_pyuvm/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/verification/test_pyuvm/test_irq/irq_utils.py b/verification/test_pyuvm/test_irq/irq_utils.py
new file mode 100644
index 00000000000..9d4005cbea5
--- /dev/null
+++ b/verification/test_pyuvm/test_irq/irq_utils.py
@@ -0,0 +1,102 @@
+#
+# Copyright (c) 2023 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: BSD-2-Clause
+
+from cocotb.triggers import FallingEdge
+from cocotb.queue import Queue
+
+from pyuvm import *
+
+
+def get_int(signal):
+    try:
+        sig = int(signal.value)
+    except ValueError:
+        sig = 0
+    return sig
+
+
+class IrqBfm(metaclass=utility_classes.Singleton):
+    """
+        Interrupt Bus Functional Model
+
+        Drive:
+            el2_veer_wrapper {
+                input logic                      nmi_int
+                input logic                      timer_int
+                input logic                      soft_int
+                input logic [pt.PIC_TOTAL_INT:1] extintsrc_req
+            }
+        Receive:
+            el2_veer_wrapper {
+                output logic trace_rv_i_interrupt_ip
+            }
+
+    """
+
+    def __init__(self):
+        self.dut = cocotb.top
+        self.interrupt_driver_queue = Queue(maxsize=1)
+        self.interrupt_source_queue = Queue(maxsize=0)
+        self.trace_interrupt_queue = Queue(maxsize=0)
+        self.interrupts = (self.dut.nmi_int,
+                           self.dut.soft_int,
+                           self.dut.timer_int,
+                           self.dut.extintsrc_req)
+
+    async def send_interrupt_source(self, ints):
+        await self.interrupt_driver_queue.put(ints)
+
+    async def get_interrupt_source(self):
+        ints = await self.interrupt_source_queue.get()
+        return ints
+
+    async def get_trace_interrupt(self):
+        ints = await self.trace_interrupt_queue.get()
+        return ints
+
+    async def reset(self):
+        await FallingEdge(self.dut.clk)
+        self.dut.soft_int.value = 0
+        self.dut.timer_int.value = 0
+        self.dut.nmi_int.value = 0
+        self.dut.extintsrc_req.value = 0
+        await FallingEdge(self.dut.clk)
+
+    async def interrupt_driver_bfm(self):
+        self.dut.soft_int.value = 0
+        self.dut.timer_int.value = 0
+        self.dut.nmi_int.value = 0
+        self.dut.extintsrc_req.value = 0
+        while True:
+            await FallingEdge(self.dut.clk)
+            try:
+                ints = self.interrupt_driver_queue.get_nowait()
+                self.dut.soft_int.value = ints.soft
+                self.dut.timer_int.value = ints.timer
+                self.dut.nmi_int.value = ints.nmi
+                self.dut.extintsrc_req.value = ints.ext
+            except QueueEmpty:
+                pass
+
+    async def interrupt_source_bfm(self):
+        while True:
+            await FallingEdge(self.dut.clk)
+            item = (
+                get_int(self.dut.soft_int),
+                get_int(self.dut.timer_int),
+                get_int(self.dut.nmi_int),
+                get_int(self.dut.extintsrc_req)
+            )
+            self.interrupt_source_queue.put_nowait(item)
+
+    async def interrupt_trace_bfm(self):
+        while True:
+            await FallingEdge(self.dut.clk)
+            item = get_int(self.dut.trace_rv_i_interrupt_ip)
+            self.trace_interrupt_queue.put_nowait(item)
+
+    def start_bfm(self):
+        cocotb.start_soon(self.interrupt_driver_bfm())
+        cocotb.start_soon(self.interrupt_source_bfm())
+        cocotb.start_soon(self.interrupt_trace_bfm())
diff --git a/verification/test_pyuvm/test_irq/irq_uvm.py b/verification/test_pyuvm/test_irq/irq_uvm.py
new file mode 100644
index 00000000000..179e49468c3
--- /dev/null
+++ b/verification/test_pyuvm/test_irq/irq_uvm.py
@@ -0,0 +1,129 @@
+#
+# Copyright (c) 2023 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: BSD-2-Clause
+
+import random
+from pyuvm import *
+from .irq_utils import IrqBfm
+
+
+class IrqRandomSeq(uvm_sequence):
+    async def body(self):
+        seqr = ConfigDB().get(None, "", "SEQR")
+        random = vIrqSeq("random")
+        await random.start(seqr)
+
+
+class vIrqSeq(uvm_sequence):
+    async def body(self):
+        for i in range(10):
+            int_tr = IrqTriggerSeqItem("irq_trigger", 0, 0, 0, 0)
+            await self.start_item(int_tr)
+            int_tr.randomize()
+            await self.finish_item(int_tr)
+
+
+class IrqTriggerSeqItem(uvm_sequence_item):
+    def __init__(self, name, nmi, soft, timer, ext):
+        super().__init__(name)
+        self.nmi = nmi
+        self.soft = soft
+        self.timer = timer
+        self.ext = ext
+
+    def __eq__(self, other):
+        same = self.nmi == other.nmi and self.soft == other.soft and self.timer == other.timer and self.ext == other.ext
+        return same
+
+    def __str__(self):
+        return f"{self.get_name()} : NMI {self.nmi}, SOFT: {self.soft}, TIMER: {self.timer}, EXT: {self.ext:0x}"
+
+    def randomize(self):
+        self.nmi = random.randrange(2)
+        self.soft = random.randrange(2)
+        self.timer = random.randrange(2)
+        self.ext = random.randrange(2)
+
+
+class IrqMonitor(uvm_monitor):
+    def __init__(self, name, parent, method_name):
+        super().__init__(name, parent)
+        self.method_name = method_name
+
+    def build_phase(self):
+        self.ap = uvm_analysis_port("ap", self)
+        self.bfm = IrqBfm()
+        self.get_method = getattr(self.bfm, self.method_name)
+
+    async def run_phase(self):
+        while True:
+            datum = await self.get_method()
+            self.logger.debug(f"MONITORED {datum}")
+            self.ap.write(datum)
+
+
+class Scoreboard(uvm_component):
+    def build_phase(self):
+        self.interrupt_source_fifo = uvm_tlm_analysis_fifo("interrupt_source_fifo",self)
+        self.interrupt_source_get_port = uvm_get_port("interrupt_source_get_port",self)
+        self.interrupt_source_export = self.interrupt_source_fifo.analysis_export
+
+        self.trace_interrupt_fifo = uvm_tlm_analysis_fifo("trace_interrupt_fifo",self)
+        self.trace_interrupt_get_port = uvm_get_port("trace_interrupt_get_port",self)
+        self.trace_interrupt_export = self.trace_interrupt_fifo.analysis_export
+
+    def connect_phase(self):
+        self.interrupt_source_get_port.connect(self.interrupt_source_fifo.get_export)
+        self.trace_interrupt_get_port.connect(self.trace_interrupt_fifo.get_export)
+
+    def check_phase(self):
+        passed = True
+        try:
+            self.errors = ConfigDB().get(self, "", "CREATE_ERRORS")
+        except UVMConfigItemNotFound:
+            self.errors = False
+        assert passed
+
+
+class IrqDriver(uvm_driver):
+    def build_phase(self):
+        self.ap = uvm_analysis_port("ap", self)
+
+    def start_of_simulation_phase(self):
+        self.bfm = IrqBfm()
+
+    async def initialize_tb(self):
+        await self.bfm.reset()
+        self.bfm.start_bfm()
+
+    async def run_phase(self):
+        await self.initialize_tb()
+        while True:
+            ints = await self.seq_item_port.get_next_item()
+            await self.bfm.send_interrupt_source(ints)
+            result = await self.bfm.get_trace_interrupt()
+            self.ap.write(result)
+            self.seq_item_port.item_done()
+
+class IrqAgent(uvm_agent):
+    def build_phase(self):
+        self.seqr = uvm_sequencer("seqr", self)
+        ConfigDB().set(None, "*", "SEQR", self.seqr)
+        self.monitor = IrqMonitor("int_monitor", self, "get_interrupt_source")
+        self.driver = IrqDriver("int_driver", self)
+
+    def connect_phase(self):
+        # Driver takes sequence items from sequencer
+        self.driver.seq_item_port.connect(self.seqr.seq_item_export)
+
+class VeerEl2Env(uvm_env):
+    def build_phase(self):
+        self.scoreboard = Scoreboard("scoreboard", self)
+        self.agent = IrqAgent("agent",self)
+
+    def connect_phase(self):
+        # Monitor pushes observed data to the scoreboard
+        self.agent.monitor.ap.connect(self.scoreboard.interrupt_source_export)
+
+        # Driver
+        self.agent.driver.ap.connect(self.scoreboard.trace_interrupt_export)
diff --git a/verification/test_pyuvm/test_irq/test_irq.py b/verification/test_pyuvm/test_irq/test_irq.py
new file mode 100644
index 00000000000..5e2182824ca
--- /dev/null
+++ b/verification/test_pyuvm/test_irq/test_irq.py
@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2023 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: BSD-2-Clause
+
+import pyuvm
+from pyuvm import *
+from .irq_uvm import VeerEl2Env, IrqRandomSeq
+from cocotb.clock import Clock
+
+@pyuvm.test()
+class BaseTest(uvm_test):
+    def build_phase(self):
+        self.set_default_logging_level(logging.DEBUG)
+        self.env = VeerEl2Env("env", self)
+
+    def end_of_elaboration_phase(self):
+        self.test_all = IrqRandomSeq.create("test_irq")
+
+    async def run_phase(self):
+        self.raise_objection()
+        clock = Clock(cocotb.top.clk, 10, units="ns")
+        cocotb.start_soon(clock.start(start_high=False))
+        await self.test_all.start()
+        self.drop_objection()
diff --git a/verification/test_pyuvm/test_pyuvm.py b/verification/test_pyuvm/test_pyuvm.py
new file mode 100644
index 00000000000..c3bf0b35101
--- /dev/null
+++ b/verification/test_pyuvm/test_pyuvm.py
@@ -0,0 +1,27 @@
+import pytest
+import os
+import subprocess
+
+
+class TestPyUVM():
+
+    @pytest.mark.parametrize("UVM_TEST", ["test_irq.test_irq"])
+    def test_pyuvm(self, UVM_TEST):
+
+        os.environ["UVM_TEST"] = UVM_TEST
+        py_command = []
+        py_command += [
+            "make all",
+        ]
+        py_command = " ".join(py_command)
+
+        print(f"\n----- PyTest -----")
+        print(f":: py_command >> {py_command}")
+        p = subprocess.run(py_command, shell=True,
+                           executable="/bin/bash", bufsize=0)
+        print(f"\n------------------")
+
+        print(f"----- Subprocess Summary -----")
+        print(f"p.check_returncode")
+        p.check_returncode()
+        print(f"------------------------------")
\ No newline at end of file

From e1e2a5fd848ce3c305487e58a42b319eb7d121a5 Mon Sep 17 00:00:00 2001
From: Michal Czyz <mczyz@antmicro.com>
Date: Mon, 5 Jun 2023 11:02:40 +0200
Subject: [PATCH 31/34] Add mock test for faster CI debugging

---
 verification/test_debug/test_debug.py | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 verification/test_debug/test_debug.py

diff --git a/verification/test_debug/test_debug.py b/verification/test_debug/test_debug.py
new file mode 100644
index 00000000000..2eee0d253b7
--- /dev/null
+++ b/verification/test_debug/test_debug.py
@@ -0,0 +1,11 @@
+#
+# Copyright (c) 2023 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: BSD-2-Clause
+
+import pytest
+import subprocess
+
+class TestDebug():
+    def test_debug(self):
+        print("This test returns true")
+        assert True == True

From ef0cd6b1d6ec13862ec582d20f7c5933c6ee7466 Mon Sep 17 00:00:00 2001
From: Michal Czyz <mczyz@antmicro.com>
Date: Mon, 5 Jun 2023 11:02:59 +0200
Subject: [PATCH 32/34] Run tests in CI

---
 .github/workflows/verification.yml            | 143 ++++++++++++++++++
 .github/workflows/verilator-nightly-build.yml |  69 +++++++++
 .gitignore                                    |   6 +
 3 files changed, 218 insertions(+)
 create mode 100644 .github/workflows/verification.yml
 create mode 100644 .github/workflows/verilator-nightly-build.yml

diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml
new file mode 100644
index 00000000000..e781a59568e
--- /dev/null
+++ b/.github/workflows/verification.yml
@@ -0,0 +1,143 @@
+name: VeeR-EL2 verification
+
+on:
+  workflow_call:
+
+env:
+  VERILATOR_VERSION: v5.010
+  MOCK_VERIFICATION_RUN: false
+
+jobs:
+  tests:
+    name: Run test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test: ["test_pyuvm"]
+        # test: ["test_pyuvm", "test_cocotb","test_dev"]
+    env:
+      CCACHE_DIR: "/opt/verilator_cache/.cache/"
+      DEBIAN_FRONTEND: "noninteractive"
+    steps:
+      - name: Setup repository
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup Cache Metadata
+        id: cache_metadata
+        run: |
+          cache_restore_name=cache_verilator_${{ env.VERILATOR_VERSION }}
+          cache_date=$(date +"%Y_%m_%d")
+          cache_name=${cache_restore_name}_${cache_date}
+
+          cache_verification_name=cache_verification_${{ matrix.test }}
+
+          echo "Cache restore name: "${cache_restore_name}
+          echo "Cache date: "${cache_date}
+          echo "Cache name: "${cache_name}
+          echo "Cache verification name: "${cache_verification_name}
+
+          echo "cache_restore_name=${cache_restore_name}" >> "$GITHUB_ENV"
+          echo "cache_date=${cache_date}" >> "$GITHUB_ENV"
+          echo "cache_name=${cache_name}" >> "$GITHUB_ENV"
+          echo "cache_verification_name=${cache_verification_name}" >> "$GITHUB_ENV"
+
+      - name: Restore verilator cache
+        id: cache-verilator-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            /opt/verilator
+            ${{ env.CCACHE_DIR }}
+          key: ${{ env.cache_name }}
+          restore-keys: ${{ env.cache_restore_name }}
+
+      - name: Setup verification test cache
+        uses: actions/cache@v3
+        id: cache-verification-setup
+        with:
+          path: |
+            /opt/verilator
+            ${{ env.CCACHE_DIR }}
+          key: ${{ env.cache_verification_name }}
+
+      - name: Install prerequisities
+        run: |
+          sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
+            autoconf automake autotools-dev \
+            bc bison build-essential \
+            ccache cpanminus curl \
+            flex \
+            gawk gcc-riscv64-unknown-elf git gperf \
+            help2man \
+            libexpat-dev libfl-dev libfl2 libgmp-dev \
+            libmpc-dev libmpfr-dev libpython3-all-dev libtool \
+            ninja-build \
+            patchutils python3 python3-dev python3-pip \
+            texinfo \
+            zlib1g zlib1g-dev
+          sudo cpanm Bit::Vector
+
+      # If PR is merged, can change to main branch
+      # https://github.com/cocotb/cocotb/pull/3316
+      - name: Get patched cocotb
+        uses: actions/checkout@v3
+        with:
+          repository: antmicro/cocotb
+          path: third_party/cocotb
+          ref: mczyz/verilator-patch-timing
+
+      - name: Run ${{ matrix.test }}
+        if: ${{ env.MOCK_VERIFICATION_RUN == 'false' }}
+        run: |
+          export PATH=/opt/verilator/bin:$PATH
+          export RV_ROOT=`pwd`
+          export PYTHONUNBUFFERED=1
+          pip3 install meson
+          pip3 install -r $RV_ROOT/verification/requirements.txt
+          pushd $RV_ROOT/verification/${{ matrix.test }}
+            python -m pytest ${{ matrix.test }}.py -sv --html=${{ matrix.test }}.html --md=$GITHUB_STEP_SUMMARY
+            mkdir -p webpage_${{ matrix.test }}
+            mv ${{ github.workspace }}/verification/${{ matrix.test }}/${{ matrix.test }}.html webpage_${{ matrix.test }}
+            mv ${{ github.workspace }}/verification/${{ matrix.test }}/assets webpage_${{ matrix.test }}
+            mv webpage_${{matrix.test}} ${{ github.workspace }}/
+
+            # TODO: fix once coverage is implemented
+            # mkdir -p coverage_${{ matrix.test }}
+            # mkdir -p ${{ github.workspace }}/results_${{ matrix.test }}
+            # mv coverage_${{matrix.test}} ${{ github.workspace }}/
+            # ./github/scripts/convert_coverage_data.sh all ${{ github.workspace }}/coverage ${{ github.workspace }}/results_${{ matrix.test }} coverage_${{ matrix.test }}
+
+          popd
+
+      - name: Run mock ${{ matrix.test }}
+        if: ${{ env.MOCK_VERIFICATION_RUN == 'true' }}
+        run: |
+          export PYTHONUNBUFFERED=1
+          export RV_ROOT=`pwd`
+          pip3 install -r $RV_ROOT/verification/requirements.txt
+          pushd $RV_ROOT/verification/test_debug
+            python -m pytest test_debug.py -v --html=${{ matrix.test }}.html --md=$GITHUB_STEP_SUMMARY
+            mkdir -p webpage_${{ matrix.test }}
+            mv ${{ github.workspace }}/verification/test_debug/${{ matrix.test }}.html webpage_${{ matrix.test }}
+            mv ${{ github.workspace }}/verification/test_debug/assets webpage_${{ matrix.test }}
+            mv webpage_${{matrix.test}} ${{ github.workspace }}/
+          popd
+
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: results
+          path: |
+            webpage_*
+
+      # TODO: fix once coverage is implemented
+      # - name: Upload coverage artifacts
+      #   if: always()
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: verification_tests_coverage_data
+      #     path: |
+      #       ./results/*.info
\ No newline at end of file
diff --git a/.github/workflows/verilator-nightly-build.yml b/.github/workflows/verilator-nightly-build.yml
new file mode 100644
index 00000000000..03aa87377d2
--- /dev/null
+++ b/.github/workflows/verilator-nightly-build.yml
@@ -0,0 +1,69 @@
+name: Verilator Nightly Build
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  verilator:
+    name: Build Verilator
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        VERILATOR_VERSION: [v5.010]
+    env:
+      CCACHE_DIR: "/opt/verilator_cache/.cache/"
+      TOOL_NAME: verilator
+      DEBIAN_FRONTEND: "noninteractive"
+
+    steps:
+      - name: Install prerequisities
+        run: |
+          sudo apt -qqy update && sudo apt -qqy --no-install-recommends install \
+          autoconf automake autotools-dev \
+          bc bison build-essential \
+          ccache curl \
+          flex \
+          gawk git gperf \
+          help2man \
+          libexpat-dev libfl-dev libfl2 libgmp-dev \
+          libmpc-dev libmpfr-dev libtool \
+          ninja-build \
+          patchutils python3 python3-pip \
+          texinfo \
+          zlib1g zlib1g-dev
+
+      - name: Setup Cache Metadata
+        id: cache_metadata
+        run: |
+          cache_date=$(date +"%Y_%m_%d")
+          cache_name=cache_${{ env.TOOL_NAME }}_${{ matrix.VERILATOR_VERSION }}_$cache_date
+          echo "Cache date: "$cache_date
+          echo "Cache name: "$cache_name
+          ccache_dir=$(ccache --show-config | grep cache_dir | awk '{print $4}')
+          echo "Ccache dir=$ccache_dir"
+          echo "cache_date=$cache_date" >> "$GITHUB_ENV"
+          echo "cache_name=$cache_name" >> "$GITHUB_ENV"
+          # echo "ccache_dir=$ccache_dir" >> "$GITHUB_ENV"
+
+      - name: Setup cache
+        uses: actions/cache@v3
+        id: cache
+        timeout-minutes: 3
+        with:
+          path: |
+            /opt/verilator
+            ${{ env.CCACHE_DIR }}
+          key: ${{ env.cache_name }}
+
+      - name: Build Verilator
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: |
+          git clone https://github.com/verilator/verilator
+          pushd verilator
+            git checkout ${{ matrix.VERILATOR_VERSION }}
+            autoconf
+            ./configure --prefix=/opt/verilator
+            make -j `nproc`
+            make install
+          popd
diff --git a/.gitignore b/.gitignore
index ab04acbab67..e2176adf9ef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,9 @@ obj_dir
 verilator-build
 program.hex
 snapshots
+__pycache__
+sim_build
+venv
+results.xml
+verification/sim
+verilator-cocotb-build

From 1df32bfa28732a781c5759ba6392e0185d6a5b91 Mon Sep 17 00:00:00 2001
From: Michal Czyz <mczyz@antmicro.com>
Date: Fri, 16 Jun 2023 20:29:26 +0200
Subject: [PATCH 33/34] Bump versions of actions for node16

---
 .github/workflows/riscv-dv.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/riscv-dv.yml b/.github/workflows/riscv-dv.yml
index a1a03d30c97..21a3ac3e473 100644
--- a/.github/workflows/riscv-dv.yml
+++ b/.github/workflows/riscv-dv.yml
@@ -24,12 +24,12 @@ jobs:
 
       - name: Create Cache Timestamp
         id: cache_timestamp
-        uses: nanzm/get-time-action@v1.1
+        uses: nanzm/get-time-action@v2.0
         with:
           format: 'YYYY-MM-DD-HH-mm-ss'
 
       - name: Setup cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         timeout-minutes: 3
         continue-on-error: true
         with:
@@ -69,12 +69,12 @@ jobs:
     steps:
       - name: Create Cache Timestamp
         id: cache_timestamp
-        uses: nanzm/get-time-action@v1.1
+        uses: nanzm/get-time-action@v2.0
         with:
           format: 'YYYY-MM-DD-HH-mm-ss'
 
       - name: Setup cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         timeout-minutes: 3
         continue-on-error: true
         with:
@@ -116,12 +116,12 @@ jobs:
 
       - name: Create Cache Timestamp
         id: cache_timestamp
-        uses: nanzm/get-time-action@v1.1
+        uses: nanzm/get-time-action@v2.0
         with:
           format: 'YYYY-MM-DD-HH-mm-ss'
 
       - name: Setup cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         timeout-minutes: 3
         continue-on-error: true
         with:
@@ -168,12 +168,12 @@ jobs:
 
       - name: Create Cache Timestamp
         id: cache_timestamp
-        uses: nanzm/get-time-action@v1.1
+        uses: nanzm/get-time-action@v2.0
         with:
           format: 'YYYY-MM-DD-HH-mm-ss'
 
       - name: Setup cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         timeout-minutes: 3
         continue-on-error: true
         with:
@@ -259,7 +259,7 @@ jobs:
           popd
 
       - name: Setup repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           submodules: recursive
           path: veer
@@ -270,12 +270,12 @@ jobs:
 
       - name: Create Cache Timestamp
         id: cache_timestamp
-        uses: nanzm/get-time-action@v1.1
+        uses: nanzm/get-time-action@v2.0
         with:
           format: 'YYYY-MM-DD-HH-mm-ss'
 
       - name: Setup cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         timeout-minutes: 3
         continue-on-error: true
         with:

From 133ee35d266fdb01c0f22676d63c227e25441e2c Mon Sep 17 00:00:00 2001
From: Maciej Kurc <mkurc@antmicro.com>
Date: Mon, 19 Jun 2023 11:27:33 +0200
Subject: [PATCH 34/34] Revert addition of the scan_rst_l input

Signed-off-by: Maciej Kurc <mkurc@antmicro.com>
---
 design/el2_veer.sv         | 8 ++++----
 design/el2_veer_wrapper.sv | 1 -
 testbench/tb_top.sv        | 1 -
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/design/el2_veer.sv b/design/el2_veer.sv
index 626209db9a3..880aab38c98 100644
--- a/design/el2_veer.sv
+++ b/design/el2_veer.sv
@@ -32,7 +32,7 @@ import el2_pkg::*;
    input logic [31:1]           rst_vec,
    input logic                  nmi_int,
    input logic [31:1]           nmi_vec,
-   output logic                 core_rst_l,   // This is "rst_l & (scan_rst_l | scan_mode)"
+   output logic                 core_rst_l,   // This is "rst_l | dbg_rst_l"
 
    output logic                 active_l2clk,
    output logic                 free_l2clk,
@@ -380,8 +380,7 @@ import el2_pkg::*;
    input logic [pt.PIC_TOTAL_INT:1]           extintsrc_req,
    input logic                   timer_int,
    input logic                   soft_int,
-   input logic                   scan_mode,
-   input logic                   scan_rst_l
+   input logic                   scan_mode
 );
 
 
@@ -857,7 +856,8 @@ import el2_pkg::*;
 
    // -----------------   DEBUG END -----------------------------
 
-   assign core_rst_l = rst_l & (scan_rst_l | scan_mode);
+   assign core_rst_l = rst_l & (dbg_core_rst_l | scan_mode);
+
    // fetch
    el2_ifu #(.pt(pt)) ifu (
                             .clk(active_l2clk),
diff --git a/design/el2_veer_wrapper.sv b/design/el2_veer_wrapper.sv
index f80f4ba5e6c..36180b36696 100644
--- a/design/el2_veer_wrapper.sv
+++ b/design/el2_veer_wrapper.sv
@@ -330,7 +330,6 @@ import el2_pkg::*;
    input logic                             i_cpu_run_req, // Async restart req to CPU
    output logic                            o_cpu_run_ack, // Core response to run req
    input logic                             scan_mode,     // To enable scan mode
-   input logic                             scan_rst_l,
    input logic                             mbist_mode     // to enable mbist
 );
 
diff --git a/testbench/tb_top.sv b/testbench/tb_top.sv
index a9665282064..ca4f3359310 100644
--- a/testbench/tb_top.sv
+++ b/testbench/tb_top.sv
@@ -752,7 +752,6 @@ el2_veer_wrapper rvtop (
     .soft_int               ('0),
     .core_id                ('0),
     .scan_mode              ( 1'b0 ),         // To enable scan mode
-    .scan_rst_l             ( 1'b1 ),
     .mbist_mode             ( 1'b0 )        // to enable mbist
 
 );