From 57b7a73c49870fa8b43f3fc876535e9546f8ea1b Mon Sep 17 00:00:00 2001 From: Pascal Gouedo Date: Mon, 16 Oct 2023 10:15:36 +0200 Subject: [PATCH] Updated Embench benchmark framework to be able to run it on different CV32E40P configurations. Signed-off-by: Pascal Gouedo --- bin/run_embench.py | 101 ++++- .../config/corev32/chips/size/chip.cfg | 4 +- .../config/corev32/chips/speed/chip.cfg | 4 +- .../embench/config/corev32_pulp/arch.cfg | 67 +++ .../boards/corev32_pulp/board.cfg | 69 ++++ .../boards/corev32_pulp/boardsupport.c | 21 + .../boards/corev32_pulp/boardsupport.h | 21 + .../config/corev32_pulp/chips/size/chip.cfg | 86 ++++ .../corev32_pulp/chips/size/chipsupport.c | 44 ++ .../corev32_pulp/chips/size/chipsupport.h | 27 ++ .../config/corev32_pulp/chips/speed/chip.cfg | 87 ++++ .../corev32_pulp/chips/speed/chipsupport.c | 51 +++ .../corev32_pulp/chips/speed/chipsupport.h | 27 ++ .../tests/embench/pylib/benchmark_size.py | 384 +++++++++++++++++ .../tests/embench/pylib/benchmark_speed.py | 387 ++++++++++++++++++ cv32e40p/tests/embench/pylib/run_corev32.py | 23 +- mk/uvmt/uvmt.mk | 5 + 17 files changed, 1390 insertions(+), 18 deletions(-) create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/arch.cfg create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/board.cfg create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/boardsupport.c create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/boardsupport.h create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/chips/size/chip.cfg create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/chips/size/chipsupport.c create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/chips/size/chipsupport.h create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chip.cfg create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chipsupport.c create mode 100644 cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chipsupport.h create mode 100755 cv32e40p/tests/embench/pylib/benchmark_size.py create mode 100755 cv32e40p/tests/embench/pylib/benchmark_speed.py diff --git a/bin/run_embench.py b/bin/run_embench.py index 281e064c1c..6523552922 100755 --- a/bin/run_embench.py +++ b/bin/run_embench.py @@ -52,7 +52,6 @@ def main(): parser = build_parser() args = parser.parse_args() - paths = build_paths(args.core) if args.debug == 'YES': logger.setLevel(logging.DEBUG) @@ -61,6 +60,14 @@ def main(): logger.info('Must specify a core to benchmark') sys.exit(1) + if args.cfg == 'default': + core_config = 'corev32' + elif args.cfg == 'pulp': + core_config = 'corev32_pulp' + else: + logger.info(f"Invalid config selected: {args.cfg}, must be 'default' or 'pulp'") + sys.exit(1) + if args.ccomp == 'notset': logger.info('Must specify a c compiler to benchmark') sys.exit(1) @@ -77,6 +84,14 @@ def main(): logger.info(f"Invalid 'parallel' option: {args.parallel}, must be 'YES' or 'NO'") sys.exit(1) + if args.absolute == 'YES': + absolute = True + elif args.absolute == 'NO': + absolute = False + else: + logger.info(f"Invalid 'absolute' option: {args.absolute}, must be 'YES' or 'NO'") + sys.exit(1) + if args.build_only == 'YES': build_only = True elif args.build_only == 'NO': @@ -85,6 +100,8 @@ def main(): logger.info(f"Invalid 'build_only' option: {args.build_only}, must be 'YES' or 'NO'") sys.exit(1) + paths = build_paths(args.core, core_config, args.logdir, args.builddir) + logger.info("Starting EMBench for core-v-verif") logger.info(f"Benchmarking core: {args.core}") logger.info(f"Type of benchmark to run: {args.type}\n\n") @@ -142,18 +159,37 @@ def main(): except: logger.fatal('EMBench python module copy failed') + # copy python module + logger.info(f"Copying {paths['libpy']}/benchmark_speed.py to {paths['embench']}/benchmark_speed.py") + try: + subprocess.run( + ['cp', '-pf', f"{paths['libpy']}/benchmark_speed.py", f"{paths['embench']}/benchmark_speed.py"] + ) + except: + logger.fatal('EMBench benchmark_speed python script copy failed') + + logger.info(f"Copying {paths['libpy']}/benchmark_size.py to {paths['embench']}/benchmark_size.py") + try: + subprocess.run( + ['cp', '-pf', f"{paths['libpy']}/benchmark_size.py", f"{paths['embench']}/benchmark_size.py"] + ) + except: + logger.fatal('EMBench benchmark_speed python script copy failed') + # ---------------------------------------------------------------------------------------------- # build benchmark object files (build_all.py) # ---------------------------------------------------------------------------------------------- cmd = [f"{paths['embench']}/build_all.py", - '--arch=corev32', - '--board=corev32', + f'--arch={core_config}', + f'--board={core_config}', f'--cflags=-I{paths["bsp"]}', f'--chip={args.type}', f'--cc={args.ccomp}', f'--warmup-heat=0', f'--cpu-mhz={args.cpu_mhz}', f'--ldflags=-T{paths["bsp"]}/link.ld', + f'--builddir={args.builddir}', + f'--logdir={args.logdir}', '--clean'] logger.info(f"Calling build script: {' '.join(cmd)}") try: @@ -219,14 +255,24 @@ def main(): logger.info(f"Starting benchmarking of {args.type}") if args.type == 'speed': - arglist = [f"{paths['embench']}/benchmark_speed.py", '--target-module=run_corev32', - f'--cpu-mhz={args.cpu_mhz}', f'--make-path={paths["make"]}', + arglist = [f"{paths['embench']}/benchmark_speed.py", + f'--target-module=run_corev32', + f'-cfg={args.cfg}', + f'--cpu-mhz={args.cpu_mhz}', + f'--make-path={paths["make"]}', + f'--builddir={args.builddir}', + f'--logdir={args.logdir}', f'--timeout={args.timeout}', f'--simulator={args.simulator}'] if parallel: - arglist.append(f'--sim-parallel') + arglist.append(f'--sim-parallel') else: - arglist = [f"{paths['embench']}/benchmark_size.py"] + arglist = [f"{paths['embench']}/benchmark_size.py", + f'--builddir={args.builddir}', + f'--logdir={args.logdir}'] + + if absolute: + arglist.append(f'--absolute') try: logger.info(f"Running: {' '.join(arglist)}") @@ -279,6 +325,12 @@ def build_parser(): help='Core to benchmark' ) + parser.add_argument( + '-cfg', + default='default', + help='Core configuration to benchmark' + ) + parser.add_argument( '-cc', '--ccomp', @@ -286,6 +338,15 @@ def build_parser(): help='C compiler for benchmark' ) + parser.add_argument( + '--absolute', + default='NO', + help=( + 'Set this option to "YES" to report absolute numbers\n'+ + 'makefile alias: EMB_ABSOLUTE' + ) + ) + parser.add_argument( '--parallel', default='NO', @@ -307,6 +368,20 @@ def build_parser(): ) ) + parser.add_argument( + '--builddir', + type=str, + default='bd', + help='Directory holding all the binaries', + ) + + parser.add_argument( + '--logdir', + type=str, + default='logs', + help='Directory in which to store logs', + ) + parser.add_argument( '-b', '--build-only', @@ -373,21 +448,21 @@ def build_parser(): return parser -def build_paths(core): +def build_paths(core, core_config, logdir, builddir): """map out necessary paths""" paths = dict() paths['cver'] = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) paths['core'] = paths['cver'] + '/' + core - paths['libcfg'] = paths['core'] + '/tests/embench/config/corev32' + paths['libcfg'] = paths['core'] + '/tests/embench/config/' + core_config paths['libpy'] = paths['core'] + '/tests/embench/pylib' paths['vlib'] = paths['core'] + '/vendor_lib' - paths['emb_logs'] = paths['core'] + '/vendor_lib/embench/logs' + paths['emb_logs'] = logdir paths['make'] = paths['core'] + '/sim/uvmt' paths['embench'] = paths['vlib'] + '/embench' - paths['emcfg'] = paths['embench'] + '/config/corev32' + paths['emcfg'] = paths['embench'] + '/config/' + core_config paths['empy'] = paths['embench'] + '/pylib' - paths['embrd'] = paths['emcfg'] + '/boards/corev32' - paths['emres'] = paths['embench'] + '/bd/src' + paths['embrd'] = paths['emcfg'] + '/boards/' + core_config + paths['emres'] = builddir + '/src' paths['bsp'] = paths['core'] + '/bsp' paths['testsem'] = paths['core'] + '/tests/programs/embench' diff --git a/cv32e40p/tests/embench/config/corev32/chips/size/chip.cfg b/cv32e40p/tests/embench/config/corev32/chips/size/chip.cfg index 12f894d3b5..ca520cb6a4 100644 --- a/cv32e40p/tests/embench/config/corev32/chips/size/chip.cfg +++ b/cv32e40p/tests/embench/config/corev32/chips/size/chip.cfg @@ -76,11 +76,11 @@ # - we garbage collect unused sections on linking cflags = [ - '-c', '-Os', '-ffunction-sections', '-mabi=ilp32', '-march=rv32imc' + '-c', '-Os', '-ffunction-sections', '-mabi=ilp32', '-march=rv32imc_zicsr' ] ldflags = [ - '-Wl,-gc-sections', '-Wl,-A,elf32lriscv', '-nostartfiles', '-nostdlib', '-mabi=ilp32', '-march=rv32imc' + '-Wl,-gc-sections', '-Wl,-A,elf32lriscv', '-nostartfiles', '-nostdlib', '-mabi=ilp32', '-march=rv32imc_zicsr' ] dummy_libs = ['crt0', 'libm', 'libc', 'libgcc'] diff --git a/cv32e40p/tests/embench/config/corev32/chips/speed/chip.cfg b/cv32e40p/tests/embench/config/corev32/chips/speed/chip.cfg index d8762b60b9..da0600a653 100644 --- a/cv32e40p/tests/embench/config/corev32/chips/speed/chip.cfg +++ b/cv32e40p/tests/embench/config/corev32/chips/speed/chip.cfg @@ -76,11 +76,11 @@ # - we garbage collect unused sections on linking cflags = [ - '-c', '-O2', '-ffunction-sections', '-mabi=ilp32', '-march=rv32im' + '-c', '-O2', '-ffunction-sections', '-mabi=ilp32', '-march=rv32im_zicsr' ] ldflags = [ - '-Wl,-gc-sections', '-Wl,-A,elf32lriscv', '-nostartfiles', '-mabi=ilp32', '-march=rv32im' + '-Wl,-gc-sections', '-Wl,-A,elf32lriscv', '-nostartfiles', '-mabi=ilp32', '-march=rv32im_zicsr' ] user_libs = ['-lm'] diff --git a/cv32e40p/tests/embench/config/corev32_pulp/arch.cfg b/cv32e40p/tests/embench/config/corev32_pulp/arch.cfg new file mode 100644 index 0000000000..0583bdfe6e --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/arch.cfg @@ -0,0 +1,67 @@ +############################################################################### +# +# Copyright 2020 OpenHW Group +# +# Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://solderpad.org/licenses/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 WITH SHL-2.0 +# +############################################################################### + +# This is a python setting of parameters for the architecture. The following +# parameters may be set (other keys are silently ignored). Defaults are shown +# in brackets +# - cc ('cc') +# - ld (same value as for cc) +# - cflags ([]) +# - ldflags ([]) +# - cc_define_pattern ('-D{0}') +# - cc_incdir_pattern ('-I{0}') +# - cc_input_pattern ('{0}') +# - cc_output_pattern ('-o {0}') +# - ld_input_pattern ('{0}') +# - ld_output_pattern ('-o {0}') +# - user_libs ([]) +# - dummy_libs ([]) +# - cpu_mhz (1) +# - warmup_heat (1) + +# The "flags" and "libs" parameters (cflags, ldflags, user_libs, dummy_libs) +# should be lists of arguments to be passed to the compile or link line as +# appropriate. Patterns are Python format patterns used to create arguments. +# Thus for GCC or Clang/LLVM defined constants can be passed using the prefix +# '-D', and the pattern '-D{0}' would be appropriate (which happens to be the +# default). + +# "user_libs" may be absolute file names or arguments to the linker. In the +# latter case corresponding arguments in ldflags may be needed. For example +# with GCC or Clang/LLVM is "-l" flags are used in "user_libs", the "-L" flags +# may be needed in "ldflags". + +# Dummy libs have their source in the "support" subdirectory. Thus if 'crt0' +# is specified, there should be a source file 'dummy-crt0.c' in the support +# directory. + +# There is no need to set an unused parameter, and this file may be empty to +# set no flags. + +# Parameter values which are duplicated in architecture, board, chip or +# command line are used in the following order of priority +# - default value +# - architecture specific value +# - chip specific value +# - board specific value +# - command line value + +# For flags, this priority is applied to individual flags, not the complete +# list of flags. diff --git a/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/board.cfg b/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/board.cfg new file mode 100644 index 0000000000..5d7d45435f --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/board.cfg @@ -0,0 +1,69 @@ +############################################################################### +# +# Copyright 2020 OpenHW Group +# +# Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://solderpad.org/licenses/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 WITH SHL-2.0 +# +############################################################################### + +# This is a python setting of parameters for the board. The following +# parameters may be set (other keys are silently ignored). Defaults are shown +# in brackets +# - cc ('cc') +# - ld (same value as for cc) +# - cflags ([]) +# - ldflags ([]) +# - cc_define_pattern ('-D{0}') +# - cc_incdir_pattern ('-I{0}') +# - cc_input_pattern ('{0}') +# - cc_output_pattern ('-o {0}') +# - ld_input_pattern ('{0}') +# - ld_output_pattern ('-o {0}') +# - user_libs ([]) +# - dummy_libs ([]) +# - cpu_mhz (1) +# - warmup_heat (1) + +# The "flags" and "libs" parameters (cflags, ldflags, user_libs, dummy_libs) +# should be lists of arguments to be passed to the compile or link line as +# appropriate. Patterns are Python format patterns used to create arguments. +# Thus for GCC or Clang/LLVM defined constants can be passed using the prefix +# '-D', and the pattern '-D{0}' would be appropriate (which happens to be the +# default). + +# "user_libs" may be absolute file names or arguments to the linker. In the +# latter case corresponding arguments in ldflags may be needed. For example +# with GCC or Clang/LLVM is "-l" flags are used in "user_libs", the "-L" flags +# may be needed in "ldflags". + +# Dummy libs have their source in the "support" subdirectory. Thus if 'crt0' +# is specified, there should be a source file 'dummy-crt0.c' in the support +# directory. + +# There is no need to set an unused parameter, and this file may be empty to +# set no flags. + +# Parameter values which are duplicated in architecture, board, chip or +# command line are used in the following order of priority +# - default value +# - architecture specific value +# - chip specific value +# - board specific value +# - command line value + +# For flags, this priority is applied to individual flags, not the complete +# list of flags. + +cpu_mhz = 1 diff --git a/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/boardsupport.c b/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/boardsupport.c new file mode 100644 index 0000000000..f2e674dea5 --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/boardsupport.c @@ -0,0 +1,21 @@ +/* +** +** Copyright 2020 OpenHW Group +** +** Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** https://solderpad.org/licenses/ +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +** +******************************************************************************* +*/ + +#include "boardsupport.h" + diff --git a/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/boardsupport.h b/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/boardsupport.h new file mode 100644 index 0000000000..6223cd6846 --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/boards/corev32_pulp/boardsupport.h @@ -0,0 +1,21 @@ +/* +** +** Copyright 2020 OpenHW Group +** +** Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** https://solderpad.org/licenses/ +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +** +******************************************************************************* +*/ + + + diff --git a/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chip.cfg b/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chip.cfg new file mode 100644 index 0000000000..458cfe11f0 --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chip.cfg @@ -0,0 +1,86 @@ +############################################################################### +# +# Copyright 2020 OpenHW Group +# +# Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://solderpad.org/licenses/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 WITH SHL-2.0 +# +############################################################################### + +# This is a python setting of parameters for the chip. The following +# parameters may be set (other keys are silently ignored). Defaults are shown +# in brackets +# - cc ('cc') +# - ld (same value as for cc) +# - cflags ([]) +# - ldflags ([]) +# - cc_define_pattern ('-D{0}') +# - cc_incdir_pattern ('-I{0}') +# - cc_input_pattern ('{0}') +# - cc_output_pattern ('-o {0}') +# - ld_input_pattern ('{0}') +# - ld_output_pattern ('-o {0}') +# - user_libs ([]) +# - dummy_libs ([]) +# - cpu_mhz (1) +# - warmup_heat (1) + +# The "flags" and "libs" parameters (cflags, ldflags, user_libs, dummy_libs) +# should be lists of arguments to be passed to the compile or link line as +# appropriate. Patterns are Python format patterns used to create arguments. +# Thus for GCC or Clang/LLVM defined constants can be passed using the prefix +# '-D', and the pattern '-D{0}' would be appropriate (which happens to be the +# default). + +# "user_libs" may be absolute file names or arguments to the linker. In the +# latter case corresponding arguments in ldflags may be needed. For example +# with GCC or Clang/LLVM is "-l" flags are used in "user_libs", the "-L" flags +# may be needed in "ldflags". + +# Dummy libs have their source in the "support" subdirectory. Thus if 'crt0' +# is specified, there should be a source file 'dummy-crt0.c' in the support +# directory. + +# There is no need to set an unused parameter, and this file may be empty to +# set no flags. + +# Parameter values which are duplicated in architecture, board, chip or +# command line are used in the following order of priority +# - default value +# - architecture specific value +# - chip specific value +# - board specific value +# - command line value + +# For flags, this priority is applied to individual flags, not the complete +# list of flags. + +# This is the generic framework for compilers, where the only common set up +# is: + +# '-c' is used to specify generation of object files when compiling a + +# each global data and function is put in its own section + +# - we garbage collect unused sections on linking + +cflags = [ + '-c', '-Os', '-ffunction-sections', '-mabi=ilp32', '-march=rv32imc_zicsr_zifencei_xcvhwlp_xcvmem_xcvmac_xcvbi_xcvalu_xcvsimd_xcvbitmanip' +] + +ldflags = [ + '-Wl,-gc-sections', '-Wl,-A,elf32lriscv', '-nostartfiles', '-nostdlib', '-mabi=ilp32', '-march=rv32imc_zicsr_zifencei_xcvhwlp_xcvmem_xcvmac_xcvbi_xcvalu_xcvsimd_xcvbitmanip' +] + +dummy_libs = ['crt0', 'libm', 'libc', 'libgcc'] diff --git a/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chipsupport.c b/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chipsupport.c new file mode 100644 index 0000000000..c540d4ad43 --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chipsupport.c @@ -0,0 +1,44 @@ +/* +** +** Copyright 2020 OpenHW Group +** +** Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** https://solderpad.org/licenses/ +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +** +******************************************************************************* +*/ + +#include +#include +#include "chipsupport.h" + +void +initialise_board () +{ + printf("Initialize board corev32 \n"); + __asm__ volatile ("li a0, 0" : : : "memory"); +} + +void __attribute__ ((noinline)) __attribute__ ((externally_visible)) +start_trigger () +{ + + + __asm__ volatile ("li a0, 0" : : : "memory"); +} + +void __attribute__ ((noinline)) __attribute__ ((externally_visible)) +stop_trigger () +{ + +} + \ No newline at end of file diff --git a/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chipsupport.h b/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chipsupport.h new file mode 100644 index 0000000000..72a58c8a31 --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/chips/size/chipsupport.h @@ -0,0 +1,27 @@ +/* +** +** Copyright 2020 OpenHW Group +** +** Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** https://solderpad.org/licenses/ +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +** +******************************************************************************* +*/ + +#ifndef CHIPSUPPORT_H +#define CHIPSUPPORT_H + +#define CPU_MHZ 1 + +#define TICKS_ADDR (*((volatile uint32_t*)0x15001004)) + +#endif diff --git a/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chip.cfg b/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chip.cfg new file mode 100644 index 0000000000..434b05a4e5 --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chip.cfg @@ -0,0 +1,87 @@ +############################################################################### +# +# Copyright 2020 OpenHW Group +# +# Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://solderpad.org/licenses/ +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 WITH SHL-2.0 +# +############################################################################### + +# This is a python setting of parameters for the chip. The following +# parameters may be set (other keys are silently ignored). Defaults are shown +# in brackets +# - cc ('cc') +# - ld (same value as for cc) +# - cflags ([]) +# - ldflags ([]) +# - cc_define_pattern ('-D{0}') +# - cc_incdir_pattern ('-I{0}') +# - cc_input_pattern ('{0}') +# - cc_output_pattern ('-o {0}') +# - ld_input_pattern ('{0}') +# - ld_output_pattern ('-o {0}') +# - user_libs ([]) +# - dummy_libs ([]) +# - cpu_mhz (1) +# - warmup_heat (1) + +# The "flags" and "libs" parameters (cflags, ldflags, user_libs, dummy_libs) +# should be lists of arguments to be passed to the compile or link line as +# appropriate. Patterns are Python format patterns used to create arguments. +# Thus for GCC or Clang/LLVM defined constants can be passed using the prefix +# '-D', and the pattern '-D{0}' would be appropriate (which happens to be the +# default). + +# "user_libs" may be absolute file names or arguments to the linker. In the +# latter case corresponding arguments in ldflags may be needed. For example +# with GCC or Clang/LLVM is "-l" flags are used in "user_libs", the "-L" flags +# may be needed in "ldflags". + +# Dummy libs have their source in the "support" subdirectory. Thus if 'crt0' +# is specified, there should be a source file 'dummy-crt0.c' in the support +# directory. + +# There is no need to set an unused parameter, and this file may be empty to +# set no flags. + +# Parameter values which are duplicated in architecture, board, chip or +# command line are used in the following order of priority +# - default value +# - architecture specific value +# - chip specific value +# - board specific value +# - command line value + +# For flags, this priority is applied to individual flags, not the complete +# list of flags. + +# This is the generic framework for compilers, where the only common set up +# is: + +# '-c' is used to specify generation of object files when compiling a + +# each global data and function is put in its own section + +# - we garbage collect unused sections on linking + +cflags = [ + '-c', '-O2', '-ffunction-sections', '-mabi=ilp32', '-march=rv32im_zicsr_zifencei_xcvhwlp_xcvmem_xcvmac_xcvbi_xcvalu_xcvsimd_xcvbitmanip' +] + +ldflags = [ + '-Wl,-gc-sections', '-Wl,-A,elf32lriscv', '-nostartfiles', '-mabi=ilp32', '-march=rv32im_zicsr_zifencei_xcvhwlp_xcvmem_xcvmac_xcvbi_xcvalu_xcvsimd_xcvbitmanip' +] +user_libs = ['-lm'] + +#dummy_libs = ['libm'] diff --git a/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chipsupport.c b/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chipsupport.c new file mode 100644 index 0000000000..d3c89e72b7 --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chipsupport.c @@ -0,0 +1,51 @@ +/* +** +** Copyright 2020 OpenHW Group +** +** Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** https://solderpad.org/licenses/ +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +** +******************************************************************************* +*/ + +#include +#include +#include "chipsupport.h" + +void +initialise_board () +{ + printf("Initialize board corev32 \n"); + __asm__ volatile ("li a0, 0" : : : "memory"); +} + +void __attribute__ ((noinline)) __attribute__ ((externally_visible)) +start_trigger () +{ + printf("start of test \n"); + //reset cycle counter + TICKS_ADDR = 0; + + __asm__ volatile ("li a0, 0" : : : "memory"); +} + +void __attribute__ ((noinline)) __attribute__ ((externally_visible)) +stop_trigger () +{ + uint32_t cycle_cnt = TICKS_ADDR; + printf("end of test \n"); + printf("Result is given in CPU cycles \n"); + printf("RES: %d \n", cycle_cnt); + + _exit(0); +} + \ No newline at end of file diff --git a/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chipsupport.h b/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chipsupport.h new file mode 100644 index 0000000000..72a58c8a31 --- /dev/null +++ b/cv32e40p/tests/embench/config/corev32_pulp/chips/speed/chipsupport.h @@ -0,0 +1,27 @@ +/* +** +** Copyright 2020 OpenHW Group +** +** Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** https://solderpad.org/licenses/ +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +** +******************************************************************************* +*/ + +#ifndef CHIPSUPPORT_H +#define CHIPSUPPORT_H + +#define CPU_MHZ 1 + +#define TICKS_ADDR (*((volatile uint32_t*)0x15001004)) + +#endif diff --git a/cv32e40p/tests/embench/pylib/benchmark_size.py b/cv32e40p/tests/embench/pylib/benchmark_size.py new file mode 100755 index 0000000000..da5b58c049 --- /dev/null +++ b/cv32e40p/tests/embench/pylib/benchmark_size.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 + +# Script to benchmark size + +# Copyright (C) 2017, 2019 Embecosm Limited +# +# Contributor: Graham Markall +# Contributor: Jeremy Bennett +# +# This file is part of Embench. + +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Compute the size benchmark for a set of compiled Embench programs. +""" + +import argparse +import codecs +import os +import sys + +from json import loads +from elftools.elf.elffile import ELFFile + +sys.path.append( + os.path.join(os.path.abspath(os.path.dirname(__file__)), 'pylib') +) + +from embench_core import check_python_version +from embench_core import log +from embench_core import gp +from embench_core import setup_logging +from embench_core import log_args +from embench_core import find_benchmarks +from embench_core import log_benchmarks +from embench_core import embench_stats +from embench_core import output_format + + +def build_parser(): + """Build a parser for all the arguments""" + parser = argparse.ArgumentParser(description='Compute the size benchmark') + + parser.add_argument( + '--builddir', + type=str, + default='bd', + help='Directory holding all the binaries', + ) + parser.add_argument( + '--logdir', + type=str, + default='logs', + help='Directory in which to store logs', + ) + parser.add_argument( + '--baselinedir', + type=str, + default='baseline-data', + help='Directory which contains baseline data', + ) + parser.add_argument( + '--absolute', + action='store_true', + help='Specify to show absolute results', + ) + parser.add_argument( + '--relative', + dest='absolute', + action='store_false', + help='Specify to show relative results (the default)', + ) + parser.add_argument( + '--json-output', + dest='output_format', + action='store_const', + const=output_format.JSON, + help='Specify to output in JSON format', + ) + parser.add_argument( + '--text-output', + dest='output_format', + action='store_const', + const=output_format.TEXT, + help='Specify to output as plain text (the default)', + ) + parser.add_argument( + '--baseline-output', + dest='output_format', + action='store_const', + const=output_format.BASELINE, + help='Specify to output in a format suitable for use as a baseline' + ) + parser.add_argument( + '--json-comma', + action='store_true', + help='Specify to append a comma to the JSON output', + ) + parser.add_argument( + '--no-json-comma', + dest='json_comma', + action='store_false', + help='Specify to not append a comma to the JSON output', + ) + # List arguments are empty by default, a user specified value then takes + # precedence. If the list is empty after parsing, then we can install a + # default value. + parser.add_argument( + '--text', + type=str, + default=[], + action='append', + help='Section name(s) containing code' + ) + parser.add_argument( + '--data', + type=str, + default=[], + action='append', + help='Section name(s) containing non-zero initialized writable data' + ) + parser.add_argument( + '--rodata', + type=str, + default=[], + action='append', + help='Section name(s) containing read only data' + ) + parser.add_argument( + '--bss', + type=str, + default=[], + action='append', + help='Section name(s) containing zero initialized writable data' + ) + parser.add_argument( + '--metric', + type=str, + default=[], + action='append', + choices=['text', 'rodata', 'data', 'bss'], + help='Sections to include in metric: one or more of "text", "rodata", ' + + '"data" or "bss". Default "text"', + ) + + return parser + + +def validate_args(args): + """Check that supplied args are all valid. By definition logging is + working when we get here. + + Update the gp dictionary with all the useful info""" + if os.path.isabs(args.builddir): + gp['bd'] = args.builddir + else: + gp['bd'] = os.path.join(gp['rootdir'], args.builddir) + + if not os.path.isdir(gp['bd']): + log.error(f'ERROR: build directory {gp["bd"]} not found: exiting') + sys.exit(1) + + if not os.access(gp['bd'], os.R_OK): + log.error(f'ERROR: Unable to read build directory {gp["bd"]}: exiting') + sys.exit(1) + + if os.path.isabs(args.baselinedir): + gp['baseline_dir'] = args.baselinedir + else: + gp['baseline_dir'] = os.path.join(gp['rootdir'], args.baselinedir) + + gp['absolute'] = args.absolute + if args.output_format: + gp['output_format'] = args.output_format + else: + gp['output_format'] = output_format.TEXT + + # Sort out the list of section names to use + gp['secnames'] = dict() + + for argname in ['text', 'rodata', 'data', 'bss']: + secnames = getattr(args, argname) + if secnames: + gp['secnames'][argname] = secnames + else: + gp['secnames'][argname] = ['.' + argname] + + # If no sections are specified, we just use .text + if args.metric: + gp['metric'] = args.metric + else: + gp['metric'] = ['text'] + +def get_section_group_by_name(elf, name): + """ Get a group of sections matching with the provided name from the file. + Return None if no such section exists. + """ + # The first time this method is called, construct a name to number + # mapping + # + if elf._section_name_map is None: + elf._section_name_map = {} + for i, sec in enumerate(elf.iter_sections()): + elf._section_name_map[sec.name] = i + + sec_group = {} + for i, sec in enumerate(elf.iter_sections()): + if sec.name.startswith(name): + secnum = elf._section_name_map.get(sec.name, None) + sec_group[sec.name] = elf.get_section(secnum) + + return None if sec_group is None else sec_group + +def benchmark_size(bench, metrics): + """Compute the total size of the desired sections in a benchmark. Returns + the size in bytes, which may be zero if the section wasn't found.""" + appexe = os.path.join(gp['bd_benchdir'], bench, bench) + sec_sizes = {} + + # If the benchmark failed to build, then return a 0 size instead of + # crashing when failing to open the file. + if not os.path.exists(appexe): + return {} + + with open(appexe, 'rb') as fileh: + elf = ELFFile(fileh) + for metric in metrics: + for secname in gp['secnames'][metric]: + sec_sizes[secname] = 0 + sections = get_section_group_by_name(elf, secname) + if sections: + for sec in sections: + sec_sizes[secname] += sections[sec]['sh_size'] + + # Return the section (group) size + return sec_sizes + + +def collect_data(benchmarks): + """Collect and log all the raw and optionally relative data associated with + the list of benchmarks supplied in the "benchmarks" argument. Return + the raw data and relative data as a list. The raw data may be empty if + there is a failure. The relative data will be empty if only absolute + results have been requested. + + Note that we manually generate the JSON output, rather than using the + dumps method, because the result will be manually edited, and we want + to guarantee the layout.""" + + # Baseline data is held external to the script. Import it here. + size_baseline = os.path.join(gp['baseline_dir'], 'size.json') + with open(size_baseline) as fileh: + baseline_all = loads(fileh.read()) + + # Compute the baseline data we need + baseline = {} + + for bench, data in baseline_all.items(): + baseline[bench] = 0 + for sec in gp['metric']: + baseline[bench] += data[sec] + + successful = True + raw_section_data = {} + raw_totals = {} + rel_data = {} + + # Collect data + all_metrics = ['text', 'rodata', 'data', 'bss'] + for bench in benchmarks: + if gp['output_format'] == output_format.BASELINE: + raw_section_data[bench] = benchmark_size(bench, all_metrics) + else: + raw_section_data[bench] = benchmark_size(bench, gp['metric']) + raw_totals[bench] = sum(raw_section_data[bench].values()) + + # Want relative results (the default). If baseline is zero, just + # use 0.0 as the value. Note this is inverted compared to the + # speed benchmark, so SMALL is good. + if baseline[bench] > 0: + rel_data[bench] = raw_totals[bench] / baseline[bench] + else: + rel_data[bench] = 0.0 + + # Output it + if gp['output_format'] == output_format.JSON: + log.info(' "size results" :') + log.info(' { "detailed size results" :') + for bench in benchmarks: + res_output = '' + if gp['absolute']: + res_output = f'{raw_totals[bench]}' + else: + res_output = f'{rel_data[bench]:.2f}' + + if bench == benchmarks[0]: + log.info(' { ' + f'"{bench}" : {res_output},') + elif bench == benchmarks[-1]: + log.info(f' "{bench}" : {res_output}') + else: + log.info(f' "{bench}" : {res_output},') + + log.info(' },') + elif gp['output_format'] == output_format.TEXT: + log.info('Benchmark Size Bytes') + log.info('--------- ----- -----') + for bench in benchmarks: + res_output_abs = '' + res_output_rel = '' + res_output_abs = f'{raw_totals[bench]:8,}' + res_output_rel = f' {rel_data[bench]:6.2f}' + log.info(f'{bench:15} {res_output_rel:8} {res_output_abs:8}') + elif gp['output_format'] == output_format.BASELINE: + log.info('{') + for bench in benchmarks: + res_output = '' + for metric in all_metrics: + if metric != all_metrics[0]: + res_output += ',\n' + + value = 0 + secname = '.' + metric + if raw_section_data[bench].get(secname): + value = raw_section_data[bench][secname] + res_output += f' "{metric}" : {value}' + + if bench == benchmarks[-1]: + log.info(f' "{bench}" : {{\n{res_output}\n }}') + else: + log.info(f' "{bench}" : {{\n{res_output}\n }},') + log.info('}') + + if successful: + return raw_totals, rel_data + + # Otherwise failure return + return [], [] + + +def main(): + """Main program driving measurement of benchmark size""" + # Establish the root directory of the repository, since we know this file is + # in that directory. + gp['rootdir'] = os.path.abspath(os.path.dirname(__file__)) + + # Parse arguments using standard technology + parser = build_parser() + args = parser.parse_args() + + # Establish logging + setup_logging(args.logdir, 'size') + log_args(args) + + # Check args are OK (have to have logging and build directory set up first) + validate_args(args) + + # Find the benchmarks + benchmarks = find_benchmarks() + log_benchmarks(benchmarks) + + # Collect the size data for the benchmarks + raw_data, rel_data = collect_data(benchmarks) + + # We can't compute geometric SD on the fly, so we need to collect all the + # data and then process it in two passes. We could do the first processing + # as we collect the data, but it is clearer to do the three things + # separately. Given the size of datasets with which we are concerned the + # compute overhead is not significant. + if raw_data: + if gp['output_format'] != output_format.BASELINE: + opt_comma = ',' if args.json_comma else '' + embench_stats(benchmarks, raw_data, rel_data, 'size', opt_comma) + log.info('All benchmarks sized successfully') + else: + log.info('ERROR: Failed to compute size benchmarks') + sys.exit(1) + + +# Make sure we have new enough Python and only run if this is the main package + +check_python_version(3, 6) +if __name__ == '__main__': + sys.exit(main()) diff --git a/cv32e40p/tests/embench/pylib/benchmark_speed.py b/cv32e40p/tests/embench/pylib/benchmark_speed.py new file mode 100755 index 0000000000..2d6d16a619 --- /dev/null +++ b/cv32e40p/tests/embench/pylib/benchmark_speed.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 + +# Script to benchmark execution speed. + +# Copyright (C) 2017, 2019 Embecosm Limited +# +# Contributor: Graham Markall +# Contributor: Jeremy Bennett +# +# This file is part of Embench. + +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Benchmark speed. + +This version is suitable when using a version of GDB which can launch a GDB +server to use as a target. +""" + +import argparse +import importlib +import os +import subprocess +import sys +import threading +import queue +import time + +from json import loads + +sys.path.append( + os.path.join(os.path.abspath(os.path.dirname(__file__)), 'pylib') +) + +from embench_core import check_python_version +from embench_core import log +from embench_core import gp +from embench_core import setup_logging +from embench_core import log_args +from embench_core import find_benchmarks +from embench_core import log_benchmarks +from embench_core import embench_stats +from embench_core import output_format + + +def get_common_args(): + """Build a parser for all the arguments""" + parser = argparse.ArgumentParser(description='Compute the size benchmark') + + parser.add_argument( + '--builddir', + type=str, + default='bd', + help='Directory holding all the binaries', + ) + parser.add_argument( + '--logdir', + type=str, + default='logs', + help='Directory in which to store logs', + ) + parser.add_argument( + '--baselinedir', + type=str, + default='baseline-data', + help='Directory which contains baseline data', + ) + parser.add_argument( + '--absolute', + action='store_true', + help='Specify to show absolute results', + ) + parser.add_argument( + '--relative', + dest='absolute', + action='store_false', + help='Specify to show relative results (the default)', + ) + parser.add_argument( + '--json-output', + dest='output_format', + action='store_const', + const=output_format.JSON, + help='Specify to output in JSON format', + ) + parser.add_argument( + '--text-output', + dest='output_format', + action='store_const', + const=output_format.TEXT, + help='Specify to output as plain text (the default)', + ) + parser.add_argument( + '--baseline-output', + dest='output_format', + action='store_const', + const=output_format.BASELINE, + help='Specify to output in a format suitable for use as a baseline' + ) + parser.add_argument( + '--json-comma', + action='store_true', + help='Specify to append a comma to the JSON output', + ) + parser.add_argument( + '--no-json-comma', + dest='json_comma', + action='store_false', + help='Specify to not append a comma to the JSON output', + ) + parser.add_argument( + '--target-module', + type=str, + required=True, + help='Python module with routines to run benchmarks', + ) + parser.add_argument( + '--timeout', + type=int, + default=30, + help='Timeout used for running each benchmark program' + ) + parser.add_argument( + '--sim-parallel', + action='store_true', + default=False, + help='Launch all benchmarks in parallel' + ) + parser.add_argument( + '--sim-serial', + dest='sim_parallel', + action='store_false', + help='Launch all benchmarks in series (the default)' + ) + + return parser.parse_known_args() + + +def validate_args(args): + """Check that supplied args are all valid. By definition logging is + working when we get here. + + Update the gp dictionary with all the useful info""" + if os.path.isabs(args.builddir): + gp['bd'] = args.builddir + else: + gp['bd'] = os.path.join(gp['rootdir'], args.builddir) + + if not os.path.isdir(gp['bd']): + log.error(f'ERROR: build directory {gp["bd"]} not found: exiting') + sys.exit(1) + + if not os.access(gp['bd'], os.R_OK): + log.error(f'ERROR: Unable to read build directory {gp["bd"]}: exiting') + sys.exit(1) + + if os.path.isabs(args.baselinedir): + gp['baseline_dir'] = args.baselinedir + else: + gp['baseline_dir'] = os.path.join(gp['rootdir'], args.baselinedir) + + gp['absolute'] = args.absolute + if args.output_format: + gp['output_format'] = args.output_format + else: + gp['output_format'] = output_format.TEXT + + gp['timeout'] = args.timeout + gp['sim_parallel'] = args.sim_parallel + + try: + newmodule = importlib.import_module(args.target_module) + except ImportError as error: + log.error( + f'ERROR: Target module import failure: {error}: exiting' + ) + sys.exit(1) + + globals()['get_target_args'] = newmodule.get_target_args + globals()['build_benchmark_cmd'] = newmodule.build_benchmark_cmd + globals()['decode_results'] = newmodule.decode_results + + +def benchmark_speed(bench, target_args): + """Time the benchmark. "target_args" is a namespace of arguments + specific to the target. Result is a time in milliseconds, or zero on + failure. + + For the parallel option, this method must be thread-safe.""" + succeeded = True + appdir = os.path.join(gp['bd_benchdir'], bench) + appexe = os.path.join(appdir, bench) + + if os.path.isfile(appexe): + arglist = build_benchmark_cmd(bench, target_args) + try: + res = subprocess.run( + arglist, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=appdir, + timeout=gp['timeout'], + ) + if res.returncode != 0: + log.warning(f'Warning: Run of {bench} failed.') + succeeded = False + except subprocess.TimeoutExpired: + log.warning(f'Warning: Run of {bench} timed out.') + succeeded = False + else: + log.warning(f'Warning: {bench} executable not found.') + succeeded = False + + # Process results + if succeeded: + exec_time = decode_results( + res.stdout.decode('utf-8'), res.stderr.decode('utf-8') + ) + succeeded = exec_time > 0 + + if succeeded: + return exec_time + else: + for arg in arglist: + if arg == arglist[0]: + comm = arg + elif arg == '-ex': + comm += ' ' + arg + else: + comm += " '" + arg + "'" + + log.debug('Args to subprocess:') + log.debug(f'{comm}') + if 'res' in locals(): + log.debug(res.stdout.decode('utf-8')) + log.debug(res.stderr.decode('utf-8')) + return 0.0 + +def run_threads(bench, target_args, data_collect_q): + item = benchmark_speed(bench, target_args) + data_collect_q.put_nowait([bench, item]) + +def collect_data(benchmarks, remnant): + """Collect and log all the raw and optionally relative data associated with + the list of benchmarks supplied in the "benchmarks" argument. "remant" + is left over args from the command line, which may be useful to the + benchmark running procs. + + Return the raw data and relative data as a list. The raw data may be + empty if there is a failure. The relative data will be empty if only + absolute results have been requested.""" + + # Baseline data is held external to the script. Import it here. + speed_baseline = os.path.join(gp['baseline_dir'], 'speed.json') + with open(speed_baseline) as fileh: + baseline = loads(fileh.read()) + + # Parse target specific args + target_args = get_target_args(remnant) + + # Collect data + successful = True + raw_data = {} + rel_data = {} + + # Run the benchmarks in parallel + if gp['sim_parallel']: + collect_data_q = queue.Queue() + benchmark_threads = list() + for bench in benchmarks: + curr_thread = threading.Thread(target=run_threads, args=(bench, target_args, collect_data_q)) + benchmark_threads.append(curr_thread) + curr_thread.start() + time.sleep(30) + # Join threads + for thd in benchmark_threads: + thd.join() + # All thread have returned + # Go through Q and make sure all benchmarks returned + while not collect_data_q.empty(): + [bench_id, raw_score] = collect_data_q.get() + raw_data[bench_id] = raw_score + # Run the benchmarks in serial + else: + for bench in benchmarks: + raw_data[bench] = benchmark_speed(bench, target_args) + + for bench in benchmarks: + rel_data[bench] = 0.0 + if raw_data[bench] == 0.0: + del raw_data[bench] + del rel_data[bench] + successful = False + else: + rel_data[bench] = baseline[bench] / raw_data[bench] + + # Output it + if gp['output_format'] == output_format.JSON: + log.info(' "speed results" :') + log.info(' { "detailed speed results" :') + for bench in benchmarks: + output = '' + if raw_data[bench] != 0.0: + if gp['absolute']: + output = f'{round(raw_data[bench])}' + else: + output = f'{rel_data[bench]:.2f}' + + if bench == benchmarks[0]: + log.info(f' {{ ' + f'"{bench}" : {output},') + elif bench == benchmarks[-1]: + log.info(f' "{bench}" : {output}') + else: + log.info(f' "{bench}" : {output},') + log.info(' },') + elif gp['output_format'] == output_format.TEXT: + log.info('Benchmark Speed Cycles') + log.info('--------- ----- ------') + for bench in benchmarks: + output_rel = '' + output_abs = '' + if (bench in raw_data and raw_data[bench] != 0.0): + output_rel = f' {rel_data[bench]:6.2f}' + output_abs = f'{round(raw_data[bench]):8,}' + # Want relative results (the default). Only use non-zero values. + log.info(f'{bench:15} {output_rel:8} {output_abs:8}') + elif gp['output_format'] == output_format.BASELINE: + log.info('{') + for bench in benchmarks: + if bench == benchmarks[-1]: + log.info(f' "{bench}" : {raw_data[bench]}') + else: + log.info(f' "{bench}" : {raw_data[bench]},') + log.info('}') + + if successful: + return raw_data, rel_data + + # Otherwise failure return + return [], [] + + +def main(): + """Main program driving measurement of benchmark size""" + # Establish the root directory of the repository, since we know this file is + # in that directory. + gp['rootdir'] = os.path.abspath(os.path.dirname(__file__)) + + # Parse arguments common to all speed testers, and get list of those + # remaining. + args, remnant = get_common_args() + + # Establish logging + setup_logging(args.logdir, 'speed') + log_args(args) + + # Check args are OK (have to have logging and build directory set up first) + validate_args(args) + + # Find the benchmarks + benchmarks = find_benchmarks() + log_benchmarks(benchmarks) + + # Collect the speed data for the benchmarks. Pass any remaining args. + raw_data, rel_data = collect_data(benchmarks, remnant) + + # We can't compute geometric SD on the fly, so we need to collect all the + # data and then process it in two passes. We could do the first processing + # as we collect the data, but it is clearer to do the three things + # separately. Given the size of datasets with which we are concerned the + # compute overhead is not significant. + if raw_data: + if gp['output_format'] != output_format.BASELINE: + opt_comma = ',' if args.json_comma else '' + embench_stats(benchmarks, raw_data, rel_data, 'speed', opt_comma) + log.info('All benchmarks run successfully') + else: + log.info('ERROR: Failed to compute speed benchmarks') + sys.exit(1) + + +# Make sure we have new enough Python and only run if this is the main package + +check_python_version(3, 6) +if __name__ == '__main__': + sys.exit(main()) diff --git a/cv32e40p/tests/embench/pylib/run_corev32.py b/cv32e40p/tests/embench/pylib/run_corev32.py index 6959e54456..7fd2b9be94 100644 --- a/cv32e40p/tests/embench/pylib/run_corev32.py +++ b/cv32e40p/tests/embench/pylib/run_corev32.py @@ -55,6 +55,13 @@ def get_target_args(remnant): """Parse left over arguments""" parser = argparse.ArgumentParser(description='Get target specific args') + parser.add_argument( + '-cfg', + type=str, + required=True, + help='Core configuration to benchmark' + ) + parser.add_argument( '--cpu-mhz', type=int, @@ -76,6 +83,12 @@ def get_target_args(remnant): help='Simulator to run the benchmarks' ) + parser.add_argument( + '--sim-parallel', + action='store_true', + default=False, + help='Launch all benchmarks in parallel' + ) return parser.parse_args(remnant) @@ -87,8 +100,16 @@ def build_benchmark_cmd(bench, args): global cpu_per cpu_per = float(1/(args.cpu_mhz*1_000_000)) + if args.cfg == 'default': + core_config = f'CFG=default' + else: + core_config = f'CFG=pulp' + #Utilize "make test" environment in core-v-verif - return ['make', '-C', args.make_path, 'test', f"TEST=emb_{bench}", f"SIMULATOR={args.simulator}", 'USE_ISS=NO'] + if args.sim_parallel == 'YES': + return ['make', '-C', args.make_path, 'test', f"TEST=emb_{bench}", f"SIMULATOR={args.simulator}", 'USE_ISS=NO', core_config, 'COMP=NO', 'USER_RUN_FLAGS=+rand_stall_obi_disable'] + else: + return ['make', '-C', args.make_path, 'test', f"TEST=emb_{bench}", f"SIMULATOR={args.simulator}", 'USE_ISS=NO', core_config, 'USER_RUN_FLAGS=+rand_stall_obi_disable'] def decode_results(stdout_str, stderr_str): diff --git a/mk/uvmt/uvmt.mk b/mk/uvmt/uvmt.mk index b04d9bc6b5..f272055d36 100644 --- a/mk/uvmt/uvmt.mk +++ b/mk/uvmt/uvmt.mk @@ -117,6 +117,7 @@ EMB_TARGET ?= 0 EMB_CPU_MHZ ?= 1 EMB_TIMEOUT ?= 3600 EMB_PARALLEL_ARG = $(if $(filter $(YES_VALS),$(EMB_PARALLEL)),YES,NO) +EMB_ABSOLUTE_ARG = $(if $(filter $(YES_VALS),$(EMB_ABSOLUTE)),YES,NO) EMB_BUILD_ONLY_ARG = $(if $(filter $(YES_VALS),$(EMB_BUILD_ONLY)),YES,NO) EMB_DEBUG_ARG = $(if $(filter $(YES_VALS),$(EMB_DEBUG)),YES,NO) @@ -345,11 +346,15 @@ dah: embench: $(EMBENCH_PKG) $(CORE_V_VERIF)/bin/run_embench.py \ -c $(CV_CORE) \ + -cfg $(CFG) \ -cc $(RISCV_EXE_PREFIX)$(RISCV_CC) \ -sim $(SIMULATOR) \ -t $(EMB_TYPE) \ --timeout $(EMB_TIMEOUT) \ --parallel $(EMB_PARALLEL_ARG) \ + --absolute $(EMB_ABSOLUTE_ARG) \ + --builddir $(SIM_CFG_RESULTS)/bd \ + --logdir $(SIM_CFG_RESULTS)/logs \ -b $(EMB_BUILD_ONLY_ARG) \ -tgt $(EMB_TARGET) \ -f $(EMB_CPU_MHZ) \