diff --git a/cpupins/cpupin-eth-mockdlh-np02srv003-1.json b/cpupins/cpupin-eth-mockdlh-np02srv003-1.json new file mode 100644 index 0000000..28cea65 --- /dev/null +++ b/cpupins/cpupin-eth-mockdlh-np02srv003-1.json @@ -0,0 +1,21 @@ +{ + "rubberdaq_test_mock_dlh": { + "--run_secs": { + "parent": "1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63,65,67,69,71,73,75,77,79,81,83,85,87,89,91,93,95,97,99,101,103,105,107,109,111", + "threads": { + "rte-worker-1": "1", + "rte-worker-3": "3", + "rte-worker-5": "5", + "rte-worker-57": "57", + "rte-worker-59": "59", + "rte-worker-61": "61", + + "cleanup-.": "1,3,5,7,57,59,61,63", + + "producer-.": "9,11,13,15,65,67,69,71", + + "consumer-.": "17,19,21,23,73,75,77,79" + } + } + } +} diff --git a/cpupins/cpupin-eth-mockdlh-np02srv004-0.json b/cpupins/cpupin-eth-mockdlh-np02srv004-0.json new file mode 100644 index 0000000..b3e14c9 --- /dev/null +++ b/cpupins/cpupin-eth-mockdlh-np02srv004-0.json @@ -0,0 +1,21 @@ +{ + "rubberdaq_test_mock_dlh": { + "--run_secs": { + "parent": "0-63,128-191", + "threads": { + "rte-worker-1": "1", + "rte-worker-3": "2", + "rte-worker-5": "3", + "rte-worker-129": "129", + "rte-worker-130": "130", + "rte-worker-131": "131", + + "cleanup-.": "0-7,128-135", + + "producer-.": "8-15,136-143", + + "consumer-.": "16-23,144-151" + } + } + } +} diff --git a/cpupins/cpupin-eth-mockdlh_grouped-np02srv004-1.json b/cpupins/cpupin-eth-mockdlh_grouped-np02srv004-1.json new file mode 100644 index 0000000..f21b9e4 --- /dev/null +++ b/cpupins/cpupin-eth-mockdlh_grouped-np02srv004-1.json @@ -0,0 +1,14 @@ +{ + "rubberdaq_test_mock_dlh": { + "--run_secs": { + "parent": "64-127,192-255", + "threads": { + "cleanup-.": "64-71,192-199", + + "producer-.": "72-79,200-207", + + "consumer-.": "80-87,208-215" + } + } + } +} diff --git a/cpupins/cpupin-eth-mockdlh_mixed-np02srv004-1.json b/cpupins/cpupin-eth-mockdlh_mixed-np02srv004-1.json new file mode 100644 index 0000000..e69fb95 --- /dev/null +++ b/cpupins/cpupin-eth-mockdlh_mixed-np02srv004-1.json @@ -0,0 +1,41 @@ +{ + "rubberdaq_test_mock_dlh": { + "--run_secs": { + "parent": "64-127,192-255", + "threads": { + "cleanup-0": "64,192", + "cleanup-1": "64,192", + "cleanup-2": "67,195", + "cleanup-3": "67,195", + "cleanup-4": "72,200", + "cleanup-5": "72,200", + "cleanup-6": "75,203", + "cleanup-7": "75,203", + "cleanup-8": "80,208", + "cleanup-9": "80,208", + + "producer-0": "65,193", + "producer-1": "65,193", + "producer-2": "68,196", + "producer-3": "68,196", + "producer-4": "73,201", + "producer-5": "73,201", + "producer-6": "76,204", + "producer-7": "76,204", + "producer-8": "81,209", + "producer-9": "81,209", + + "consumer-0": "66,194", + "consumer-1": "66,194", + "consumer-2": "69,197", + "consumer-3": "69,197", + "consumer-4": "74,202", + "consumer-5": "74,202", + "consumer-6": "77,205", + "consumer-7": "77,205", + "consumer-8": "82,210", + "consumer-9": "82,210" + } + } + } +} diff --git a/scripts/start_uprof_np02srv001.sh b/scripts/start_uprof_np02srv001.sh new file mode 100755 index 0000000..16f2e8f --- /dev/null +++ b/scripts/start_uprof_np02srv001.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ $# -ne 3 ]; then + echo "Usage: ./start_uprof.sh " + exit 2 +fi + +output_dir=$1 +test_name=$2 +duration=$3 + +echo "start uprof monitoring" +/opt/AMDuProf_4.1-424/bin/AMDuProfPcm -a -s -d $duration -t 2500 -m memory,ipc,l1,l2,l3 -A package -k -o $output_dir/$test_name/uprof-$test_name.csv & +/opt/AMDuProf_4.1-424/bin/AMDuProfCLI-bin timechart --event power --interval 15000 --duration $duration -o $output_dir/$test_name + +cd $output_dir/$test_name/AMDuProf-SWP-Timechart_*/ +mv timechart.csv ../timechart-$test_name.csv + +cd ../ diff --git a/tests/AMD_debug/np02srv003_callbacktest.sh b/tests/AMD_debug/np02srv003_callbacktest.sh new file mode 100755 index 0000000..bffe5cf --- /dev/null +++ b/tests/AMD_debug/np02srv003_callbacktest.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# need to source the environment file first + +if [ $# -ne 3 ]; then + echo "Usage: ./AMD_callback_test.sh " + echo "Ex: ./AMD_callback_test.sh callbacktest_results/ cpupins/cpupin-eth-mockdlh-np02srv003-1.json" + exit 2 +fi + +# Script parameters +test_time=90 +output_path=$1 +cpupin_file=$2 +test_name=$3 + +# NUMA0 cores +cpu_affinity=("1-111:2") + +## Callbacks +echo "> consumer_mode : callback" +#test_name=v4_2_1-np02srv003-1-eth-callback_Intel +sar -P ALL 5 18 >> $output_path/core_utilization-v4_2_1-np02srv003-1-eth-callback_${test_name}.csv & + +taskset -c $cpu_affinity rubberdaq_test_mock_dlh --run_secs $test_time -n 10 --cb & +sleep 4 +readout-affinity.py --pinfile $cpupin_file +sleep 120 + + +## Queues +echo "> consumer_mode : queue" +#test_name=v4_2_1-np02srv003-1-eth-queue_Intel +sar -P ALL 5 18 >> $output_path/core_utilization-v4_2_1-np02srv003-1-eth-queue_${test_name}.csv & + +taskset -c $cpu_affinity rubberdaq_test_mock_dlh --run_secs $test_time -n 10 --ct & +sleep 4 +readout-affinity.py --pinfile $cpupin_file +sleep 120 + + +echo "Finished !" diff --git a/tests/AMD_debug/np02srv004_callbacktest.sh b/tests/AMD_debug/np02srv004_callbacktest.sh new file mode 100755 index 0000000..88938be --- /dev/null +++ b/tests/AMD_debug/np02srv004_callbacktest.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# need to source the environment file first + +if [ $# -ne 3 ]; then + echo "Usage: ./AMD_callback_test.sh " + echo "Ex: ./AMD_callback_test.sh callbacktest_results/ ../../cpupins/cpupin-eth-mockdlh_mixed-np02srv004-1.json AMD_mixed" + exit 2 +fi + +# Script parameters +test_time=90 +output_path=$1 +cpupin_file=$2 +test_name=$3 + +# NUMA0 cores +cpu_affinity=("64-127,192-255") + +## Callbacks +echo "> consumer_mode : callback" +# Start uProf monitoring +sudo /opt/AMDuProf_4.0-341/bin/AMDuProfPcm -a -s -d 95 -t 250 -m memory,ipc,l1,l2,l3 -A package,ccd -k -o $output_path/uprof-v4_2_1-np02srv004-1-eth-callback_$test_name.csv & + +# cpu core utilization monitoring at 5 second intervals +sar -P ALL 5 18 >> $output_path/core_utilization-v4_2_1-np02srv004-1-eth-callback_${test_name}.csv & + +taskset -c $cpu_affinity rubberdaq_test_mock_dlh --run_secs $test_time -n 10 --cb & +sleep 4 +readout-affinity.py --pinfile $cpupin_file +sleep 120 + + +## Queues +echo "> consumer_mode : queue" +sudo /opt/AMDuProf_4.0-341/bin/AMDuProfPcm -a -s -d 95 -t 250 -m memory,ipc,l1,l2,l3 -A package,ccd -k -o $output_path/uprof-v4_2_1-np02srv004-1-eth-queue_$test_name.csv & + +sar -P ALL 5 18 >> $output_path/core_utilization-v4_2_1-np02srv004-1-eth-queue_${test_name}.csv & + +taskset -c $cpu_affinity rubberdaq_test_mock_dlh --run_secs $test_time -n 10 --ct & +sleep 4 +readout-affinity.py --pinfile $cpupin_file +sleep 120 + + +echo "Finished !" diff --git a/tools/Performance_report.ipynb b/tools/Performance_report.ipynb index 140d7e6..8a1ecf1 100644 --- a/tools/Performance_report.ipynb +++ b/tools/Performance_report.ipynb @@ -10,19 +10,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "22d8e991", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Cheking list of packages need it\n", - "Ready to run and process\n" - ] - } - ], + "outputs": [], "source": [ "from basic_functions import *\n", "\n", @@ -38,6 +29,9 @@ " 'Socket1 L2 Cache Misses', 'Socket1 L2 Cache Hits', \n", " 'Socket1 L3 Cache Misses', 'Socket1 L3 Cache Hits']\n", "\n", + "#uprof_columns_list_0 = [' Utilization (%) Socket0', 'Total Mem Bw (GB/s) Socket0', 'IPC (Sys + User) Socket1.1', \n", + "# 'L2 Miss (pti) Socket1.1', 'L2 Access (pti) Socket1.1', \n", + "# 'L3 Miss Socket1.1', 'L3 Miss % Socket1.1']\n", "uprof_columns_list_0 = [' Utilization (%) Socket0', 'Total Mem Bw (GB/s) Socket0', 'IPC (Sys + User) Socket0', \n", " 'L2 Miss (pti) Socket0', 'L2 Access (pti) Socket0', \n", " 'L3 Miss Socket0', 'L3 Miss % Socket0']\n", @@ -170,7 +164,7 @@ " plt.savefig('{}/{}_results_cache_{}_{}_socket1.png'.format(output_dir, pdf_name, info[1], info[4]))\n", " plt.close() \n", "\n", - "def create_report_performance(input_dir, output_dir, daqconfs_cpupins_folder_parent_dir, process_pcm_files=False, process_uprof_files=False, print_info=True, streams='8, 16, 24, 32, 40, and 48', pdf_name='performance_report', repin_threads_file=None, elog_message=['TBA']): \n", + "def create_report_performance(input_dir, output_dir, daqconfs_cpupins_folder_parent_dir, process_pcm_files=False, process_uprof_files=False, print_info=True, streams='8, 16, 24, 32, 40, and 48', pdf_name='performance_report', repin_threads_file=None, elog_message=['TBA'], use_daqconf=True): \n", " # Function to \n", " directory([input_dir, output_dir])\n", " pcm_file, uprof_file, core_utilization_file, reformated_uprof_file, reformated_core_utilization_file, all_file, all_plots_file = make_name_list(input_dir)\n", @@ -270,10 +264,14 @@ " if print_info:\n", " pdf.write(5, 'Configurations: \\n', 'B')\n", " for i, (file_i, file_ii) in enumerate(zip(all_file, reformated_core_utilization_file)):\n", + " print(file_i, file_ii)\n", " info = break_file_name(file_i)\n", " file_daqconf_i='daqconf-{}-{}-{}-{}'.format(info[4], info[5], info[2], info[3])\n", " var_i='ru{}{}{}'.format(info[2], info[4], '0')\n", - " json_info(file_daqconf=file_daqconf_i, file_core=file_ii, input_directory=daqconfs_cpupins_folder_parent_dir, input_dir=input_dir, var=var_i, pdf=pdf, if_pdf=print_info, repin_threads_file=repin_threads_file)\n", + " if use_daqconf:\n", + " json_info(file_daqconf=file_daqconf_i, file_core=file_ii, input_directory=daqconfs_cpupins_folder_parent_dir, input_dir=input_dir, var=var_i, pdf=pdf, if_pdf=print_info, repin_threads_file=repin_threads_file)\n", + " else:\n", + " json_info_general(file_core=file_ii, input_directory=daqconfs_cpupins_folder_parent_dir, input_dir=input_dir, var=var_i, pdf=pdf, if_pdf=print_info, file_cpupin=repin_threads_file[i], use_daqconf=use_daqconf)\n", " pdf.cell(0, 10, 'Table {}. CPU core pins information for the \"{}\" test.'.format(i+2, info[5]))\n", " pdf.ln(10)\n", " \n", @@ -391,28 +389,21 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "ea693364", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data saved to CSV successfully: /eos/user/m/mman/performancetest/tools/queuetest_results/grafana-v4_2_1-np02srv003-0-eth-queuetest_Intel.csv\n", - "done :-)\n" - ] - } - ], + "outputs": [], "source": [ - "# mman spsc queue test app\n", + "# mman callback test app\n", "\n", "grafana_url = 'http://np04-srv-009.cern.ch:3000'\n", "dashboard_uid = ['91zWmJEVk']\n", "host_used = 'np02-srv-003' \n", - "delta_time = [['2024-01-25 13:57:17', '2024-01-25 14:04:27']]\n", - "output_csv_file = ['v4_2_1-np02srv003-0-eth-queuetest_Intel']\n", - "results_path = '/eos/user/m/mman/performancetest/tools/queuetest_results'\n", + "delta_time = [['2024-02-26 11:08:56', '2024-02-26 11:10:32'],\n", + " ['2024-02-26 11:11:00', '2024-02-26 11:12:34']]\n", + "output_csv_file = ['v4_2_1-np02srv003-1-eth-callback_Intel',\n", + " 'v4_2_1-np02srv003-1-eth-queue_Intel']\n", + "results_path = '/eos/user/m/mman/performancetest/tools/callbacktest_results'\n", "\n", "for delta_time_list, output_csv_file_list in zip(delta_time, output_csv_file):\n", " extract_data_and_stats_from_panel(grafana_url, dashboard_uid, delta_time=delta_time_list, host=host_used, \n", @@ -455,67 +446,25 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "4c688dcc", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Finish the processing of the data.\n", - "0 0.049003\n", - "1 0.150302\n", - "2 0.062094\n", - "3 0.507142\n", - "4 0.175968\n", - " ... \n", - "211 0.119392\n", - "212 0.052317\n", - "213 0.046248\n", - "214 0.123093\n", - "215 0.101806\n", - "Name: Socket0 L3 Cache Misses, Length: 216, dtype: float64\n", - "0 0.302006\n", - "1 0.527486\n", - "2 0.314720\n", - "3 1.011582\n", - "4 0.678650\n", - " ... \n", - "211 0.445688\n", - "212 0.279797\n", - "213 0.275933\n", - "214 0.446962\n", - "215 0.499568\n", - "Length: 216, dtype: float64\n", - "0 16.225837\n", - "1 28.493905\n", - "2 19.730078\n", - "3 50.133504\n", - "4 25.929179\n", - " ... \n", - "211 26.788277\n", - "212 18.698232\n", - "213 16.760741\n", - "214 27.540037\n", - "215 20.378907\n", - "Length: 216, dtype: float64\n", - "THE END\n" - ] - } - ], + "outputs": [], "source": [ - "## mman readout queue perftest\n", - "results_path = '/eos/user/m/mman/performancetest/tools/queuetest_results'\n", - "report_path = '/eos/user/m/mman/performancetest/tools/performance_reports'\n", - "performancetest_path = '/eos/user/m/mman/performancetest'\n", + "## mman readout callback perftest\n", + "results_path = '/eos/home-m/mman/performancetest/tools/callbacktest_results_v2'\n", + "report_path = '/eos/home-m/mman/performancetest/tools/performance_reports'\n", + "performancetest_path = '/eos/home-m/mman/performancetest'\n", + "repin_files = ['cpupin-eth-mockdlh_grouped-np02srv004-1.json','cpupin-eth-mockdlh_mixed-np02srv004-1.json','cpupin-eth-mockdlh_grouped-np02srv004-1.json','cpupin-eth-mockdlh_mixed-np02srv004-1.json']\n", "\n", "create_report_performance(input_dir=results_path, output_dir=report_path, \n", " daqconfs_cpupins_folder_parent_dir=performancetest_path, \n", " process_pcm_files=True, process_uprof_files=True, \n", - " print_info=True, streams='4', \n", - " pdf_name='queuetest_AMD_debug',\n", - " elog_message=['Unable to push within timeout period (timeout period was 0 milliseconds)', 'Unable to push within timeout period (timeout period was 0 milliseconds)'])\n", + " print_info=True, streams='10',\n", + " pdf_name='callbacktest_v2',\n", + " elog_message=['', '','',''],\n", + " repin_threads_file=repin_files,\n", + " use_daqconf=False)\n", "\n", "print('THE END')" ] @@ -588,7 +537,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d2504ce3", + "id": "56c6f624", "metadata": {}, "outputs": [], "source": [] diff --git a/tools/basic_functions.py b/tools/basic_functions.py index 03f518d..8152974 100644 --- a/tools/basic_functions.py +++ b/tools/basic_functions.py @@ -590,7 +590,7 @@ def json_info(file_daqconf, file_core, input_directory, input_dir, var, pdf, if_ for m, value_m in enumerate(data_readout): if value_m in ['thread_pinning_file']: if repin_threads_file: - pdf.write(5, ' * {}: {} \n'.format(value_m, repin_threads_file)) + pdf.write(5, ' * {}: {} \n'.format(value_m, repin_threads_file[value_m])) else: pdf.write(5, ' * {}: {} \n'.format(value_m, data_readout_list[value_m])) @@ -601,16 +601,33 @@ def json_info(file_daqconf, file_core, input_directory, input_dir, var, pdf, if_ pinning_table, cpu_core_table, cpu_utilization_table = extract_table_data(input_dir, file_core, data_list, emu_mode=emu_mode) pdf.ln(5) table_cpupins(columns_data=[pinning_table, cpu_core_table, cpu_utilization_table], pdf=pdf, if_pdf=if_pdf) + +# Function to extract the information of the cpupinning info for a non-DAQ-run (eg. test apps) +def json_info_general(file_core, input_directory, input_dir, var, pdf, if_pdf=False, file_cpupin=None, emu_mode=True, use_daqconf=False): + emu_mode = None + + data_list = cpupining_info(input_directory, file_cpupin, var, use_daqconf) + if if_pdf: + pdf.set_font('Times', '', 10) + pdf.write(5, ' * {} \n'.format(file_cpupin)) + + pinning_table, cpu_core_table, cpu_utilization_table = extract_table_data(input_dir, file_core, data_list, emu_mode=emu_mode) + pdf.ln(5) + table_cpupins(columns_data=[pinning_table, cpu_core_table, cpu_utilization_table], pdf=pdf, if_pdf=if_pdf) # Function to extract the information of the cpupinning file. -def cpupining_info(input_dir, file, var): +def cpupining_info(input_dir, file, var, use_daqconf=True): with open('{}/cpupins/{}'.format(input_dir, file), 'r') as ff: data_cpupins = json.load(ff) - info_daq_application = json.dumps(data_cpupins['daq_application']['--name {}'.format(var)], skipkeys = True, allow_nan = True) + if use_daqconf: + info_daq_application = json.dumps(data_cpupins['daq_application']['--name {}'.format(var)], skipkeys = True, allow_nan = True) + else: + info_daq_application = json.dumps(data_cpupins['rubberdaq_test_mock_dlh']['--run_secs'], skipkeys = True, allow_nan = True) data_list = json.loads(info_daq_application) return data_list + # Function to extract the "CPU" and "Utilization" of the pins from the CSV file. def core_utilization(input_dir, file): CPU_plot, User_plot = [], [] @@ -659,9 +676,16 @@ def extract_table_data(input_dir, file_core, data_list, emu_mode): # Calculate averages for each CPU core configuration for cpu_cores_i in cpu_core_table: - cpu_cores = [int(num) for num in cpu_cores_i.split(',')] + cpu_cores = [] + for cores in cpu_cores_i.split(','): + if '-' in cores: + core_range = range(int(cores.split('-')[0]), int(cores.split('-')[1])+1) + else: + core_range = [int(cores)] + cpu_cores+=core_range for core_i in cpu_cores: + deno += 1 sum_utilization += cpu_utilization[core_i]