Mman/callbacktests

cpupins/cpupin-eth-mockdlh-np02srv003-1.json

@@ -0,0 +1,21 @@
+{
+  "rubberdaq_test_mock_dlh": {
+    "--run_secs": {
+      "parent": "1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63,65,67,69,71,73,75,77,79,81,83,85,87,89,91,93,95,97,99,101,103,105,107,109,111",
+      "threads": {
+        "rte-worker-1": "1",
+        "rte-worker-3": "3",
+        "rte-worker-5": "5",
+        "rte-worker-57": "57",
+        "rte-worker-59": "59",
+        "rte-worker-61": "61",
+
+        "cleanup-.": "1,3,5,7,57,59,61,63",
+
+        "producer-.": "9,11,13,15,65,67,69,71",
+
+	"consumer-.": "17,19,21,23,73,75,77,79"
+      }
+    }
+  }
+}

cpupins/cpupin-eth-mockdlh-np02srv004-0.json

@@ -0,0 +1,21 @@
+{
+  "rubberdaq_test_mock_dlh": {
+    "--run_secs": {
+      "parent": "0-63,128-191",
+      "threads": {
+        "rte-worker-1": "1",
+        "rte-worker-3": "2",
+        "rte-worker-5": "3",
+        "rte-worker-129": "129",
+        "rte-worker-130": "130",
+        "rte-worker-131": "131",
+
+        "cleanup-.": "0-7,128-135",
+
+        "producer-.": "8-15,136-143",
+
+	"consumer-.": "16-23,144-151"
+      }
+    }
+  }
+}

cpupins/cpupin-eth-mockdlh_grouped-np02srv004-1.json

@@ -0,0 +1,14 @@
+{
+  "rubberdaq_test_mock_dlh": {
+    "--run_secs": {
+      "parent": "64-127,192-255",
+      "threads": {
+        "cleanup-.": "64-71,192-199",
+
+        "producer-.": "72-79,200-207",
+
+	"consumer-.": "80-87,208-215"
+      }
+    }
+  }
+}

cpupins/cpupin-eth-mockdlh_mixed-np02srv004-1.json

@@ -0,0 +1,41 @@
+{
+  "rubberdaq_test_mock_dlh": {
+    "--run_secs": {
+      "parent": "64-127,192-255",
+      "threads": {
+        "cleanup-0": "64,192",
+	"cleanup-1": "64,192",
+	"cleanup-2": "67,195",
+	"cleanup-3": "67,195",
+	"cleanup-4": "72,200",
+	"cleanup-5": "72,200",
+	"cleanup-6": "75,203",
+	"cleanup-7": "75,203",
+	"cleanup-8": "80,208",
+	"cleanup-9": "80,208",
+
+        "producer-0": "65,193",
+	"producer-1": "65,193",
+	"producer-2": "68,196",
+	"producer-3": "68,196",
+	"producer-4": "73,201",
+	"producer-5": "73,201",
+	"producer-6": "76,204",
+        "producer-7": "76,204",
+	"producer-8": "81,209",
+        "producer-9": "81,209",
+
+	"consumer-0": "66,194",
+	"consumer-1": "66,194",
+	"consumer-2": "69,197",
+	"consumer-3": "69,197",
+	"consumer-4": "74,202",
+        "consumer-5": "74,202",
+	"consumer-6": "77,205",
+        "consumer-7": "77,205",
+	"consumer-8": "82,210",
+        "consumer-9": "82,210"
+      }
+    }
+  }
+}

scripts/start_uprof_np02srv001.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ $# -ne 3 ]; then
+  echo "Usage: ./start_uprof.sh <output_directory> <test_name> <duration in seconds>"
+  exit 2
+fi
+
+output_dir=$1
+test_name=$2
+duration=$3
+
+echo "start uprof monitoring"
+/opt/AMDuProf_4.1-424/bin/AMDuProfPcm -a -s -d $duration -t 2500 -m memory,ipc,l1,l2,l3 -A package -k -o $output_dir/$test_name/uprof-$test_name.csv &
+/opt/AMDuProf_4.1-424/bin/AMDuProfCLI-bin timechart --event power --interval 15000 --duration $duration -o $output_dir/$test_name 
+
+cd $output_dir/$test_name/AMDuProf-SWP-Timechart_*/
+mv timechart.csv ../timechart-$test_name.csv
+
+cd ../

tests/AMD_debug/np02srv003_callbacktest.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# need to source the environment file first
+
+if [ $# -ne 3 ]; then
+  echo "Usage: ./AMD_callback_test.sh <output_path> <cpupin_file> <test_name>"
+  echo "Ex: ./AMD_callback_test.sh callbacktest_results/ cpupins/cpupin-eth-mockdlh-np02srv003-1.json"
+  exit 2
+fi
+
+# Script parameters
+test_time=90
+output_path=$1
+cpupin_file=$2
+test_name=$3
+
+# NUMA0 cores
+cpu_affinity=("1-111:2")
+
+## Callbacks
+echo "> consumer_mode : callback"
+#test_name=v4_2_1-np02srv003-1-eth-callback_Intel
+sar -P ALL 5 18 >> $output_path/core_utilization-v4_2_1-np02srv003-1-eth-callback_${test_name}.csv &
+
+taskset -c $cpu_affinity rubberdaq_test_mock_dlh --run_secs $test_time -n 10 --cb &
+sleep 4
+readout-affinity.py --pinfile $cpupin_file
+sleep 120
+
+
+## Queues
+echo "> consumer_mode : queue"
+#test_name=v4_2_1-np02srv003-1-eth-queue_Intel
+sar -P ALL 5 18 >> $output_path/core_utilization-v4_2_1-np02srv003-1-eth-queue_${test_name}.csv &
+
+taskset -c $cpu_affinity rubberdaq_test_mock_dlh --run_secs $test_time -n 10 --ct &
+sleep 4
+readout-affinity.py --pinfile $cpupin_file
+sleep 120
+
+
+echo "Finished !"

tests/AMD_debug/np02srv004_callbacktest.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# need to source the environment file first
+
+if [ $# -ne 3 ]; then
+  echo "Usage: ./AMD_callback_test.sh <output_path> <cpupin_file> <test_name>"
+  echo "Ex: ./AMD_callback_test.sh callbacktest_results/ ../../cpupins/cpupin-eth-mockdlh_mixed-np02srv004-1.json AMD_mixed"
+  exit 2
+fi
+
+# Script parameters
+test_time=90
+output_path=$1
+cpupin_file=$2
+test_name=$3
+
+# NUMA0 cores
+cpu_affinity=("64-127,192-255")
+
+## Callbacks
+echo "> consumer_mode : callback"
+# Start uProf monitoring
+sudo /opt/AMDuProf_4.0-341/bin/AMDuProfPcm -a -s -d 95 -t 250 -m memory,ipc,l1,l2,l3 -A package,ccd -k -o $output_path/uprof-v4_2_1-np02srv004-1-eth-callback_$test_name.csv &
+
+# cpu core utilization monitoring at 5 second intervals
+sar -P ALL 5 18 >> $output_path/core_utilization-v4_2_1-np02srv004-1-eth-callback_${test_name}.csv &
+
+taskset -c $cpu_affinity rubberdaq_test_mock_dlh --run_secs $test_time -n 10 --cb &
+sleep 4
+readout-affinity.py --pinfile $cpupin_file
+sleep 120
+
+
+## Queues
+echo "> consumer_mode : queue"
+sudo /opt/AMDuProf_4.0-341/bin/AMDuProfPcm -a -s -d 95 -t 250 -m memory,ipc,l1,l2,l3 -A package,ccd -k -o $output_path/uprof-v4_2_1-np02srv004-1-eth-queue_$test_name.csv &
+
+sar -P ALL 5 18 >> $output_path/core_utilization-v4_2_1-np02srv004-1-eth-queue_${test_name}.csv &
+
+taskset -c $cpu_affinity rubberdaq_test_mock_dlh --run_secs $test_time -n 10 --ct &
+sleep 4
+readout-affinity.py --pinfile $cpupin_file
+sleep 120
+
+
+echo "Finished !"

tools/Performance_report.ipynb

@@ -10,19 +10,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "22d8e991",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Cheking list of packages need it\n",
-      "Ready to run and process\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from basic_functions import *\n",
     "\n",
@@ -38,6 +29,9 @@
     "                      'Socket1 L2 Cache Misses', 'Socket1 L2 Cache Hits', \n",
     "                      'Socket1 L3 Cache Misses', 'Socket1 L3 Cache Hits']\n",
     "\n",
+    "#uprof_columns_list_0 = [' Utilization (%) Socket0', 'Total Mem Bw (GB/s) Socket0', 'IPC (Sys + User) Socket1.1', \n",
+    "#                        'L2 Miss (pti) Socket1.1', 'L2 Access (pti) Socket1.1', \n",
+    "#                        'L3 Miss Socket1.1', 'L3 Miss % Socket1.1']\n",
     "uprof_columns_list_0 = [' Utilization (%) Socket0', 'Total Mem Bw (GB/s) Socket0', 'IPC (Sys + User) Socket0', \n",
     "                        'L2 Miss (pti) Socket0', 'L2 Access (pti) Socket0', \n",
     "                        'L3 Miss Socket0', 'L3 Miss % Socket0']\n",
@@ -170,7 +164,7 @@
     "    plt.savefig('{}/{}_results_cache_{}_{}_socket1.png'.format(output_dir, pdf_name, info[1], info[4]))\n",
     "    plt.close() \n",
     "\n",
-    "def create_report_performance(input_dir, output_dir, daqconfs_cpupins_folder_parent_dir, process_pcm_files=False, process_uprof_files=False, print_info=True, streams='8, 16, 24, 32, 40, and 48', pdf_name='performance_report', repin_threads_file=None, elog_message=['TBA']):    \n",
+    "def create_report_performance(input_dir, output_dir, daqconfs_cpupins_folder_parent_dir, process_pcm_files=False, process_uprof_files=False, print_info=True, streams='8, 16, 24, 32, 40, and 48', pdf_name='performance_report', repin_threads_file=None, elog_message=['TBA'], use_daqconf=True):    \n",
     "    # Function to \n",
     "    directory([input_dir, output_dir])\n",
     "    pcm_file, uprof_file, core_utilization_file, reformated_uprof_file, reformated_core_utilization_file, all_file, all_plots_file = make_name_list(input_dir)\n",
@@ -270,10 +264,14 @@
     "    if print_info:\n",
     "        pdf.write(5, 'Configurations: \\n', 'B')\n",
     "        for i, (file_i, file_ii) in enumerate(zip(all_file, reformated_core_utilization_file)):\n",
+    "            print(file_i, file_ii)\n",
     "            info = break_file_name(file_i)\n",
     "            file_daqconf_i='daqconf-{}-{}-{}-{}'.format(info[4], info[5], info[2], info[3])\n",
     "            var_i='ru{}{}{}'.format(info[2], info[4], '0')\n",
-    "            json_info(file_daqconf=file_daqconf_i, file_core=file_ii, input_directory=daqconfs_cpupins_folder_parent_dir, input_dir=input_dir, var=var_i, pdf=pdf, if_pdf=print_info, repin_threads_file=repin_threads_file)\n",
+    "            if use_daqconf:\n",
+    "                json_info(file_daqconf=file_daqconf_i, file_core=file_ii, input_directory=daqconfs_cpupins_folder_parent_dir, input_dir=input_dir, var=var_i, pdf=pdf, if_pdf=print_info, repin_threads_file=repin_threads_file)\n",
+    "            else:\n",
+    "                json_info_general(file_core=file_ii, input_directory=daqconfs_cpupins_folder_parent_dir, input_dir=input_dir, var=var_i, pdf=pdf, if_pdf=print_info, file_cpupin=repin_threads_file[i], use_daqconf=use_daqconf)\n",
     "            pdf.cell(0, 10, 'Table {}. CPU core pins information for the \"{}\" test.'.format(i+2, info[5]))\n",
     "            pdf.ln(10)\n",
     "            \n",
@@ -391,28 +389,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "ea693364",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Data saved to CSV successfully: /eos/user/m/mman/performancetest/tools/queuetest_results/grafana-v4_2_1-np02srv003-0-eth-queuetest_Intel.csv\n",
-      "done :-)\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "# mman spsc queue test app\n",
+    "# mman callback test app\n",
     "\n",
     "grafana_url = 'http://np04-srv-009.cern.ch:3000'\n",
     "dashboard_uid = ['91zWmJEVk']\n",
     "host_used = 'np02-srv-003'  \n",
-    "delta_time = [['2024-01-25 13:57:17', '2024-01-25 14:04:27']]\n",
-    "output_csv_file = ['v4_2_1-np02srv003-0-eth-queuetest_Intel']\n",
-    "results_path = '/eos/user/m/mman/performancetest/tools/queuetest_results'\n",
+    "delta_time = [['2024-02-26 11:08:56', '2024-02-26 11:10:32'],\n",
+    "              ['2024-02-26 11:11:00', '2024-02-26 11:12:34']]\n",
+    "output_csv_file = ['v4_2_1-np02srv003-1-eth-callback_Intel',\n",
+    "                   'v4_2_1-np02srv003-1-eth-queue_Intel']\n",
+    "results_path = '/eos/user/m/mman/performancetest/tools/callbacktest_results'\n",
     "\n",
     "for delta_time_list, output_csv_file_list in zip(delta_time, output_csv_file):\n",
     "    extract_data_and_stats_from_panel(grafana_url, dashboard_uid, delta_time=delta_time_list, host=host_used, \n",
@@ -455,67 +446,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "4c688dcc",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Finish the processing of the data.\n",
-      "0      0.049003\n",
-      "1      0.150302\n",
-      "2      0.062094\n",
-      "3      0.507142\n",
-      "4      0.175968\n",
-      "         ...   \n",
-      "211    0.119392\n",
-      "212    0.052317\n",
-      "213    0.046248\n",
-      "214    0.123093\n",
-      "215    0.101806\n",
-      "Name: Socket0 L3 Cache Misses, Length: 216, dtype: float64\n",
-      "0      0.302006\n",
-      "1      0.527486\n",
-      "2      0.314720\n",
-      "3      1.011582\n",
-      "4      0.678650\n",
-      "         ...   \n",
-      "211    0.445688\n",
-      "212    0.279797\n",
-      "213    0.275933\n",
-      "214    0.446962\n",
-      "215    0.499568\n",
-      "Length: 216, dtype: float64\n",
-      "0      16.225837\n",
-      "1      28.493905\n",
-      "2      19.730078\n",
-      "3      50.133504\n",
-      "4      25.929179\n",
-      "         ...    \n",
-      "211    26.788277\n",
-      "212    18.698232\n",
-      "213    16.760741\n",
-      "214    27.540037\n",
-      "215    20.378907\n",
-      "Length: 216, dtype: float64\n",
-      "THE END\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "## mman readout queue perftest\n",
-    "results_path = '/eos/user/m/mman/performancetest/tools/queuetest_results'\n",
-    "report_path = '/eos/user/m/mman/performancetest/tools/performance_reports'\n",
-    "performancetest_path = '/eos/user/m/mman/performancetest'\n",
+    "## mman readout callback perftest\n",
+    "results_path = '/eos/home-m/mman/performancetest/tools/callbacktest_results_v2'\n",
+    "report_path = '/eos/home-m/mman/performancetest/tools/performance_reports'\n",
+    "performancetest_path = '/eos/home-m/mman/performancetest'\n",
+    "repin_files = ['cpupin-eth-mockdlh_grouped-np02srv004-1.json','cpupin-eth-mockdlh_mixed-np02srv004-1.json','cpupin-eth-mockdlh_grouped-np02srv004-1.json','cpupin-eth-mockdlh_mixed-np02srv004-1.json']\n",
     "\n",
     "create_report_performance(input_dir=results_path, output_dir=report_path, \n",
     "                          daqconfs_cpupins_folder_parent_dir=performancetest_path, \n",
     "                          process_pcm_files=True, process_uprof_files=True, \n",
-    "                          print_info=True, streams='4', \n",
-    "                          pdf_name='queuetest_AMD_debug',\n",
-    "                          elog_message=['Unable to push within timeout period (timeout period was 0 milliseconds)', 'Unable to push within timeout period (timeout period was 0 milliseconds)'])\n",
+    "                          print_info=True, streams='10',\n",
+    "                          pdf_name='callbacktest_v2',\n",
+    "                          elog_message=['', '','',''],\n",
+    "                          repin_threads_file=repin_files,\n",
+    "                          use_daqconf=False)\n",
     "\n",
     "print('THE END')"
    ]
@@ -588,7 +537,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "d2504ce3",
+   "id": "56c6f624",
    "metadata": {},
    "outputs": [],
    "source": []