[PH] Performance Harness Updates

.github/workflows/performance_harness_run.yaml

@@ -12,6 +12,16 @@ on:
       override-test-params:
         description: 'Override perf harness params'
         type: string
+      override-leap:
+        description: Override leap target
+        type: string
+      override-leap-prerelease:
+        type: choice
+        description: Override leap prelease
+        options:
+        - default
+        - true
+        - false
 
 permissions:
   packages: read
@@ -28,15 +38,25 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       test-params: ${{steps.overrides.outputs.test-params}}
+      leap-target: ${{steps.overrides.outputs.leap-target}}
+      leap-prerelease: ${{steps.overrides.outputs.leap-prerelease}}
     steps:
       - name: Setup Input Params
         id: overrides
         run: |
           echo test-params=findMax testBpOpMode >> $GITHUB_OUTPUT
+          echo leap-target="DEFAULT" >> $GITHUB_OUTPUT
+          echo leap-prerelease="false" >> $GITHUB_OUTPUT
 
           if [[ "${{inputs.override-test-params}}" != "" ]]; then
             echo test-params=${{inputs.override-test-params}} >> $GITHUB_OUTPUT
           fi
+          if [[ "${{inputs.override-leap}}" != "" ]]; then
+            echo leap-target=${{inputs.override-leap}} >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{inputs.override-leap-prerelease}}" == +(true|false) ]]; then
+            echo leap-prerelease=${{inputs.override-leap-prerelease}} >> $GITHUB_OUTPUT
+          fi
 
   platforms:
     name: Run Platforms Workflow
@@ -91,30 +111,52 @@ jobs:
     name: Tests
     needs: [v, platforms, reuse-build, build-base]
     if: always() && needs.platforms.result == 'success' && (needs.build-base.result == 'success' || needs.reuse-build.result == 'success')
-    runs-on: ubuntu-latest
+    runs-on: ["Leap-Perf-Ubuntu-22-16x64x600"]
     container:
       image: ${{fromJSON(needs.platforms.outputs.p)[github.event.inputs.platform-choice].image}}
     steps:
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
           name: ${{github.event.inputs.platform-choice}}-build
-      - name: Run Performance Test
+      - name: Extract Build Directory
         run: |
           zstdcat build.tar.zst | tar x
+      - if: ${{ needs.v.outputs.leap-target != 'DEFAULT' }}
+        name: Download Prev Leap Version
+        uses: AntelopeIO/asset-artifact-download-action@v3
+        with:
+          owner: AntelopeIO
+          repo: leap
+          target: '${{needs.v.outputs.leap-target}}'
+          prereleases: ${{fromJSON(needs.v.outputs.leap-prerelease)}}
+          file: 'leap.*${{github.event.inputs.platform-choice}}.*(x86_64|amd64).deb'
+      - if: ${{ needs.v.outputs.leap-target != 'DEFAULT' }}
+        name: Install leap & replace binaries for PH use
+        run: |
+          apt-get update
+          apt-get install -y ./leap*.deb
+          rm build/bin/nodeos
+          rm build/bin/cleos
+          cp /usr/bin/nodeos build/bin
+          cp /usr/bin/cleos build/bin
+          ./build/bin/nodeos --full-version
+          ./build/bin/cleos version full
+      - name: Run Performance Test
+        run: |
           cd build
           ./tests/PerformanceHarnessScenarioRunner.py ${{needs.v.outputs.test-params}}
       - name: Prepare results
         id: prep-results
         run: |
-          tar -pc build/PerformanceHarnessScenarioRunnerLogs | zstd --long -T0 -9 > PerformanceHarnessScenarioRunnerLogs.tar.zst
+          tar -pc build/PHSRLogs | zstd --long -T0 -9 > PHSRLogs.tar.zst
       - name: Upload results
         uses: AntelopeIO/upload-artifact-large-chunks-action@v1
         with:
           name: performance-test-results
-          path: PerformanceHarnessScenarioRunnerLogs.tar.zst
+          path: PHSRLogs.tar.zst
       - name: Upload report
         uses: actions/upload-artifact@v3
         with:
           name: performance-test-report
-          path: ./build/PerformanceHarnessScenarioRunnerLogs/**/report.json
+          path: ./build/PHSRLogs/**/report.json

.github/workflows/ph_backward_compatibility.yaml

@@ -67,7 +67,8 @@ jobs:
           rm build/bin/cleos
           cp /usr/bin/nodeos build/bin
           cp /usr/bin/cleos build/bin
-          ./build/bin/nodeos --version
+          ./build/bin/nodeos --full-version
+          ./build/bin/cleos version full
       - if: ${{ matrix.release == '3.1' || matrix.release == '3.2' }}
         name: Run Performance Tests (<v4.0)
         run: |

tests/CMakeLists.txt

@@ -274,15 +274,15 @@ set_property(TEST gelf_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME performance_test_bp COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testBpOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 --calc-chain-threads lmax overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_api COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testApiOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 --calc-chain-threads lmax overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-add_test(NAME performance_test_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testApiOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --tps-limit-per-generator 25 --api-nodes-read-only-threads 2 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME performance_test_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testApiOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --tps-limit-per-generator 25 --api-nodes-read-only-threads 2 --read-only-write-window-time-us 1000 --read-only-read-window-time-us 165000 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_cpu_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testBpOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_p2p COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_http COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --endpoint-mode http --producer-nodes 1 --validation-nodes 1 --api-nodes 1 --target-tps 10 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_transfer_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --user-trx-data-file tests/PerformanceHarness/userTrxDataTransfer.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_new_acct_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --user-trx-data-file tests/PerformanceHarness/userTrxDataNewAccount.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_cpu_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME performance_test_basic_ram_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "r" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/ramTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-add_test(NAME performance_test_basic_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --endpoint-mode http --producer-nodes 1 --validation-nodes 1 --api-nodes 1 --api-nodes-read-only-threads 2 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME performance_test_basic_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --endpoint-mode http --producer-nodes 1 --validation-nodes 1 --api-nodes 1 --api-nodes-read-only-threads 2 --read-only-write-window-time-us 1000 --read-only-read-window-time-us 165000 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST performance_test_bp PROPERTY LABELS long_running_tests)
 set_property(TEST performance_test_api PROPERTY LABELS long_running_tests)
 set_property(TEST performance_test_read_only_trxs PROPERTY LABELS long_running_tests)