Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Jenkins][Tests] Add stage with -DMIOPEN_USE_COMPOSABLEKERNEL=Off after #2517 #2630. #2631

Merged
merged 7 commits into from
Dec 21, 2023
Merged

[Jenkins][Tests] Add stage with -DMIOPEN_USE_COMPOSABLEKERNEL=Off after #2517 #2630. #2631

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7c6f2a4
[HOTFIX] Fix build with -DMIOPEN_USE_COMPOSABLEKERNEL=Off after #2517.
junliume Dec 21, 2023
623d146
add -DMIOPEN_USE_COMPOSABLEKERNEL=Off stage
junliume Dec 21, 2023
b0d49a2
Merge branch 'develop' into atamazov-fix-nock-after-pr2517
junliume Dec 21, 2023
5c71307
make NOCK stage anyAPU and build ONLY
junliume Dec 21, 2023
82ae483
Adopt recommended changes
junliume Dec 21, 2023
8a976bb
rename config_targets to make_targets
junliume Dec 21, 2023
fe1b539
Extend GTest DISCOVERY_TIMEOUT to 5 mins
junliume Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 45 additions & 32 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def show_node_info() {
def cmake_build(Map conf=[:]){

def compiler = conf.get("compiler","/opt/rocm/llvm/bin/clang++")
def config_targets = conf.get("config_targets","check")
def make_targets = conf.get("make_targets","check")
def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "")
def build_envs = "CTEST_PARALLEL_LEVEL=4 MIOPEN_CONV_PRECISE_ROCBLAS_TIMING=0 " + conf.get("build_env","")
def prefixpath = conf.get("prefixpath","/opt/rocm")
Expand All @@ -47,13 +47,13 @@ def cmake_build(Map conf=[:]){
def package_build = (conf.get("package_build","") == "true")

if (package_build == true) {
config_targets = "package"
make_targets = "package"
}

def miopen_install_path = "${env.WORKSPACE}/install"
if(conf.get("build_install","") == "true")
{
config_targets = 'install ' + config_targets
make_targets = 'install ' + make_targets
setup_args = " -DBUILD_DEV=Off -DCMAKE_INSTALL_PREFIX=${miopen_install_path}" + setup_args
} else{
setup_args = ' -DBUILD_DEV=On' + setup_args
Expand Down Expand Up @@ -110,7 +110,7 @@ def cmake_build(Map conf=[:]){
def setup_cmd = conf.get("setup_cmd", "${cmake_envs} cmake ${setup_args} .. ")
// WORKAROUND_SWDEV_290754
// It seems like this W/A is not required since 4.5.
def build_cmd = conf.get("build_cmd", "LLVM_PATH=/opt/rocm/llvm ${build_envs} dumb-init make -j\$(nproc) ${config_targets}")
def build_cmd = conf.get("build_cmd", "LLVM_PATH=/opt/rocm/llvm ${build_envs} dumb-init make -j\$(nproc) ${make_targets}")
def execute_cmd = conf.get("execute_cmd", "")

def cmd = conf.get("cmd", """
Expand Down Expand Up @@ -149,7 +149,7 @@ def cmake_build(Map conf=[:]){
def cmake_fin_build_cmd(prefixpath){
def flags = "-DCMAKE_INSTALL_PREFIX=${prefixpath} -DCMAKE_BUILD_TYPE=release"
def compiler = 'clang++'
def config_targets = "install"
def make_targets = "install"
def compilerpath = "/opt/rocm/llvm/bin/" + compiler
def configargs = ""
if (prefixpath != "")
Expand All @@ -164,7 +164,7 @@ def cmake_fin_build_cmd(prefixpath){
mkdir build
cd build
CXX=${compilerpath} cmake ${configargs} ${flags} ..
dumb-init make -j\$(nproc) ${config_targets}
dumb-init make -j\$(nproc) ${make_targets}
"""
return fin_cmd
}
Expand Down Expand Up @@ -406,8 +406,8 @@ def CheckPerfDbValid(Map conf=[:]){
/// * The default compiler is usually not specified.
/// BuildType := { Release* | Debug | Install } [ BuildTypeModifier ]
/// * BuildTypeModifier := { NOCOMGR | Embedded | Static | Normal-Find | Fast-Find
/// CK | NOMLIR | Tensile | Tensile-Latest | Package | ... }
/// TestSet := { All | Smoke* | Performance Dataset } [ Codecov ]
/// NOCK | NOMLIR | Tensile | Tensile-Latest | Package | ... }
/// TestSet := { All | Smoke* | <Performance Dataset> | Build-only } [ Codecov ]
/// * "All" corresponds to "cmake -DMIOPEN_TEST_ALL=On".
/// * "Smoke" (-DMIOPEN_TEST_ALL=Off) is the default and usually not specified.
/// * "Codecov" is optional code coverage analysis.
Expand Down Expand Up @@ -613,7 +613,7 @@ pipeline {
fin_flags = "-DMIOPEN_BACKEND=HIPNOGPU"
}
steps{
buildHipClangJobAndReboot(setup_flags: fin_flags, config_targets: "all", build_fin: "ON", needs_gpu:false, needs_reboot:false, build_install: "true")
buildHipClangJobAndReboot(setup_flags: fin_flags, make_targets: "all", build_fin: "ON", needs_gpu:false, needs_reboot:false, build_install: "true")
}
}
}
Expand All @@ -633,7 +633,7 @@ pipeline {
}
agent{ label rocmnode("vega || gfx908 || gfx90a") }
steps{
buildHipClangJobAndReboot(config_targets: Smoke_targets)
buildHipClangJobAndReboot(make_targets: Smoke_targets)
}
}
stage('Fp32 Hip Debug AnyGPU') {
Expand All @@ -646,7 +646,7 @@ pipeline {
}
agent{ label rocmnode("vega || gfx908 || gfx90a") }
steps{
buildHipClangJobAndReboot(build_type: 'debug', config_targets: Smoke_targets)
buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets)
}
}
stage('Fp32 Hip Debug gfx908') {
Expand All @@ -659,7 +659,7 @@ pipeline {
}
agent{ label rocmnode("gfx908") }
steps{
buildHipClangJobAndReboot(build_type: 'debug', config_targets: Smoke_targets)
buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets)
}
}
stage('Fp32 Hip Debug gfx90a') {
Expand All @@ -672,7 +672,7 @@ pipeline {
}
agent{ label rocmnode("gfx90a") }
steps{
buildHipClangJobAndReboot(build_type: 'debug', config_targets: Smoke_targets)
buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets)
}
}
stage('Fp32 Hip Debug gfx94X') {
Expand All @@ -685,7 +685,7 @@ pipeline {
}
agent{ label rocmnode("gfx94X") }
steps{
buildHipClangJobAndReboot(build_type: 'debug', config_targets: Smoke_targets)
buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets)
}
}
}
Expand Down Expand Up @@ -729,6 +729,19 @@ pipeline {
buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ')
}
}
stage('Fp32 Hip Debug NOCK AnyGPU Build-Only') {
when {
beforeAgent true
expression { params.TARGET_VEGA20 || params.TARGET_VEGA10 || params.TARGET_GFX908 || params.TARGET_GFX90A }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[R] Add TARGET_ANYGPU and use it here and everywhere.

Suggested change
expression { params.TARGET_VEGA20 || params.TARGET_VEGA10 || params.TARGET_GFX908 || params.TARGET_GFX90A }
expression { params.TARGET_ANYGPU }

Use case: The user wants to avoid usage of any gfx908 nodes. Clearing the params.TARGET_GFX908 flag does not guarantee this because ANYGPU stages occupy the 1st available node, which could be gfx908. Disabling ANYGPU stages is only possible via disabling Vega, gfx908 and gfx90a targets, which is not what the user wants.

With the new flag, the user can clear params.TARGET_GFX908 and params.TARGET_ANYGPU and obtain the desired effect.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hold off on this one. While we are adding targets, I'll make sure they do not immediately becomes CI bottleneck.

}
options {
retry(2)
}
agent{ label rocmnode("vega || gfx908 || gfx90a") }
steps{
buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "")
}
}
stage('Fp32 Hip Debug Embedded Vega20') {
when {
beforeAgent true
Expand Down Expand Up @@ -768,11 +781,11 @@ pipeline {
}
agent{ label rocmnode("vega || gfx908 || gfx90a") }
environment{
config_targets = "test_conv2d"
make_targets = "test_conv2d"
execute_cmd = "MIOPEN_CONV_PRECISE_ROCBLAS_TIMING=0 bin/test_conv2d --disable-verification-cache"
}
steps{
buildHipClangJobAndReboot(config_targets: config_targets, execute_cmd: execute_cmd, find_mode: "Normal")
buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal")
}
}
stage('Fp32 Hip Fast-Find AnyGPU') {
Expand All @@ -785,11 +798,11 @@ pipeline {
}
agent{ label rocmnode("vega || gfx908 || gfx90a") }
environment{
config_targets = "test_conv2d"
make_targets = "test_conv2d"
execute_cmd = "MIOPEN_FIND_MODE=2 CTEST_PARALLEL_LEVEL=4 MIOPEN_CONV_PRECISE_ROCBLAS_TIMING=0 bin/test_conv2d --disable-verification-cache"
}
steps{
buildHipClangJobAndReboot( config_targets: config_targets, execute_cmd: execute_cmd)
buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd)
}
}
stage('Fp32 Hip AnyGPU') {
Expand Down Expand Up @@ -822,7 +835,7 @@ pipeline {
}
agent{ label rocmnode("vega20") }
steps{
buildHipClangJobAndReboot( setup_flags: Fp16_flags, config_targets: Smoke_targets)
buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets)
}
}
stage('Bf16 Hip Vega20') {
Expand All @@ -835,7 +848,7 @@ pipeline {
}
agent{ label rocmnode("vega20") }
steps{
buildHipClangJobAndReboot(setup_flags: Bf16_flags, config_targets: Smoke_targets)
buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets)
}
}
stage('Fp16 Hip gfx908') {
Expand All @@ -848,7 +861,7 @@ pipeline {
}
agent{ label rocmnode("gfx908") }
steps{
buildHipClangJobAndReboot( setup_flags: Fp16_flags, config_targets: Smoke_targets)
buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets)
}
}
stage('Bf16 Hip gfx908') {
Expand All @@ -861,7 +874,7 @@ pipeline {
}
agent{ label rocmnode("gfx908") }
steps{
buildHipClangJobAndReboot(setup_flags: Bf16_flags, config_targets: Smoke_targets)
buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets)
}
}
stage('Fp16 Hip gfx90a') {
Expand All @@ -874,7 +887,7 @@ pipeline {
}
agent{ label rocmnode("gfx90a") }
steps{
buildHipClangJobAndReboot( setup_flags: Fp16_flags, config_targets: Smoke_targets)
buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets)
}
}
stage('Bf16 Hip gfx90a') {
Expand All @@ -887,7 +900,7 @@ pipeline {
}
agent{ label rocmnode("gfx90a") }
steps{
buildHipClangJobAndReboot(setup_flags: Bf16_flags, config_targets: Smoke_targets)
buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets)
}
}
stage('Fp16 Hip gfx94X') {
Expand All @@ -900,7 +913,7 @@ pipeline {
}
agent{ label rocmnode("gfx94X") }
steps{
buildHipClangJobAndReboot( setup_flags: Fp16_flags, config_targets: Smoke_targets)
buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets)
}
}
stage('Bf16 Hip gfx94X') {
Expand All @@ -913,7 +926,7 @@ pipeline {
}
agent{ label rocmnode("gfx94X") }
steps{
buildHipClangJobAndReboot(setup_flags: Bf16_flags, config_targets: Smoke_targets)
buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets)
}
}
}
Expand All @@ -939,11 +952,11 @@ pipeline {
agent{ label rocmnode("gfx908") }
environment{
setup_flags="-DMIOPEN_TEST_DBSYNC=1"
config_targets='test_db_sync'
execute_cmd='./bin/test_db_sync'
make_targets='test_db_sync'
execute_cmd='./bin/test_db_sync'
}
steps{
buildHipClangJobAndReboot(lfs_pull: true, setup_flags: setup_flags, config_targets: config_targets, execute_cmd: execute_cmd,
buildHipClangJobAndReboot(lfs_pull: true, setup_flags: setup_flags, make_targets: make_targets, execute_cmd: execute_cmd,
needs_gpu:false, needs_reboot:false, build_install: "true")
}
}
Expand All @@ -958,11 +971,11 @@ pipeline {
agent{ label rocmnode("gfx90a") }
environment{
setup_flags="-DMIOPEN_TEST_DBSYNC=1"
config_targets='test_db_sync'
execute_cmd='./bin/test_db_sync'
make_targets='test_db_sync'
execute_cmd='./bin/test_db_sync'
}
steps{
buildHipClangJobAndReboot(lfs_pull: true, setup_flags: setup_flags, config_targets: config_targets, execute_cmd: execute_cmd,
buildHipClangJobAndReboot(lfs_pull: true, setup_flags: setup_flags, make_targets: make_targets, execute_cmd: execute_cmd,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😄 Tautology ;)

needs_gpu:false, needs_reboot:false, build_install: "true")
}
}
Expand Down
3 changes: 2 additions & 1 deletion test/gtest/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ function(add_gtest TEST_NAME)
endif()
if(NOT WIN32) # TODO: cannot run on Windows due to missing DLL dependencies
# Enable CMake to discover the test binary
gtest_discover_tests(test_${TEST_NAME} PROPERTIES ENVIRONMENT "MIOPEN_USER_DB_PATH=${CMAKE_CURRENT_BINARY_DIR};MIOPEN_TEST_FLOAT_ARG=${MIOPEN_TEST_FLOAT_ARG};MIOPEN_TEST_ALL=${MIOPEN_TEST_ALL};MIOPEN_TEST_MLIR=${MIOPEN_TEST_MLIR};MIOPEN_TEST_COMPOSABLEKERNEL=${MIOPEN_TEST_COMPOSABLEKERNEL}")
# Extend GTest DISCOVERY_TIMEOUT to 5 mins
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@junliume 👍 I run into the same problem

gtest_discover_tests(test_${TEST_NAME} DISCOVERY_TIMEOUT 300 PROPERTIES ENVIRONMENT "MIOPEN_USER_DB_PATH=${CMAKE_CURRENT_BINARY_DIR};MIOPEN_TEST_FLOAT_ARG=${MIOPEN_TEST_FLOAT_ARG};MIOPEN_TEST_ALL=${MIOPEN_TEST_ALL};MIOPEN_TEST_MLIR=${MIOPEN_TEST_MLIR};MIOPEN_TEST_COMPOSABLEKERNEL=${MIOPEN_TEST_COMPOSABLEKERNEL}")
endif()

if(WIN32)
Expand Down