From 0a56f16154ae68f4ae7732ee32a95cd54ce51b1d Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Mon, 25 Nov 2024 20:22:55 +0100 Subject: [PATCH] CI: implement GPU test support (#96) Implement support for running unit tests on Nvidia and AMD GPU runner. Implement unit tests for all unit test generator functions. --- .ci/CI/src/Bootloader.jl | 35 +++++ .ci/CI/src/CI.jl | 1 + .ci/CI/src/modules/UnitTest.jl | 115 ++++++++++++++-- .ci/CI/test/UnitTest/amdgpu_unit_test.jl | 74 +++++++++++ .ci/CI/test/UnitTest/cuda_unit_test.jl | 72 ++++++++++ .ci/CI/test/UnitTest/nightly_unit_test.jl | 94 +++++++++++++ .ci/CI/test/UnitTest/normal_unit_test.jl | 155 ++++++++++++++++++++++ .ci/CI/test/UnitTest/runtests.jl | 6 + .ci/CI/test/UnitTest/utils.jl | 66 +++++++++ .ci/CI/test/UnitTest/verify_unit_test.jl | 37 ++++++ .ci/CI/test/generate_job_yaml.jl | 19 --- .ci/CI/test/runtests.jl | 2 + .ci/CI/test/utils.jl | 45 +++++++ 13 files changed, 693 insertions(+), 28 deletions(-) create mode 100644 .ci/CI/test/UnitTest/amdgpu_unit_test.jl create mode 100644 .ci/CI/test/UnitTest/cuda_unit_test.jl create mode 100644 .ci/CI/test/UnitTest/nightly_unit_test.jl create mode 100644 .ci/CI/test/UnitTest/normal_unit_test.jl create mode 100644 .ci/CI/test/UnitTest/runtests.jl create mode 100644 .ci/CI/test/UnitTest/utils.jl create mode 100644 .ci/CI/test/UnitTest/verify_unit_test.jl create mode 100644 .ci/CI/test/utils.jl diff --git a/.ci/CI/src/Bootloader.jl b/.ci/CI/src/Bootloader.jl index cf9ce9a..f60c4f7 100644 --- a/.ci/CI/src/Bootloader.jl +++ b/.ci/CI/src/Bootloader.jl @@ -186,10 +186,45 @@ function main() test_package, unit_test_julia_versions, target_branch, + CPU, tools_git_repo, get_unit_test_nightly_baseimage(), ) + if haskey(ENV, "CI_ENABLE_CUDA_TESTS") || haskey(ENV, "CI_ENABLE_AMDGPU_TESTS") + for version in ["rc", "nightly"] + if version in unit_test_julia_versions + @info "Remove unit test version $(version) for GPU tests" + filter!(v -> v != version, unit_test_julia_versions) + end + end + end + + if haskey(ENV, "CI_ENABLE_CUDA_TESTS") + @info "Generate CUDA unit tests" + add_unit_test_job_yaml!( + job_yaml, + test_package, + unit_test_julia_versions, + target_branch, + CUDA, + tools_git_repo, + ) + end + + if haskey(ENV, "CI_ENABLE_AMDGPU_TESTS") + @info "Generate AMDGPU unit tests" + + add_unit_test_job_yaml!( + job_yaml, + test_package, + unit_test_julia_versions, + target_branch, + AMDGPU, + tools_git_repo, + ) + end + add_integration_test_job_yaml!(job_yaml, test_package, target_branch, tools_git_repo) add_unit_test_verify_job_yaml!(job_yaml, target_branch, tools_git_repo) diff --git a/.ci/CI/src/CI.jl b/.ci/CI/src/CI.jl index 129c13b..9f17bac 100644 --- a/.ci/CI/src/CI.jl +++ b/.ci/CI/src/CI.jl @@ -1,6 +1,7 @@ module CI include("./modules/Utils.jl") +include("./modules/UnitTest.jl") include("./modules/IntegTest.jl") include("./modules/GitLabTargetBranch.jl") include("./SetupDevEnv.jl") diff --git a/.ci/CI/src/modules/UnitTest.jl b/.ci/CI/src/modules/UnitTest.jl index 809718f..03f3bc7 100644 --- a/.ci/CI/src/modules/UnitTest.jl +++ b/.ci/CI/src/modules/UnitTest.jl @@ -1,3 +1,7 @@ +""" +Specify target processor for the tests. +""" +@enum TestPlatform CPU CUDA AMDGPU ONEAPI METAL """ add_unit_test_job_yaml!( @@ -5,6 +9,7 @@ test_package::TestPackage, julia_versions::Vector{String}, target_branch::AbstractString, + test_platform::TestPlatform=CPU, tools_git_repo::ToolsGitRepo=ToolsGitRepo( "https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git", "dev" ), @@ -19,6 +24,7 @@ to be directly translated to GitLab CI yaml. - `test_package::TestPackage`: Properties of the package to be tested, such as name and version. - `julia_versions::Vector{String}`: Julia version used for the tests. - `target_branch::AbstractString`: A different job code is generated depending on the target branch. +- `test_platform::TestPlatform`: Set target platform test, e.g. CPU, Nvidia GPU or AMD GPU. - `tools_git_repo::ToolsGitRepo`: URL and branch of the Git repository from which the CI tools are cloned in unit test job. - `nightly_base_image::AbstractString`: Name of the job base image if the Julia version is nightly. @@ -28,11 +34,16 @@ function add_unit_test_job_yaml!( test_package::TestPackage, julia_versions::Vector{String}, target_branch::AbstractString, + test_platform::TestPlatform=CPU, tools_git_repo::ToolsGitRepo=ToolsGitRepo( "https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git", "dev" ), nightly_base_image::AbstractString="debian:bookworm-slim", ) + if test_platform in [ONEAPI, METAL] + throw(ArgumentError("argument test_platform not implemented for $(test_platform)")) + end + if !haskey(job_dict, "stages") job_dict["stages"] = [] end @@ -40,13 +51,24 @@ function add_unit_test_job_yaml!( push!(job_dict["stages"], "unit-test") for version in julia_versions + test_platform_name = lowercase(string(test_platform)) if version != "nightly" - job_dict["unit_test_julia_$(replace(version, "." => "_"))"] = _get_normal_unit_test( - version, test_package, target_branch, tools_git_repo - ) + if test_platform == AMDGPU + job_dict["unit_test_julia_$(test_platform_name)_$(replace(version, "." => "_"))"] = _get_amdgpu_unit_test( + version, test_package, target_branch, tools_git_repo + ) + else + job_dict["unit_test_julia_$(test_platform_name)_$(replace(version, "." => "_"))"] = _get_normal_unit_test( + version, test_package, target_branch, test_platform, tools_git_repo + ) + end else - job_dict["unit_test_julia_nightly"] = _get_nightly_unit_test( - test_package, target_branch, tools_git_repo, nightly_base_image + job_dict["unit_test_julia_$(test_platform_name)_nightly"] = _get_nightly_unit_test( + test_package, + target_branch, + test_platform, + tools_git_repo, + nightly_base_image, ) end end @@ -79,6 +101,9 @@ function add_unit_test_verify_job_yaml!( ) # verification script that no custom URLs are used in unit tests if target_branch != "main" + if !haskey(job_dict, "stages") + job_dict["stages"] = [] + end push!(job_dict["stages"], "verify-unit-test-deps") job_dict["verify-unit-test-deps"] = Dict( "image" => "julia:1.10", @@ -99,6 +124,7 @@ end version::AbstractString, test_package::TestPackage, target_branch::AbstractString, + test_platform::TestPlatform, tools_git_repo::ToolsGitRepo, ) @@ -108,6 +134,7 @@ Creates a normal unit test job for a specific Julia version. - `version::AbstractString`: Julia version used for the tests. - `test_package::TestPackage`: Properties of the package to be tested, such as name and version. - `target_branch::AbstractString`: A different job code is generated depending on the target branch. +- `test_platform::TestPlatform`: Set target platform test, e.g. CPU, Nvidia GPU or AMD GPU. - `tools_git_repo::ToolsGitRepo`: URL and branch of the Git repository from which the CI tools are cloned in unit test job. @@ -119,6 +146,7 @@ function _get_normal_unit_test( version::AbstractString, test_package::TestPackage, target_branch::AbstractString, + test_platform::TestPlatform, tools_git_repo::ToolsGitRepo, )::Dict job_yaml = Dict() @@ -130,6 +158,14 @@ function _get_normal_unit_test( ) job_yaml["image"] = "julia:$(version)" + if !haskey(job_yaml, "variables") + job_yaml["variables"] = Dict() + end + + for tp in instances(TestPlatform) + job_yaml["variables"]["TEST_$(tp)"] = (tp == test_platform) ? "1" : "0" + end + script = [ "apt update && apt install -y git", "git clone --depth 1 -b $(tools_git_repo.branch) $(tools_git_repo.url) /tmp/integration_test_tools/", @@ -157,7 +193,20 @@ function _get_normal_unit_test( job_yaml["script"] = script job_yaml["interruptible"] = true - job_yaml["tags"] = ["cpuonly"] + + if test_platform == CPU + job_yaml["tags"] = ["cpuonly"] + elseif test_platform == CUDA + job_yaml["tags"] = ["cuda", "x86_64"] + elseif test_platform == AMDGPU + job_yaml["tags"] = ["rocm", "x86_64"] + else + throw( + ArgumentError( + "test_platform argument with value $(test_platform) not supported" + ), + ) + end return job_yaml end @@ -166,13 +215,16 @@ end _get_nightly_unit_test( test_package::TestPackage, target_branch::AbstractString, + test_platform::TestPlatform, tools_git_repo::ToolsGitRepo, + nightly_base_image::AbstractString, ) Creates a unit test job which uses the Julia nightly version. # Args - `test_package::TestPackage`: Properties of the package to be tested, such as name and version. +- `test_platform::TestPlatform`: Set target platform test, e.g. CPU, Nvidia GPU or AMD GPU. - `target_branch::AbstractString`: A different job code is generated depending on the target branch. - `tools_git_repo::ToolsGitRepo`: URL and branch of the Git repository from which the CI tools are cloned in unit test job. @@ -185,10 +237,13 @@ Returns a dict containing the unit test, which can be output directly as GitLab function _get_nightly_unit_test( test_package::TestPackage, target_branch::AbstractString, + test_platform::TestPlatform, tools_git_repo::ToolsGitRepo, nightly_base_image::AbstractString, ) - job_yaml = _get_normal_unit_test("1", test_package, target_branch, tools_git_repo) + job_yaml = _get_normal_unit_test( + "1", test_package, target_branch, test_platform, tools_git_repo + ) job_yaml["image"] = nightly_base_image if !haskey(job_yaml, "variables") @@ -218,8 +273,50 @@ fi", "cp -r \$JULIA_EXTRACT_FOLDER/* /usr", ] - job_yaml["image"] = "debian:bookworm-slim" - job_yaml["allow_failure"] = true return job_yaml end + +""" + _get_amdgpu_unit_test( + version::AbstractString, + test_package::TestPackage, + target_branch::AbstractString, + tools_git_repo::ToolsGitRepo, + ) + +Creates a amdgpu unit test job for a specific Julia version. Use a different base image and install +Julia in it. + +# Args +- `version::AbstractString`: Julia version used for the tests. +- `test_package::TestPackage`: Properties of the package to be tested, such as name and version. +- `target_branch::AbstractString`: A different job code is generated depending on the target branch. +- `tools_git_repo::ToolsGitRepo`: URL and branch of the Git repository from which the CI tools are + cloned in unit test job. + +Return + +Returns a dict containing the unit test, which can be output directly as GitLab CI yaml. +""" +function _get_amdgpu_unit_test( + version::AbstractString, + test_package::TestPackage, + target_branch::AbstractString, + tools_git_repo::ToolsGitRepo, +)::Dict + job_yaml = _get_normal_unit_test( + version, test_package, target_branch, AMDGPU, tools_git_repo + ) + job_yaml["image"] = "rocm/dev-ubuntu-24.04:6.2.4-complete" + job_yaml["before_script"] = [ + "curl -fsSL https://install.julialang.org | sh -s -- -y -p /julia", + "export PATH=/julia/bin:\$PATH", + "echo \$PATH", + "juliaup add $(version)", + "juliaup default $(version)", + ] + job_yaml["tags"] = ["rocm", "x86_64"] + + return job_yaml +end diff --git a/.ci/CI/test/UnitTest/amdgpu_unit_test.jl b/.ci/CI/test/UnitTest/amdgpu_unit_test.jl new file mode 100644 index 0000000..4149b9d --- /dev/null +++ b/.ci/CI/test/UnitTest/amdgpu_unit_test.jl @@ -0,0 +1,74 @@ +@testset "test _get_amdgpu_unit_test" begin + test_package = CI.TestPackage("QEDfoo", "/path/to/project", "42.0") + git_url = "http://github.com/name/repo" + git_branch = "branch" + + expected_job = get_generic_unit_job("1.11", test_package) + + expected_job["script"] = get_main_unit_job_script_section(git_url, git_branch) + + expected_job["image"] = "rocm/dev-ubuntu-24.04:6.2.4-complete" + expected_job["before_script"] = get_amdgpu_before_script("1.11") + expected_job["variables"]["TEST_AMDGPU"] = "1" + expected_job["tags"] = ["rocm", "x86_64"] + + job_yaml = CI._get_amdgpu_unit_test( + "1.11", test_package, "main", CI.ToolsGitRepo(git_url, git_branch) + ) + + @test keys(expected_job) == keys(job_yaml) + + for k in keys(expected_job) + @test ( + @assert job_yaml[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * yaml_diff(job_yaml[k], expected_job[k]) + ); + true + ) + end + + @test (@assert job_yaml == expected_job yaml_diff(job_yaml, expected_job); + true) + + @testset "test public interface" begin + julia_versions = Vector{String}(["1.9", "1.10", "1.11"]) + + job_dict = Dict() + CI.add_unit_test_job_yaml!( + job_dict, + test_package, + julia_versions, + "main", + CI.AMDGPU, + CI.ToolsGitRepo(git_url, git_branch), + ) + + for julia_version in julia_versions + job_name = "unit_test_julia_amdgpu_$(replace(julia_version, "." => "_"))" + expected_job["before_script"] = get_amdgpu_before_script(julia_version) + + @test haskey(job_dict, job_name) + @test job_dict["stages"] == ["unit-test"] + + unit_test_job = job_dict[job_name] + + @test keys(expected_job) == keys(unit_test_job) + + for k in keys(expected_job) + @test ( + @assert unit_test_job[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * yaml_diff(unit_test_job[k], expected_job[k]) + ); + true + ) + end + + @test ( + @assert unit_test_job == expected_job yaml_diff( + unit_test_job, expected_job + ); + true + ) + end + end +end diff --git a/.ci/CI/test/UnitTest/cuda_unit_test.jl b/.ci/CI/test/UnitTest/cuda_unit_test.jl new file mode 100644 index 0000000..3e7e0d4 --- /dev/null +++ b/.ci/CI/test/UnitTest/cuda_unit_test.jl @@ -0,0 +1,72 @@ +@testset "test _get_normal_unit_test CUDA platform" begin + julia_version = "1.10" + test_package = CI.TestPackage("QEDfoo", "/path/to/project", "42.0") + git_url = "http://github.com/name/repo" + git_branch = "branch" + + expected_job = get_generic_unit_job(julia_version, test_package) + + expected_job["script"] = get_main_unit_job_script_section(git_url, git_branch) + + expected_job["variables"]["TEST_CUDA"] = "1" + expected_job["tags"] = ["cuda", "x86_64"] + + job_yaml = CI._get_normal_unit_test( + julia_version, test_package, "main", CI.CUDA, CI.ToolsGitRepo(git_url, git_branch) + ) + + @test keys(expected_job) == keys(job_yaml) + + for k in keys(expected_job) + @test ( + @assert job_yaml[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * yaml_diff(job_yaml[k], expected_job[k]) + ); + true + ) + end + + @test (@assert job_yaml == expected_job yaml_diff(job_yaml, expected_job); + true) + + @testset "test public interface" begin + julia_versions = Vector{String}(["1.9", "1.10", "1.11"]) + + job_dict = Dict() + CI.add_unit_test_job_yaml!( + job_dict, + test_package, + julia_versions, + "main", + CI.CUDA, + CI.ToolsGitRepo(git_url, git_branch), + ) + + for julia_version in julia_versions + job_name = "unit_test_julia_cuda_$(replace(julia_version, "." => "_"))" + expected_job["image"] = "julia:$(julia_version)" + @test haskey(job_dict, job_name) + @test job_dict["stages"] == ["unit-test"] + + unit_test_job = job_dict[job_name] + + @test keys(expected_job) == keys(unit_test_job) + + for k in keys(expected_job) + @test ( + @assert unit_test_job[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * yaml_diff(unit_test_job[k], expected_job[k]) + ); + true + ) + end + + @test ( + @assert unit_test_job == expected_job yaml_diff( + unit_test_job, expected_job + ); + true + ) + end + end +end diff --git a/.ci/CI/test/UnitTest/nightly_unit_test.jl b/.ci/CI/test/UnitTest/nightly_unit_test.jl new file mode 100644 index 0000000..0c0a7f1 --- /dev/null +++ b/.ci/CI/test/UnitTest/nightly_unit_test.jl @@ -0,0 +1,94 @@ +@testset "test _get_nightly_unit_test" begin + for nightly_image in ["debian:bookworm-slim", "custom_image:latest"] + julia_version = "a" + test_package = CI.TestPackage("QEDfoo", "/path/to/project", "42.0") + git_url = "http://github.com/name/repo" + git_branch = "branch" + + expected_job = get_generic_unit_job(julia_version, test_package) + + expected_job["script"] = get_dev_unit_job_script_section(git_url, git_branch) + + expected_job["variables"]["TEST_CPU"] = "1" + expected_job["variables"]["JULIA_DOWNLOAD"] = "/julia/download" + expected_job["variables"]["JULIA_EXTRACT"] = "/julia/extract" + expected_job["tags"] = ["cpuonly"] + expected_job["image"] = nightly_image + + expected_job["before_script"] = [ + "apt update && apt install -y wget", + "mkdir -p \$JULIA_DOWNLOAD", + "mkdir -p \$JULIA_EXTRACT", + "if [[ \$CI_RUNNER_EXECUTABLE_ARCH == \"linux/arm64\" ]]; then + wget https://julialangnightlies-s3.julialang.org/bin/linux/aarch64/julia-latest-linux-aarch64.tar.gz -O \$JULIA_DOWNLOAD/julia-nightly.tar.gz +elif [[ \$CI_RUNNER_EXECUTABLE_ARCH == \"linux/amd64\" ]]; then + wget https://julialangnightlies-s3.julialang.org/bin/linux/x86_64/julia-latest-linux-x86_64.tar.gz -O \$JULIA_DOWNLOAD/julia-nightly.tar.gz +else + echo \"unknown runner architecture -> \$CI_RUNNER_EXECUTABLE_ARCH\" + exit 1 +fi", + "tar -xf \$JULIA_DOWNLOAD/julia-nightly.tar.gz -C \$JULIA_EXTRACT", + "JULIA_EXTRACT_FOLDER=\${JULIA_EXTRACT}/\$(ls \$JULIA_EXTRACT | grep -m1 julia)", + "cp -r \$JULIA_EXTRACT_FOLDER/* /usr", + ] + expected_job["allow_failure"] = true + + job_yaml = CI._get_nightly_unit_test( + test_package, "dev", CI.CPU, CI.ToolsGitRepo(git_url, git_branch), nightly_image + ) + + @test keys(expected_job) == keys(job_yaml) + + for k in keys(expected_job) + @test ( + @assert job_yaml[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * yaml_diff(job_yaml[k], expected_job[k]) + ); + true + ) + end + + @test (@assert job_yaml == expected_job yaml_diff(job_yaml, expected_job); + true) + + @testset "test public interface" begin + julia_versions = Vector{String}(["nightly"]) + + job_dict = Dict() + CI.add_unit_test_job_yaml!( + job_dict, + test_package, + julia_versions, + "dev", + CI.CPU, + CI.ToolsGitRepo(git_url, git_branch), + ) + + job_name = "unit_test_julia_cpu_nightly" + # default image + expected_job["image"] = "debian:bookworm-slim" + @test haskey(job_dict, job_name) + @test job_dict["stages"] == ["unit-test"] + + unit_test_job = job_dict[job_name] + + @test keys(expected_job) == keys(unit_test_job) + + for k in keys(expected_job) + @test ( + @assert unit_test_job[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * yaml_diff(unit_test_job[k], expected_job[k]) + ); + true + ) + end + + @test ( + @assert unit_test_job == expected_job yaml_diff( + unit_test_job, expected_job + ); + true + ) + end + end +end diff --git a/.ci/CI/test/UnitTest/normal_unit_test.jl b/.ci/CI/test/UnitTest/normal_unit_test.jl new file mode 100644 index 0000000..abf038c --- /dev/null +++ b/.ci/CI/test/UnitTest/normal_unit_test.jl @@ -0,0 +1,155 @@ +@testset "test _get_normal_unit_test()" begin + @testset "test script section of unit test job targeting dev branch" begin + git_url = "http://github.com/name/repo" + git_branch = "branch" + + expected_script = get_dev_unit_job_script_section(git_url, git_branch) + + job_yaml = CI._get_normal_unit_test( + "0", + CI.TestPackage("", "", ""), + "dev", + CI.CPU, + CI.ToolsGitRepo(git_url, git_branch), + ) + + @test ( + @assert job_yaml["script"] == expected_script compare_lists( + job_yaml["script"], expected_script + ); + true + ) + end + + @testset "test script section of unit test job targeting main branch" begin + git_url = "http://github.com/name/repo" + git_branch = "branch" + + expected_script = get_main_unit_job_script_section(git_url, git_branch) + + job_yaml = CI._get_normal_unit_test( + "0", + CI.TestPackage("", "", ""), + "main", + CI.CPU, + CI.ToolsGitRepo(git_url, git_branch), + ) + + @test ( + @assert job_yaml["script"] == expected_script compare_lists( + job_yaml["script"], expected_script + ); + true + ) + end + + @testset "test whole unit test job targeting dev branch" begin + julia_version = "1.8" + test_package = CI.TestPackage("QEDfoo", "/path/to/project", "42.0") + git_url = "http://github.com/name/repo" + git_branch = "branch" + + expected_job = get_generic_unit_job(julia_version, test_package) + + expected_job["script"] = get_dev_unit_job_script_section(git_url, git_branch) + + expected_job["variables"]["TEST_CPU"] = "1" + expected_job["tags"] = ["cpuonly"] + + job_yaml = CI._get_normal_unit_test( + julia_version, test_package, "dev", CI.CPU, CI.ToolsGitRepo(git_url, git_branch) + ) + + @test keys(expected_job) == keys(job_yaml) + + for k in keys(expected_job) + @test ( + @assert job_yaml[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * yaml_diff(job_yaml[k], expected_job[k]) + ); + true + ) + end + + @test (@assert job_yaml == expected_job yaml_diff(job_yaml, expected_job); + true) + end + + @testset "test whole unit test job targeting main branch" begin + julia_version = "1.9" + test_package = CI.TestPackage("QEDfoo", "/path/to/project", "42.0") + git_url = "http://github.com/name/repo" + git_branch = "branch" + + expected_job = get_generic_unit_job(julia_version, test_package) + + expected_job["script"] = get_main_unit_job_script_section(git_url, git_branch) + + expected_job["variables"]["TEST_CPU"] = "1" + expected_job["tags"] = ["cpuonly"] + + job_yaml = CI._get_normal_unit_test( + julia_version, + test_package, + "main", + CI.CPU, + CI.ToolsGitRepo(git_url, git_branch), + ) + + @test keys(expected_job) == keys(job_yaml) + + for k in keys(expected_job) + @test ( + @assert job_yaml[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * yaml_diff(job_yaml[k], expected_job[k]) + ); + true + ) + end + + @test (@assert job_yaml == expected_job yaml_diff(job_yaml, expected_job); + true) + + @testset "test public interface" begin + julia_versions = Vector{String}(["1.9", "1.10", "1.11"]) + + job_dict = Dict() + CI.add_unit_test_job_yaml!( + job_dict, + test_package, + julia_versions, + "main", + CI.CPU, + CI.ToolsGitRepo(git_url, git_branch), + ) + + for julia_version in julia_versions + job_name = "unit_test_julia_cpu_$(replace(julia_version, "." => "_"))" + expected_job["image"] = "julia:$(julia_version)" + @test haskey(job_dict, job_name) + @test job_dict["stages"] == ["unit-test"] + + unit_test_job = job_dict[job_name] + + @test keys(expected_job) == keys(unit_test_job) + + for k in keys(expected_job) + @test ( + @assert unit_test_job[k] == expected_job[k] ( + "\nkey: \"$(k)\"\n:" * + yaml_diff(unit_test_job[k], expected_job[k]) + ); + true + ) + end + + @test ( + @assert unit_test_job == expected_job yaml_diff( + unit_test_job, expected_job + ); + true + ) + end + end + end +end diff --git a/.ci/CI/test/UnitTest/runtests.jl b/.ci/CI/test/UnitTest/runtests.jl new file mode 100644 index 0000000..a8a6139 --- /dev/null +++ b/.ci/CI/test/UnitTest/runtests.jl @@ -0,0 +1,6 @@ +include("./utils.jl") +include("./normal_unit_test.jl") +include("./nightly_unit_test.jl") +include("./cuda_unit_test.jl") +include("./amdgpu_unit_test.jl") +include("./verify_unit_test.jl") diff --git a/.ci/CI/test/UnitTest/utils.jl b/.ci/CI/test/UnitTest/utils.jl new file mode 100644 index 0000000..b051b4e --- /dev/null +++ b/.ci/CI/test/UnitTest/utils.jl @@ -0,0 +1,66 @@ +""" +Returns the script section of an unit job targeting the dev branch. +""" +function get_dev_unit_job_script_section( + git_repo_url::AbstractString, git_repo_branch::AbstractString +) + return [ + "apt update && apt install -y git", + "git clone --depth 1 -b $(git_repo_branch) $(git_repo_url) /tmp/integration_test_tools/", + "julia --project=. /tmp/integration_test_tools/.ci/CI/src/SetupDevEnv.jl \${CI_PROJECT_DIR}/Project.toml", + "julia --project=. -e 'import Pkg; Pkg.instantiate()'", + "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'", + ] +end + +""" +Returns the script section of an unit job targeting the main branch. +""" +function get_main_unit_job_script_section( + git_repo_url::AbstractString, git_repo_branch::AbstractString +) + return [ + "apt update && apt install -y git", + "git clone --depth 1 -b $(git_repo_branch) $(git_repo_url) /tmp/integration_test_tools/", + "julia --project=. /tmp/integration_test_tools/.ci/CI/src/SetupDevEnv.jl \${CI_PROJECT_DIR}/Project.toml NO_MESSAGE", + "julia --project=. -e 'import Pkg; Pkg.instantiate()'", + "julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'", + ] +end + +""" +Returns a job skeleton for a unit job. +""" +function get_generic_unit_job( + julia_version::AbstractString, test_package::CI.TestPackage +)::Dict + job_yaml = Dict() + job_yaml["stage"] = "unit-test" + job_yaml["image"] = "julia:$(julia_version)" + job_yaml["variables"] = Dict( + "CI_DEV_PKG_NAME" => test_package.name, + "CI_DEV_PKG_PATH" => test_package.path, + "CI_DEV_PKG_VERSION" => test_package.version, + ) + + for tp in instances(CI.TestPlatform) + job_yaml["variables"]["TEST_$(tp)"] = "0" + end + + job_yaml["interruptible"] = true + + return job_yaml +end + +""" +Returns the before_script section of an amdgpu unit job. +""" +function get_amdgpu_before_script(julia_version::String)::Vector{String} + return [ + "curl -fsSL https://install.julialang.org | sh -s -- -y -p /julia", + "export PATH=/julia/bin:\$PATH", + "echo \$PATH", + "juliaup add $(julia_version)", + "juliaup default $(julia_version)", + ] +end diff --git a/.ci/CI/test/UnitTest/verify_unit_test.jl b/.ci/CI/test/UnitTest/verify_unit_test.jl new file mode 100644 index 0000000..c2bc5ef --- /dev/null +++ b/.ci/CI/test/UnitTest/verify_unit_test.jl @@ -0,0 +1,37 @@ +@testset "test add_unit_test_verify_job_yaml!() target branch dev" begin + for git_repo in [ + CI.ToolsGitRepo("https://github.com/name/repo", "dev"), + CI.ToolsGitRepo("foo", "bar"), + ] + expected_job = Dict( + "image" => "julia:1.10", + "stage" => "verify-unit-test-deps", + "script" => [ + "apt update && apt install -y git", + "git clone --depth 1 -b $(git_repo.branch) $(git_repo.url) /tools", + "julia /tools/.ci/verify_env.jl", + ], + "interruptible" => true, + "tags" => ["cpuonly"], + ) + + job_dict = Dict() + CI.add_unit_test_verify_job_yaml!(job_dict, "dev", git_repo) + + @test job_dict["stages"] == ["verify-unit-test-deps"] + @test ( + @assert job_dict["verify-unit-test-deps"] == expected_job yaml_diff( + job_dict["verify-unit-test-deps"], expected_job + ); + true + ) + end +end + +@testset "test add_unit_test_verify_job_yaml!() target branch main" begin + job_dict = Dict() + CI.add_unit_test_verify_job_yaml!( + job_dict, "main", CI.ToolsGitRepo("https://github.com/name/repo", "dev") + ) + @test isempty(job_dict) +end diff --git a/.ci/CI/test/generate_job_yaml.jl b/.ci/CI/test/generate_job_yaml.jl index 8625db3..739ee39 100644 --- a/.ci/CI/test/generate_job_yaml.jl +++ b/.ci/CI/test/generate_job_yaml.jl @@ -1,22 +1,3 @@ -using YAML - -""" - yaml_diff(given, expected)::AbstractString - -Generates an error string that shows a given and an expected data structure in yaml -representation. - -# Returns -- Human readable error message for the comparison of two job yaml's. -""" -function yaml_diff(given, expected)::AbstractString - output = "\ngiven:\n" - output *= String(YAML.yaml(given)) - output *= "\nexpected:\n" - output *= String(YAML.yaml(expected)) - return output -end - @testset "generate_job_yaml()" begin package_infos = CI.get_package_info() diff --git a/.ci/CI/test/runtests.jl b/.ci/CI/test/runtests.jl index 200f292..13d90ac 100644 --- a/.ci/CI/test/runtests.jl +++ b/.ci/CI/test/runtests.jl @@ -1,6 +1,8 @@ using CI using Test +include("./utils.jl") include("./get_target_branch.jl") include("./generate_job_yaml.jl") include("./setup_dev_env.jl") +include("./UnitTest/runtests.jl") diff --git a/.ci/CI/test/utils.jl b/.ci/CI/test/utils.jl new file mode 100644 index 0000000..9659bb0 --- /dev/null +++ b/.ci/CI/test/utils.jl @@ -0,0 +1,45 @@ +using YAML + +""" + yaml_diff(given::Dict, expected::Dict)::String + +Generates an error string that shows a given and an expected data structure in yaml +representation. + +# Returns +- Human readable error message for the comparison of two job yaml's. +""" +function yaml_diff(given::Dict, expected::Dict)::String + output = "\n***given***\n" + output *= String(YAML.yaml(given)) + output *= "\n***expected***\n" + output *= String(YAML.yaml(expected)) + return output +end + +""" + compare_lists(expected_list::Vector, given_list::Vector)::String + +Compares two lists and displays different lines. + +# Returns +- Human readable error message for the comparison of two lists. +""" +function compare_lists(expected_list::Vector, given_list::Vector)::String + text = "\n" + if length(expected_list) != length(given_list) + text *= "length of expected_list and given_list is different: $(length(expected_list)) != $(length(given_list))\n" + end + + min_length = min(length(expected_list), length(given_list)) + + text *= "different lines: \n" + for i in 1:min_length + if expected_list[i] != given_list[i] + text *= "$(i): $(expected_list[i])\n" + text *= " $(given_list[i])\n\n" + end + end + + return text +end