Skip to content

Commit

Permalink
modularize source files, rename source files and module names
Browse files Browse the repository at this point in the history
  • Loading branch information
@SimeonEhrig
SimeonEhrig committed Nov 6, 2024
1 parent bb3860d commit 6423e6b
Show file tree
Hide file tree
Showing 9 changed files with 311 additions and 292 deletions.
281 changes: 26 additions & 255 deletions .ci/CI/src/Bootloader.jl
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
module Bootloader

include("./get_target_branch.jl")
include("./integTestGen.jl")
include("./modules/GitLabTargetBranch.jl")
include("./modules/UnitTest.jl")
include("./modules/IntegTest.jl")

using .TargetBranch
using .integTestGen
using .GitLabTargetBranch
using .UnitTest
using .IntegGen
using IntegrationTests
using TOML
using Logging
Expand Down Expand Up @@ -35,6 +37,12 @@ function get_unit_test_julia_versions()::Vector{String}
return ["1.10", "1.11", "rc", "nightly"]
end

function get_unit_test_nightly_baseimage()::String
# TODO: support reading from env variables

return "debian:bookworm-slim"
end

function get_git_ci_tools_url_branch()::Tuple{String,String}
url = "https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git"
branch = "dev"
Expand All @@ -52,251 +60,6 @@ function get_git_ci_tools_url_branch()::Tuple{String,String}
return (url, branch)
end

function add_unit_test_job_yaml!(
job_dict::Dict,
julia_versions::Vector{String},
target_branch::String,
dev_package_name::AbstractString,
dev_package_version::AbstractString,
dev_package_path::AbstractString,
git_ci_tools_url::String="https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git",
git_ci_tools_branch::String="dev",
)
if !haskey(job_dict, "stages")
job_dict["stages"] = []
end

push!(job_dict["stages"], "unit-test")

for version in julia_versions
if version != "nightly"
job_dict["unit_test_julia_$(replace(version, "." => "_"))"] = get_normal_unit_test(
version,
target_branch,
dev_package_name,
dev_package_version,
dev_package_path,
git_ci_tools_url,
git_ci_tools_branch,
)
else
job_dict["unit_test_julia_nightly"] = get_nightly_unit_test(
target_branch,
dev_package_name,
dev_package_version,
dev_package_path,
git_ci_tools_url,
git_ci_tools_branch,
)
end
end
end

function add_unit_test_verify_job_yaml!(
job_dict::Dict,
target_branch::String,
git_ci_tools_url::String="https://github.com/QEDjl-project/QuantumElectrodynamics.jl.git",
git_ci_tools_branch::String="dev",
)
# verification script that no custom URLs are used in unit tests
if target_branch != "main"
push!(job_dict["stages"], "verify-unit-test-deps")
job_dict["verify-unit-test-deps"] = Dict(
"image" => "julia:1.10",
"stage" => "verify-unit-test-deps",
"script" => [
"apt update && apt install -y git",
"git clone --depth 1 -b $(git_ci_tools_branch) $(git_ci_tools_url) /tools",
"julia /tools/.ci/verify_env.jl",
],
"interruptible" => true,
"tags" => ["cpuonly"],
)
end
end

function get_normal_unit_test(
version::String,
target_branch::String,
dev_package_name::AbstractString,
dev_package_version::AbstractString,
dev_package_path::AbstractString,
git_ci_tools_url::String,
git_ci_tools_branch::String,
)::Dict
job_yaml = Dict()
job_yaml["stage"] = "unit-test"
job_yaml["variables"] = Dict(
"CI_DEV_PKG_NAME" => dev_package_name,
"CI_DEV_PKG_VERSION" => dev_package_version,
"CI_DEV_PKG_PATH" => dev_package_path,
)
job_yaml["image"] = "julia:$(version)"

script = [
"apt update && apt install -y git",
"git clone --depth 1 -b $(git_ci_tools_branch) $(git_ci_tools_url) /tmp/integration_test_tools/",
]

if target_branch == "main"
push!(
script,
"julia --project=. /tmp/integration_test_tools/.ci/CI/src/SetupDevEnv.jl \${CI_PROJECT_DIR}/Project.toml NO_MESSAGE",
)
else
push!(
script,
"julia --project=. /tmp/integration_test_tools/.ci/CI/src/SetupDevEnv.jl \${CI_PROJECT_DIR}/Project.toml",
)
end

script = vcat(
script,
[
"julia --project=. -e 'import Pkg; Pkg.instantiate()'",
"julia --project=. -e 'import Pkg; Pkg.test(; coverage = true)'",
],
)
job_yaml["script"] = script

job_yaml["interruptible"] = true
job_yaml["tags"] = ["cpuonly"]

return job_yaml
end

function get_nightly_unit_test(
target_branch::String,
dev_package_name::AbstractString,
dev_package_version::AbstractString,
dev_package_path::AbstractString,
git_ci_tools_url::String,
git_ci_tools_branch::String,
)
job_yaml = get_normal_unit_test(
"1",
target_branch,
dev_package_name,
dev_package_version,
dev_package_path,
git_ci_tools_url,
git_ci_tools_branch,
)
job_yaml["image"] = "debian:bookworm-slim"

if !haskey(job_yaml, "variables")
job_yaml["variables"] = Dict()
end
job_yaml["variables"]["JULIA_DOWNLOAD"] = "/julia/download"
job_yaml["variables"]["JULIA_EXTRACT"] = "/julia/extract"

job_yaml["before_script"] = [
"apt update && apt install -y wget",
"mkdir -p \$JULIA_DOWNLOAD",
"mkdir -p \$JULIA_EXTRACT",
"if [[ \$CI_RUNNER_EXECUTABLE_ARCH == \"linux/arm64\" ]]; then
wget https://julialangnightlies-s3.julialang.org/bin/linux/aarch64/julia-latest-linux-aarch64.tar.gz -O \$JULIA_DOWNLOAD/julia-nightly.tar.gz
elif [[ \$CI_RUNNER_EXECUTABLE_ARCH == \"linux/amd64\" ]]; then
wget https://julialangnightlies-s3.julialang.org/bin/linux/x86_64/julia-latest-linux-x86_64.tar.gz -O \$JULIA_DOWNLOAD/julia-nightly.tar.gz
else
echo \"unknown runner architecture -> \$CI_RUNNER_EXECUTABLE_ARCH\"
exit 1
fi",
"tar -xf \$JULIA_DOWNLOAD/julia-nightly.tar.gz -C \$JULIA_EXTRACT",
# we need to search for the julia base folder name, because the second part of the name is the git commit hash
# e.g. julia-b0c6781676f
"JULIA_EXTRACT_FOLDER=\${JULIA_EXTRACT}/\$(ls \$JULIA_EXTRACT | grep -m1 julia)",
# copy everything to /usr to make julia public available
# mv is not possible, because it cannot merge folder
"cp -r \$JULIA_EXTRACT_FOLDER/* /usr",
]

job_yaml["image"] = "debian:bookworm-slim"

job_yaml["allow_failure"] = true
return job_yaml
end

function add_integration_test_job_yaml!(
job_dict::Dict,
package_name::AbstractString,
package_version::AbstractString,
package_path::AbstractString,
git_ci_tools_url::String,
git_ci_tools_branch::String,
)
if !haskey(job_dict, "stages")
job_dict["stages"] = []
end

package_infos = integTestGen.get_package_info()
if target_branch != "main"
integTestGen.extract_env_vars_from_git_message!(package_infos)
end
integTestGen.modify_package_url!(package_infos)

custom_urls = Dict{String,String}()
for (name, info) in package_infos
if info.modified_url != ""
custom_urls[name] = info.modified_url
end
end
qed_path = mktempdir(; cleanup=false)
compat_changes = Dict{String,String}()

pkg_tree = integTestGen.build_qed_dependency_graph!(
qed_path, compat_changes, custom_urls
)
depending_pkg = IntegrationTests.depending_projects(
package_name, collect(keys(package_infos)), pkg_tree
)

if !isempty(depending_pkg)
push!(job_dict["stages"], "integ-test")
for p in depending_pkg
if target_branch == "main" && TargetBranch.is_pull_request()
integTestGen.generate_job_yaml!(
p,
"dev",
package_name,
package_version,
package_path,
job_dict,
package_infos,
git_ci_tools_url,
git_ci_tools_branch,
)
integTestGen.generate_job_yaml!(
p,
"main",
package_name,
package_version,
package_path,
job_dict,
package_infos,
git_ci_tools_url,
git_ci_tools_branch,
"integ-test",
true,
)
else
integTestGen.generate_job_yaml!(
p,
target_branch,
package_name,
package_version,
package_path,
job_dict,
package_infos,
git_ci_tools_url,
git_ci_tools_branch,
"integ-test",
)
end
end
end
end

function print_job_yaml(job_yaml::Dict, io::IO=stdout)
job_yaml_copy = deepcopy(job_yaml)

Expand Down Expand Up @@ -331,12 +94,13 @@ function print_job_yaml(job_yaml::Dict, io::IO=stdout)
end
end

if abspath(PROGRAM_FILE) == @__FILE__
# use main function to avoid to define global variables
function main()
_check_env_vars()

package_path = ENV["CI_PROJECT_DIR"]
(package_name, package_version) = get_package_name_version(package_path)
target_branch = TargetBranch.get_target()
target_branch = GitLabTargetBranch.get_target()

@info "Test package name: $(package_name)"
@info "Test package version: $(package_version)"
Expand All @@ -350,7 +114,7 @@ if abspath(PROGRAM_FILE) == @__FILE__

(git_ci_tools_url, git_ci_tools_branch) = get_git_ci_tools_url_branch()

add_unit_test_job_yaml!(
UnitTest.add_unit_test_job_yaml!(
job_yaml,
unit_test_julia_versions,
target_branch,
Expand All @@ -359,22 +123,29 @@ if abspath(PROGRAM_FILE) == @__FILE__
package_path,
git_ci_tools_url,
git_ci_tools_branch,
get_unit_test_nightly_baseimage(),
)

add_integration_test_job_yaml!(
IntegGen.add_integration_test_job_yaml!(
job_yaml,
target_branch,
package_name,
package_version,
package_path,
git_ci_tools_url,
git_ci_tools_branch,
)

add_unit_test_verify_job_yaml!(
UnitTest.add_unit_test_verify_job_yaml!(
job_yaml, target_branch, git_ci_tools_url, git_ci_tools_branch
)

print_job_yaml(job_yaml)
return print_job_yaml(job_yaml)
end

# TODO: if Julia 1.11 is minimum, replace it it with: function (@main)(args)
if abspath(PROGRAM_FILE) == @__FILE__
main()
end

end
10 changes: 5 additions & 5 deletions .ci/CI/src/CI.jl
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
module CI
include("./integTestGen.jl")
using .integTestGen
include("./modules/IntegTest.jl")
using .IntegGen

include("./modules/GitLabTargetBranch.jl")
using .GitLabTargetBranch

include("./SetupDevEnv.jl")
using .SetupDevEnv

include("./get_target_branch.jl")
using .TargetBranch

end
11 changes: 1 addition & 10 deletions .ci/CI/src/get_project_name_version_path.jl → .ci/CI/src/GetProjectNameVersionPath.jl
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,6 @@
using Pkg

"""
get_project_version_name()::Tuple{String,String}
# Return
Returns project name and version number
"""
function get_project_version_name_path()::Tuple{String,String,String}
return (Pkg.project().name, string(Pkg.project().version), dirname(Pkg.project().path))
end
include("./modules/Utils.jl")

# the script be directly executed in bash to set the environment variables
# $(julia --project=/path/to/the/actual/project get_project_version_name)
Expand Down
6 changes: 1 addition & 5 deletions .ci/CI/src/get_target_branch.jl → .ci/CI/src/modules/GitLabTargetBranch.jl
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
module TargetBranch
module GitLabTargetBranch

using HTTP
using JSON
Expand Down Expand Up @@ -125,8 +125,4 @@ function get_target()::AbstractString
end
end

if abspath(PROGRAM_FILE) == @__FILE__
print(get_target())
end

end
Loading

0 comments on commit 6423e6b

Please sign in to comment.