diff --git a/.ci/SetupDevEnv/src/SetupDevEnv.jl b/.ci/SetupDevEnv/src/SetupDevEnv.jl index 8630871..c76254d 100644 --- a/.ci/SetupDevEnv/src/SetupDevEnv.jl +++ b/.ci/SetupDevEnv/src/SetupDevEnv.jl @@ -263,6 +263,8 @@ Side effects of the function are: # Args - `repository_base_path::AbstractString`: Base folder into which the QED projects are to be cloned. +- `compat_changes::Dict{String,String}`: All QED package versions are added so that they can be set + correctly in the Compat section of the other QED packages. Existing entries are not changed. - `custom_urls::Dict{String,String}`: By default, the URL pattern `https://github.com/QEDjl-project/.jl` is used for the clone and the dev branch is checked out. The dict allows the use of custom URLs and branches for each QED project. The @@ -276,18 +278,25 @@ possible. """ function build_qed_dependency_graph!( repository_base_path::AbstractString, + compat_changes::Dict{String,String}, custom_urls::Dict{String,String}=Dict{String,String}(), )::Dict @info "build QED dependency graph" + io = IOBuffer() + println(io, "input compat_changes: $(compat_changes)") + qed_dependency_graph = Dict() qed_dependency_graph["QuantumElectrodynamics"] = _build_qed_dependency_graph!( repository_base_path, + compat_changes, custom_urls, "QuantumElectrodynamics", ["QuantumElectrodynamics"], ) + println(io, "output compat_changes: $(compat_changes)") with_logger(debuglogger) do - @debug "QED graph:\n$(_render_qed_tree(qed_dependency_graph))" + println(io, "QED graph:\n$(_render_qed_tree(qed_dependency_graph))") + @debug String(take!(io)) end return qed_dependency_graph end @@ -303,6 +312,8 @@ end # Args - `repository_base_path::AbstractString`: Base folder into which the QED projects will be cloned. +- `compat_changes::Dict{String,String}`: All QED package versions are added so that they can be set + correctly in the Compat section of the other QED packages. Existing entries are not changed. - `custom_urls::Dict{String,String}`: By default, the URL pattern `https://github.com/QEDjl-project/.jl` is used for the clone and the dev branch is checked out. The dict allows the use of custom URLs and branches for each QED project. The @@ -318,6 +329,7 @@ possible. """ function _build_qed_dependency_graph!( repository_base_path::AbstractString, + compat_changes::Dict{String,String}, custom_urls::Dict{String,String}, package_name::String, origin::Vector{String}, @@ -339,6 +351,13 @@ function _build_qed_dependency_graph!( # read dependencies from Project.toml and clone next packages until no # QED dependencies are left project_toml = TOML.parsefile(joinpath(repository_path, "Project.toml")) + + # add package version to compat entires, that they can be set correctly in the compat section of + # other packages + if !(project_toml["name"] in keys(compat_changes)) + compat_changes[project_toml["name"]] = project_toml["version"] + end + if haskey(project_toml, "deps") for dep_pkg in keys(project_toml["deps"]) # check for circular dependency @@ -359,7 +378,11 @@ function _build_qed_dependency_graph!( # handle only dependency starting with QED if startswith(dep_pkg, "QED") qed_dependency_graph[dep_pkg] = _build_qed_dependency_graph!( - repository_base_path, custom_urls, dep_pkg, vcat(origin, [dep_pkg]) + repository_base_path, + compat_changes, + custom_urls, + dep_pkg, + vcat(origin, [dep_pkg]), ) end end @@ -593,14 +616,16 @@ function set_compat_helper( name::AbstractString, version::AbstractString, project_path::AbstractString ) project_toml_path = joinpath(project_path, "Project.toml") - @info "change compat of $project_toml_path: $(name) -> $(version)" f = open(project_toml_path, "r") project_toml = TOML.parse(f) close(f) if haskey(project_toml, "compat") && haskey(project_toml["compat"], name) - project_toml["compat"][name] = version + if project_toml["compat"][name] != version + @info "change compat of $project_toml_path: $(name) -> $(version)" + project_toml["compat"][name] = version + end end # for GitHub Actions to fix permission denied error @@ -622,7 +647,7 @@ if abspath(PROGRAM_FILE) == @__FILE__ qed_path = mktempdir(; cleanup=false) - pkg_tree = build_qed_dependency_graph!(qed_path, custom_urls) + pkg_tree = build_qed_dependency_graph!(qed_path, compat_changes, custom_urls) pkg_ordering = get_package_dependecy_list(pkg_tree) required_deps = get_filtered_dependencies( diff --git a/.ci/integTestGen/Project.toml b/.ci/integTestGen/Project.toml index e6ac616..8905b8f 100644 --- a/.ci/integTestGen/Project.toml +++ b/.ci/integTestGen/Project.toml @@ -7,8 +7,10 @@ version = "0.1.0" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" IntegrationTests = "6be7cfa2-1838-408e-bc49-3a824ac3a1fb" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" [compat] diff --git a/.ci/integTestGen/src/integTestGen.jl b/.ci/integTestGen/src/integTestGen.jl index b1d2f3b..f61aa0f 100644 --- a/.ci/integTestGen/src/integTestGen.jl +++ b/.ci/integTestGen/src/integTestGen.jl @@ -1,6 +1,7 @@ module integTestGen include("get_target_branch.jl") +include("utils.jl") using Pkg: Pkg using YAML: YAML @@ -22,49 +23,6 @@ mutable struct PackageInfo PackageInfo(url, env_var) = new(url, "", env_var) end -""" - create_working_env(project_path::AbstractString, package_infos::AbstractDict{String,PackageInfo}) - -Create a temporary folder, and set up a new Project.toml and activate it. Checking the dependencies of -a project only works, if it is a dependency of the integTestGen.jl. The package to be analyzed -is only a temporary dependency, it must not change the Project.toml of integTestGen.jl permanently. -Therefore, the script generates a temporary Julia environment and adds the package -to analyze as a dependency. - -# Args - `project_path::AbstractString`: Absolute path to the project folder of the package to be analyzed - `package_infos::AbstractDict{String,PackageInfo}`: List depending QED pojects of QuantumElectrodynamics.jl. Use the list to - add the current dev branch version of the packages to the environment or a custom repository with - custom branch. -""" -function create_working_env( - project_path::AbstractString, package_infos::AbstractDict{String,PackageInfo} -) - tmp_path = mktempdir() - Pkg.activate(tmp_path) - # same dependency like in the Project.toml of integTestGen.jl - Pkg.add("Pkg") - Pkg.add("YAML") - # add main project as dependency - Pkg.develop(; path=project_path) - - for package_info in values(package_infos) - if package_info.modified_url == "" - # add current dev branch version of the package - Pkg.add(; url=package_info.url) - continue - end - split_url = split(package_info.modified_url, "#") - if length(split_url) == 2 - # add custom branch version of a custom repository - Pkg.add(; url=split_url[1], rev=split_url[2]) - else - # add current dev branch version of a custom repository - Pkg.add(; url=split_url[1]) - end - end -end - """ extract_env_vars_from_git_message!(package_infos::AbstractDict{String, PackageInfo}, var_name = "CI_COMMIT_MESSAGE") @@ -287,11 +245,19 @@ if abspath(PROGRAM_FILE) == @__FILE__ modify_package_url!(package_infos) modified_pkg = modified_package_name(package_infos) - # the script is locate in ci/integTestGen/src - # so we need to go 3 steps upwards in hierarchy to get the QuantumElectrodynamics.jl Project.toml - create_working_env(abspath(joinpath((@__DIR__), "../../..")), package_infos) + # TODO(SimeonEhrig): refactor me, that the conversion is not required anymore + custom_urls = Dict{String,String}() + for (name, info) in package_infos + if info.modified_url != "" + custom_urls[name] = info.modified_url + end + end + qed_path = mktempdir(; cleanup=false) + compat_changes = Dict{String,String}() + + pkg_tree = build_qed_dependency_graph!(qed_path, compat_changes, custom_urls) depending_pkg = IntegrationTests.depending_projects( - modified_pkg, collect(keys(package_infos)) + modified_pkg, collect(keys(package_infos)), pkg_tree ) job_yaml = Dict() diff --git a/.ci/integTestGen/src/utils.jl b/.ci/integTestGen/src/utils.jl new file mode 100644 index 0000000..d7ce9a9 --- /dev/null +++ b/.ci/integTestGen/src/utils.jl @@ -0,0 +1,228 @@ + +# TODO(SimeonEhrig): copied from SetupDevEnv.jl -> unify code +using TOML +using Logging +using LibGit2 + +debug_logger_io = IOBuffer() +debuglogger = ConsoleLogger(debug_logger_io, Logging.Debug) + +""" + _git_clone(repo_url::AbstractString, directory::AbstractString) + +Clones git repository + +# Args +- `repo_url::AbstractString`: Url of the repository. Can either be a plan URL or use the Julia Pkg + notation #. e.g. https://https://github.com/user/repo.git#dev to clone the + dev branch of the repository `repo`. +- `directory::AbstractString`: Path where the cloned repository is stored. +""" +function _git_clone(repo_url::AbstractString, directory::AbstractString) + splitted_url = split(repo_url, "#") + if length(splitted_url) < 2 + _git_clone(repo_url, "dev", directory) + else + _git_clone(splitted_url[1], splitted_url[2], directory) + end +end + +""" + _git_clone(repo_url::AbstractString, directory::AbstractString) + +Clones git repository + +# Args +- `repo_url::AbstractString`: Url of the repository. +- `branch::AbstractString`: Git branch name +- `directory::AbstractString`: Path where the cloned repository is stored. +""" +function _git_clone( + repo_url::AbstractString, branch::AbstractString, directory::AbstractString +) + @info "clone repository: $(repo_url)#$(branch) -> $(directory)" + with_logger(debuglogger) do + try + @debug "git clone --depth 1 -b $(branch) $(repo_url) $directory" + run( + pipeline( + `git clone --depth 1 -b $(branch) $(repo_url) $directory`; + stdout=devnull, + stderr=devnull, + ), + ) + catch + @debug "LibGit2.clone($(repo_url), $(directory); branch=$(branch))" + LibGit2.clone(repo_url, directory; branch=branch) + end + end +end + +""" + build_qed_dependency_graph!( + repository_base_path::AbstractString, + custom_urls::Dict{String,String}=Dict{String,String}(), + )::Dict + +Creates the dependency graph of the QED package ecosystem just by parsing Projects.toml. The +function starts by cloning the QuantumElectrodynamics.jl GitHub repository. Depending on +QuantumElectrodynamics.jl `Project.toml`, clones all directly and indirectly dependent QED.jl +GitHub repositories and constructs the dependency graph. + +Side effects of the function are: + - the Git repositories remain in the path defined in the `repository_base_path` variable + - the environment is not initialized (creation of a Manifest.toml) or changed in any other way + +# Args + +- `repository_base_path::AbstractString`: Base folder into which the QED projects are to be cloned. +- `compat_changes::Dict{String,String}`: All QED package versions are added so that they can be set + correctly in the Compat section of the other QED packages. Existing entries are not changed. +- `custom_urls::Dict{String,String}`: By default, the URL pattern + `https://github.com/QEDjl-project/.jl` is used for the clone and the dev branch + is checked out. The dict allows the use of custom URLs and branches for each QED project. The + key is the package name and the value must have the following form: `#`. + The syntax is the same as for `Pkg.add()`. + +# Returns + +Dict with the dependency graph. A leaf node has an empty dict. Duplications of dependencies are +possible. +""" +function build_qed_dependency_graph!( + repository_base_path::AbstractString, + compat_changes::Dict{String,String}, + custom_urls::Dict{String,String}=Dict{String,String}(), +)::Dict + @info "build QED dependency graph" + io = IOBuffer() + println(io, "input compat_changes: $(compat_changes)") + + qed_dependency_graph = Dict() + qed_dependency_graph["QuantumElectrodynamics"] = _build_qed_dependency_graph!( + repository_base_path, + compat_changes, + custom_urls, + "QuantumElectrodynamics", + ["QuantumElectrodynamics"], + ) + println(io, "output compat_changes: $(compat_changes)") + with_logger(debuglogger) do + println(io, "QED graph:\n$(_render_qed_tree(qed_dependency_graph))") + @debug String(take!(io)) + end + return qed_dependency_graph +end + +""" + _build_qed_dependency_graph!( + repository_base_path::AbstractString, + custom_urls::Dict{String,String}, + package_name::String, + origin::Vector{String}, + )::Dict + +# Args + +- `repository_base_path::AbstractString`: Base folder into which the QED projects will be cloned. +- `compat_changes::Dict{String,String}`: All QED package versions are added so that they can be set + correctly in the Compat section of the other QED packages. Existing entries are not changed. +- `custom_urls::Dict{String,String}`: By default, the URL pattern + `https://github.com/QEDjl-project/.jl` is used for the clone and the dev branch + is checked out. The dict allows the use of custom URLs and branches for each QED project. The + key is the package name and the value must have the following form: `#`. + The syntax is the same as for `Pkg.add()`. +- `package_name::String`: Current package to clone +- `origin::Vector{String}`: List of already visited packages + +# Returns + +Dict with the dependency graph. A leaf node has an empty dict. Duplications of dependencies are +possible. +""" +function _build_qed_dependency_graph!( + repository_base_path::AbstractString, + compat_changes::Dict{String,String}, + custom_urls::Dict{String,String}, + package_name::String, + origin::Vector{String}, +)::Dict + qed_dependency_graph = Dict() + repository_path = joinpath(repository_base_path, package_name) + if !isdir(repository_path) + if haskey(custom_urls, package_name) + _git_clone(custom_urls[package_name], repository_path) + else + _git_clone( + "https://github.com/QEDjl-project/$(package_name).jl", + "dev", + repository_path, + ) + end + end + + # read dependencies from Project.toml and clone next packages until no + # QED dependencies are left + project_toml = TOML.parsefile(joinpath(repository_path, "Project.toml")) + + # add package version to compat entires, that they can be set correctly in the compat section of + # other packages + if !(project_toml["name"] in keys(compat_changes)) + compat_changes[project_toml["name"]] = project_toml["version"] + end + + if haskey(project_toml, "deps") + for dep_pkg in keys(project_toml["deps"]) + # check for circular dependency + # actual there should be no circular dependency in graph + # if there is a circular dependency in the graph, find a good way to appease the CI + # developer + if dep_pkg in origin + dep_chain = "" + for dep in origin + dep_chain *= dep * " -> " + end + throw( + ErrorException( + "detect circular dependency in graph: $(dep_chain)$(dep_pkg)" + ), + ) + end + # handle only dependency starting with QED + if startswith(dep_pkg, "QED") + qed_dependency_graph[dep_pkg] = _build_qed_dependency_graph!( + repository_base_path, + compat_changes, + custom_urls, + dep_pkg, + vcat(origin, [dep_pkg]), + ) + end + end + end + + return qed_dependency_graph +end + +""" + _render_qed_tree(graph)::String + +Renders a given graph in ASCII art for debugging purposes. + +# Returns + +Rendered graph +""" +function _render_qed_tree(graph::Dict)::String + io = IOBuffer() + _render_qed_tree(io, graph, 0, "") + return String(take!(io)) +end + +function _render_qed_tree(io::IO, graph::Dict, level::Integer, input_string::String) + for key in keys(graph) + println(io, repeat(".", level) * key) + _render_qed_tree(io, graph[key], level + 1, input_string) + end + return input_string +end