From 4492d1eefa39a122958da57680bfb61f8400cb0b Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Wed, 23 Jan 2019 23:46:51 +0800 Subject: [PATCH] Julia: add windows-cpu build (#13937) - Julia v0.7 - Julia v1.0 --- ci/docker/runtime_functions.sh | 16 +----- ci/jenkins/Jenkins_steps.groovy | 28 ++++++++++ ci/jenkins/Jenkinsfile_windows_cpu | 8 ++- ci/windows/test_jl07_cpu.ps1 | 56 +++++++++++++++++++ ci/windows/test_jl10_cpu.ps1 | 56 +++++++++++++++++++ julia/.gitignore | 1 + julia/Project.toml | 26 +++++++++ julia/REQUIRE | 2 +- julia/appveyor.yml | 56 ------------------- julia/deps/build.jl | 2 +- julia/src/util.jl | 6 +- .../apache_rat_license_check/rat-excludes | 1 + tools/license_header.py | 3 + 13 files changed, 184 insertions(+), 77 deletions(-) create mode 100644 ci/windows/test_jl07_cpu.ps1 create mode 100644 ci/windows/test_jl10_cpu.ps1 create mode 100644 julia/Project.toml delete mode 100644 julia/appveyor.yml diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4c70c1accd02..9f9dd0c37e27 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -922,29 +922,21 @@ unittest_ubuntu_cpu_julia() { export PATH="$1/bin:$PATH" export MXNET_HOME='/work/mxnet' export JULIA_DEPOT_PATH='/work/julia-depot' - export DEVDIR="$JULIA_DEPOT_PATH/dev" julia -e 'using InteractiveUtils; versioninfo()' - # install package - mkdir -p $DEVDIR - ln -sf ${MXNET_HOME}/julia ${DEVDIR}/MXNet - - # register MXNet.jl and dependencies - julia -e 'using Pkg; Pkg.develop("MXNet")' - # FIXME export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so' export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH # use the prebuilt binary from $MXNET_HOME/lib - julia -e 'using Pkg; Pkg.build("MXNet")' + julia --project=./julia -e 'using Pkg; Pkg.build("MXNet")' # run the script `julia/test/runtests.jl` - julia -e 'using Pkg; Pkg.test("MXNet")' + julia --project=./julia -e 'using Pkg; Pkg.test("MXNet")' # See https://github.com/dmlc/MXNet.jl/pull/303#issuecomment-341171774 - julia -e 'using MXNet; mx._sig_checker()' + julia --project=./julia -e 'using MXNet; mx._sig_checker()' } unittest_ubuntu_cpu_julia07() { @@ -1280,10 +1272,8 @@ deploy_jl_docs() { export PATH="/work/julia10/bin:$PATH" export MXNET_HOME='/work/mxnet' export JULIA_DEPOT_PATH='/work/julia-depot' - export DEVDIR="$JULIA_DEPOT_PATH/dev" julia -e 'using InteractiveUtils; versioninfo()' - mkdir -p $DEVDIR # FIXME export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so' diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 9db3588a346d..984672c8e4d1 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1210,6 +1210,34 @@ def test_windows_python3_cpu() { }] } +def test_windows_julia07_cpu() { + return ['Julia 0.7: CPU Win': { + node(NODE_WINDOWS_CPU) { + ws('workspace/ut-julia07-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + unstash 'windows_package_cpu' + powershell 'ci/windows/test_jl07_cpu.ps1' + } + } + } + }] +} + +def test_windows_julia10_cpu() { + return ['Julia 1.0: CPU Win': { + node(NODE_WINDOWS_CPU) { + ws('workspace/ut-julia10-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + unstash 'windows_package_cpu' + powershell 'ci/windows/test_jl10_cpu.ps1' + } + } + } + }] +} + def test_qemu_armv7_cpu() { return ['ARMv7 QEMU': { node(NODE_LINUX_CPU) { diff --git a/ci/jenkins/Jenkinsfile_windows_cpu b/ci/jenkins/Jenkinsfile_windows_cpu index a8746db73d34..5bc40d625930 100644 --- a/ci/jenkins/Jenkinsfile_windows_cpu +++ b/ci/jenkins/Jenkinsfile_windows_cpu @@ -35,12 +35,14 @@ utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ custom_steps.compile_windows_cpu() - ]) + ]) utils.parallel_stage('Tests', [ custom_steps.test_windows_python2_cpu(), - custom_steps.test_windows_python3_cpu() - ]) + custom_steps.test_windows_python3_cpu(), + custom_steps.test_windows_julia07_cpu(), + custom_steps.test_windows_julia10_cpu() + ]) } , failure_handler: { diff --git a/ci/windows/test_jl07_cpu.ps1 b/ci/windows/test_jl07_cpu.ps1 new file mode 100644 index 000000000000..6cd34ef209de --- /dev/null +++ b/ci/windows/test_jl07_cpu.ps1 @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +7z x -y windows_package.7z + +# set default output encoding to utf8 +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' + +$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package') +$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7.0-win64.exe" +$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot') + +$JULIA_DIR = [System.IO.Path]::GetFullPath('.\julia07') +$JULIA = "$JULIA_DIR\bin\julia" + +# Download most recent Julia Windows binary +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12 +(New-Object System.Net.WebClient).DownloadFile($env:JULIA_URL, "julia-binary.exe") +if (! $?) { Throw ("Error on downloading Julia Windows binary") } + +# Run installer silently, output to C:\julia07\julia +Start-Process -Wait "julia-binary.exe" -ArgumentList "/S /D=$JULIA_DIR" +if (! $?) { Throw ("Error on installing Julia") } + +& $JULIA -e "using InteractiveUtils; versioninfo()" + +dir + +$src=' + using Pkg + Pkg.activate(".\\julia") + Pkg.build() + Pkg.test() +' + +$src > .\ci-build.jl + +# Redirect all stderr output to stdout, +# since Julia loggers output stuffs to stderr. +# Then, stderr triggers powershell NativeCommandError. +& $JULIA .\ci-build.jl 2>&1 | %{ "$_" } +if ($LastExitCode -eq 1) { Throw ("Error") } diff --git a/ci/windows/test_jl10_cpu.ps1 b/ci/windows/test_jl10_cpu.ps1 new file mode 100644 index 000000000000..96c419066354 --- /dev/null +++ b/ci/windows/test_jl10_cpu.ps1 @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +7z x -y windows_package.7z + +# set default output encoding to utf8 +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' + +$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package') +$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/1.0/julia-1.0.3-win64.exe" +$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot') + +$JULIA_DIR = [System.IO.Path]::GetFullPath('.\julia10') +$JULIA = "$JULIA_DIR\bin\julia" + +# Download most recent Julia Windows binary +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12 +(New-Object System.Net.WebClient).DownloadFile($env:JULIA_URL, "julia-binary.exe") +if (! $?) { Throw ("Error on downloading Julia Windows binary") } + +# Run installer silently, output to C:\julia10\julia +Start-Process -Wait "julia-binary.exe" -ArgumentList "/S /D=$JULIA_DIR" +if (! $?) { Throw ("Error on installing Julia") } + +& $JULIA -e "using InteractiveUtils; versioninfo()" + +dir + +$src=' + using Pkg + Pkg.activate(".\\julia") + Pkg.build() + Pkg.test() +' + +$src > .\ci-build.jl + +# Redirect all stderr output to stdout, +# since Julia loggers output stuffs to stderr. +# Then, stderr triggers powershell NativeCommandError. +& $JULIA .\ci-build.jl 2>&1 | %{ "$_" } +if ($LastExitCode -eq 1) { Throw ("Error") } diff --git a/julia/.gitignore b/julia/.gitignore index 3687ed485c5a..e7b35fa85d96 100644 --- a/julia/.gitignore +++ b/julia/.gitignore @@ -8,3 +8,4 @@ deps/src deps/usr deps/deps.jl .vscode +/Manifest.toml diff --git a/julia/Project.toml b/julia/Project.toml new file mode 100644 index 000000000000..82a94c5a0767 --- /dev/null +++ b/julia/Project.toml @@ -0,0 +1,26 @@ +name = "MXNet" +uuid = "a7949054-b901-59c6-b8e3-7238c29bf7f0" +authors = ["Chiyuan Zhang ", "Valentin Churavy ", "Iblis Lin "] +version = "1.5.0" + +[deps] +BinDeps = "9e28174c-4ba2-5203-b857-d8d62c4213ee" +Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Reexport = "189a3867-3050-52da-a836-e630ba90ab69" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[compat] +julia = "≥0.7" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/julia/REQUIRE b/julia/REQUIRE index b53f0c3cc0ec..8008da3d2aaa 100644 --- a/julia/REQUIRE +++ b/julia/REQUIRE @@ -1,4 +1,4 @@ -julia 0.6 +julia 0.7 Formatting BinDeps JSON diff --git a/julia/appveyor.yml b/julia/appveyor.yml deleted file mode 100644 index 50e275cfa8a8..000000000000 --- a/julia/appveyor.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -environment: - matrix: - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" - -branches: - only: - - master - - stable - -notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - -install: - - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12" -# If there's a newer build queued for the same PR, cancel this one - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } - -# Download most recent Julia Windows binary - - ps: (new-object net.webclient).DownloadFile( - $env:JULIA_URL, - "C:\projects\julia-binary.exe") -# Run installer silently, output to C:\projects\julia - - C:\projects\julia-binary.exe /S /D=C:\projects\julia - -build_script: -# Need to convert from shallow to complete for Pkg.clone to work - - IF EXIST .git\shallow (git fetch --unshallow) - - C:\projects\julia\bin\julia -e "versioninfo(); - Pkg.clone(pwd(), \"MXNet\"); Pkg.build(\"MXNet\")" - -test_script: - - C:\projects\julia\bin\julia --check-bounds=yes -e "Pkg.test(\"MXNet\")" - diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 7a37803f306a..9a719be850f5 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -86,7 +86,7 @@ if HAS_CUDA if HAS_CUDNN @info("Found a CuDNN installation.") end - @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", nothing))") + @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", "nothing"))") else @info("Did not find a CUDA installation, using CPU-only version of MXNet.") end diff --git a/julia/src/util.jl b/julia/src/util.jl index ae2e50b92c55..ac7f4fc71653 100644 --- a/julia/src/util.jl +++ b/julia/src/util.jl @@ -35,12 +35,12 @@ function get_mnist_ubyte() filenames = Dict((x[1] => joinpath(mnist_dir, x[2]) for x ∈ pairs(filenames))) if !all(isfile, values(filenames)) cd(mnist_dir) do - mnist_dir = download("http://data.mxnet.io/mxnet/data/mnist.zip", "mnist.zip") + data = download("http://data.mxnet.io/mxnet/data/mnist.zip", "mnist.zip") try - run(`unzip -u $mnist_dir`) + run(`unzip -u $data`) catch try - run(pipe(`7z x $mnist_dir`,stdout = devnull)) + run(pipeline(`7z x $data`,stdout = devnull)) catch error("Extraction Failed:No extraction program found in path") end diff --git a/tests/nightly/apache_rat_license_check/rat-excludes b/tests/nightly/apache_rat_license_check/rat-excludes index 782ef40b7e35..93ac16e42b7d 100755 --- a/tests/nightly/apache_rat_license_check/rat-excludes +++ b/tests/nightly/apache_rat_license_check/rat-excludes @@ -48,6 +48,7 @@ moderngpu/* deformable_im2col.cuh deformable_im2col.h REQUIRE +Project.toml include/* .*.iml .*.json.ref diff --git a/tools/license_header.py b/tools/license_header.py index 11cc92839993..b9acbf167c17 100755 --- a/tools/license_header.py +++ b/tools/license_header.py @@ -91,6 +91,9 @@ # Licensed under 2-Clause BSD in header 'example/ssd/dataset/pycocotools/coco.py', + + # Julia package metadata, generated by Pkg3.jl + 'julia/Project.toml', ] # language extensions and the according commment mark