Skip to content

Commit

Permalink
Upgrade source-build to BB2 for Julia 1.3 (#450)
Browse files Browse the repository at this point in the history 
1. Remove obsoleted patches
2. Add new patches(LLVMBuilder at v6.0.1-7+nowasm)
3. Source-build enhancement for Julia 1.3
    
LLVM/Clang binaries from BB2 can be reused, but we do need to download and patch LLVM/Clang's source code for building libcxxffi.
  • Loading branch information
@Gnimuc
Gnimuc authored Dec 12, 2019
1 parent d488dba commit c5bdd65
Show file tree
Hide file tree
Showing 55 changed files with 9,792 additions and 3,286 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,7 @@
.DS_Store
docs/build
docs/site
deps/usr
deps/deps.jl
deps/build.log
Manifest.toml
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Note that Windows support is still in its early stage, so feel free to submit fe
### Build from source
#### Build requirements

In addition to the [system requirements](https://github.com/JuliaLang/julia#required-build-tools-and-external-libraries) to build julia itself(note you need to build julia with `USE_BINARYBUILDER=0`), the following are required:
In addition to the [system requirements](https://github.com/JuliaLang/julia#required-build-tools-and-external-libraries) to build julia itself, the following are required:

- *Debian/Ubuntu*: `libedit-dev`, `libncurses5-dev`
- *RedHat/CentOS*: `libedit-devel`
Expand Down
252 changes: 95 additions & 157 deletions deps/BuildBootstrap.Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,145 +1,99 @@
JULIA_SRC := $(subst \,/,$(BASE_JULIA_SRC))
JULIA_BIN := $(subst \,/,$(BASE_JULIA_BIN))
# Cxx source build
# download sources
LLVM_VER := 6.0.1
LLVM_URL_PREFIX := http://releases.llvm.org/$(LLVM_VER)
LLVM_TAR := llvm-$(LLVM_VER).src.tar.xz
CLANG_TAR := cfe-$(LLVM_VER).src.tar.xz
COMPILER_RT_TAR := compiler-rt-$(LLVM_VER).src.tar.xz
LIBCXX_TAR := libcxx-$(LLVM_VER).src.tar.xz
LIBCXXABI_TAR := libcxxabi-$(LLVM_VER).src.tar.xz
POLLY_TAR := polly-$(LLVM_VER).src.tar.xz
LIBUNWIND_TAR := libunwind-$(LLVM_VER).src.tar.xz
LLD_TAR := lld-$(LLVM_VER).src.tar.xz

TIMEOUT := 180
CURL := curl -fkL --connect-timeout $(TIMEOUT) -y $(TIMEOUT)

usr/download:
@[ -d usr ] || mkdir usr
mkdir -p $@

ifeq ($(LLVM_VER),)
BUILDROOT=$(JULIA_BIN)/../..
include $(JULIA_SRC)/deps/Versions.make
ifeq (exists, $(shell [ -e $(BUILDROOT)/Make.user ] && echo exists ))
include $(BUILDROOT)/Make.user
endif
endif
include Make.inc
LLVM_TARS := $(LLVM_TAR) $(CLANG_TAR) $(COMPILER_RT_TAR) $(LIBCXX_TAR) $(LIBCXXABI_TAR) $(POLLY_TAR) $(LIBUNWIND_TAR) $(LLD_TAR)

LLVM_VER_MAJ:=$(word 1, $(subst ., ,$(LLVM_VER)))
LLVM_VER_MIN:=$(word 2, $(subst ., ,$(LLVM_VER)))
# define a "short" LLVM version for easy comparisons
ifeq ($(LLVM_VER),svn)
LLVM_VER_SHORT:=svn
else
LLVM_VER_SHORT:=$(LLVM_VER_MAJ).$(LLVM_VER_MIN)
endif
LLVM_VER_PATCH:=$(word 3, $(subst ., ,$(LLVM_VER)))
ifeq ($(LLVM_VER_PATCH),)
LLVM_VER_PATCH := 0
endif
llvm_tars: usr/download
@for tar in $(LLVM_TARS); do \
if [ -e $</$$tar ]; then \
echo "$$tar already exists, please delete it manually if you'd like to re-download it."; \
else \
$(CURL) -o $</$$tar $(LLVM_URL_PREFIX)/$$tar ; \
fi \
done

ifeq ($(LLVM_VER_SHORT),$(filter $(LLVM_VER_SHORT),3.3 3.4 3.5 3.6 3.7 3.8))
LLVM_USE_CMAKE := 0
else
LLVM_USE_CMAKE := 1
endif
# unzip
usr/src: usr/download
mkdir -p $@

all: usr/lib/libcxxffi.$(SHLIB_EXT) usr/lib/libcxxffi-debug.$(SHLIB_EXT) build/clang_constants.jl
llvm-$(LLVM_VER): usr/src llvm_tars
@[ -d $</$@ ] || mkdir -p $</$@
tar -C $</$@ --strip-components=1 -xf usr/download/$(LLVM_TAR)
@[ -d $</$@/tools/clang ] || mkdir -p $</$@/tools/clang
tar -C $</$@/tools/clang --strip-components=1 -xf usr/download/$(CLANG_TAR)
@[ -d $</$@/tools/polly ] || mkdir -p $</$@/tools/polly
tar -C $</$@/tools/polly --strip-components=1 -xf usr/download/$(POLLY_TAR)
@[ -d $</$@/tools/lld ] || mkdir -p $</$@/tools/lld
tar -C $</$@/tools/lld --strip-components=1 -xf usr/download/$(LLD_TAR)
@[ -d $</$@/projects/compiler-rt ] || mkdir -p $</$@/projects/compiler-rt
tar -C $</$@/projects/compiler-rt --strip-components=1 -xf usr/download/$(COMPILER_RT_TAR)
@[ -d $</$@/projects/libcxx ] || mkdir -p $</$@/projects/libcxx
tar -C $</$@/projects/libcxx --strip-components=1 -xf usr/download/$(LIBCXX_TAR)
@[ -d $</$@/projects/libcxxabi ] || mkdir -p $</$@/projects/libcxxabi
tar -C $</$@/projects/libcxxabi --strip-components=1 -xf usr/download/$(LIBCXXABI_TAR)
@[ -d $</$@/projects/libunwind ] || mkdir -p $</$@/projects/libunwind
tar -C $</$@/projects/libunwind --strip-components=1 -xf usr/download/$(LIBUNWIND_TAR)

# apply patches
llvm-patched: llvm-$(LLVM_VER)
cd usr/src/$< && \
for p in ../../../llvm_patches/*.patch; do \
echo "Applying patch $$p"; \
patch -p1 < $$p ; \
done

# build libcxxffi
include Make.inc

JULIA_SRC := $(subst \,/,$(BASE_JULIA_SRC))
JULIA_BIN := $(subst \,/,$(BASE_JULIA_BIN))

ifeq ($(OLD_CXX_ABI),1)
CXX_ABI_SETTING=-D_GLIBCXX_USE_CXX11_ABI=0
else
CXX_ABI_SETTING=-D_GLIBCXX_USE_CXX11_ABI=1
endif

CXXJL_CPPFLAGS = -I$(JULIA_SRC)/src/support -I$(BASE_JULIA_BIN)/../include
JULIA_SOURCE_INCLUDE_DIRS := $(JULIA_SRC)/src/support
JULIA_INCLUDE_DIRS := $(JULIA_BIN)/../include
CLANG_SOURCE_INCLUDE_DIRS := usr/src/llvm-$(LLVM_VER)/tools/clang/include
CLANG_SOURCE_INCLUDE_DIRS += usr/src/llvm-$(LLVM_VER)/tools/clang/lib
CLANG_INCLUDE_DIRS := $(JULIA_BIN)/../include $(JULIA_BIN)/../include/clang
INCLUDE_DIRS := $(JULIA_SOURCE_INCLUDE_DIRS) $(JULIA_INCLUDE_DIRS) $(CLANG_SOURCE_INCLUDE_DIRS) $(CLANG_INCLUDE_DIRS)
CXXJL_CPPFLAGS = $(addprefix -I, $(INCLUDE_DIRS))

ifeq ($(JULIA_BINARY_BUILD),1)
LIBDIR := $(BASE_JULIA_BIN)/../lib/julia
else
LIBDIR := $(BASE_JULIA_BIN)/../lib
endif

CLANG_LIBS = clangFrontendTool clangBasic clangLex clangDriver clangFrontend clangParse \
clangAST clangASTMatchers clangSema clangAnalysis clangEdit \
clangRewriteFrontend clangRewrite clangSerialization clangStaticAnalyzerCheckers \
clangStaticAnalyzerCore clangStaticAnalyzerFrontend clangTooling clangToolingCore \
clangCodeGen clangARCMigrate clangFormat

ifneq ($(PREBUILT_CI_BINARIES),1)
# If clang is not built by base julia, build it ourselves
ifeq ($(BUILD_LLVM_CLANG),)
ifeq ($(LLVM_VER),svn)
$(error For julia built against llvm-svn, please built clang in tree)
endif

LLVM_TAR_EXT:=$(LLVM_VER).src.tar.xz
LLVM_CLANG_TAR:=src/cfe-$(LLVM_TAR_EXT)
LLVM_SRC_TAR:=src/llvm-$(LLVM_TAR_EXT)
LLVM_COMPILER_RT_TAR:=src/compiler-rt-$(LLVM_TAR_EXT)
LLVM_SRC_URL := http://releases.llvm.org/$(LLVM_VER)

src:
mkdir $@

# Also build a new copy of LLVM, so we get headers, tools, etc.
ifeq ($(JULIA_BINARY_BUILD),1)
LLVM_SRC_DIR := src/llvm-$(LLVM_VER)
include llvm-patches/apply-llvm-patches.mk
$(LLVM_SRC_TAR): | src
curl -Lo $@ $(LLVM_SRC_URL)/$(notdir $@)
src/llvm-$(LLVM_VER): $(LLVM_SRC_TAR)
mkdir -p $@
tar -C $@ --strip-components=1 -xf $<
build/llvm-$(LLVM_VER)/Makefile: src/llvm-$(LLVM_VER) $(LLVM_PATCH_LIST)
mkdir -p $(dir $@)
cd $(dir $@) && \
cmake -G "Unix Makefiles" -DLLVM_TARGETS_TO_BUILD="X86" \
-DLLVM_BUILD_LLVM_DYLIB=ON -DCMAKE_BUILD_TYPE=Release \
-DLLVM_LINK_LLVM_DYLIB=ON -DLLVM_ENABLE_THREADS=OFF \
-DCMAKE_CXX_COMPILER_ARG1="$(CXX_ABI_SETTING)" \
../../src/llvm-$(LLVM_VER)
build/llvm-$(LLVM_VER)/bin/llvm-config: build/llvm-$(LLVM_VER)/Makefile
cd build/llvm-$(LLVM_VER) && $(MAKE)
LLVM_HEADER_DIRS = src/llvm-$(LLVM_VER)/include build/llvm-$(LLVM_VER)/include
CLANG_CMAKE_DEP = build/llvm-$(LLVM_VER)/bin/llvm-config
LLVM_CONFIG = ../llvm-$(LLVM_VER)/bin/llvm-config
else
CLANG_CMAKE_OPTS += -DLLVM_TABLEGEN_EXE=$(BASE_JULIA_BIN)/../tools/llvm-tblgen
endif

JULIA_LDFLAGS = -L$(BASE_JULIA_BIN)/../lib -L$(BASE_JULIA_BIN)/../lib/julia
CLANG_LIBS := clangFrontendTool clangBasic clangLex clangDriver clangFrontend clangParse
CLANG_LIBS += clangAST clangASTMatchers clangSema clangAnalysis clangEdit
CLANG_LIBS += clangRewriteFrontend clangRewrite clangSerialization clangStaticAnalyzerCheckers
CLANG_LIBS += clangStaticAnalyzerCore clangStaticAnalyzerFrontend clangTooling clangToolingCore
CLANG_LIBS += clangCodeGen clangARCMigrate clangFormat
LINKED_LIBS = $(addprefix -l,$(CLANG_LIBS))

$(LLVM_CLANG_TAR): | src
curl -Lo $@ $(LLVM_SRC_URL)/$(notdir $@)
$(LLVM_COMPILER_RT_TAR): | src
$(JLDOWNLOAD) $@ $(LLVM_SRC_URL)/$(notdir $@)
src/clang-$(LLVM_VER): $(LLVM_CLANG_TAR)
mkdir -p $@
tar -C $@ --strip-components=1 -xf $<
build/clang-$(LLVM_VER)/Makefile: src/clang-$(LLVM_VER) $(CLANG_CMAKE_DEP)
mkdir -p $(dir $@)
cd $(dir $@) && \
cmake -G "Unix Makefiles" \
-DLLVM_BUILD_LLVM_DYLIB=ON -DCMAKE_BUILD_TYPE=Release \
-DLLVM_LINK_LLVM_DYLIB=ON -DLLVM_ENABLE_THREADS=OFF \
-DCMAKE_CXX_COMPILER_ARG1="$(CXX_ABI_SETTING)" \
-DLLVM_CONFIG=$(LLVM_CONFIG) $(CLANG_CMAKE_OPTS) ../../src/clang-$(LLVM_VER)
build/clang-$(LLVM_VER)/lib/libclangCodeGen.a: build/clang-$(LLVM_VER)/Makefile
cd build/clang-$(LLVM_VER) && $(MAKE)
LIB_DEPENDENCY += build/clang-$(LLVM_VER)/lib/libclangCodeGen.a
JULIA_LDFLAGS += -Lbuild/clang-$(LLVM_VER)/lib
CXXJL_CPPFLAGS += -Isrc/clang-$(LLVM_VER)/lib -Ibuild/clang-$(LLVM_VER)/include \
-Isrc/clang-$(LLVM_VER)/include
else # BUILD_LLVM_CLANG
JULIA_LDFLAGS = -L$(BASE_JULIA_BIN)/../lib -L$(BASE_JULIA_BIN)/../lib/julia
CXXJL_CPPFLAGS += -I$(JULIA_SRC)/deps/srccache/llvm-$(LLVM_VER)/tools/clang/lib \
-I$(JULIA_SRC)/deps/llvm-$(LLVM_VER)/tools/clang/lib
endif
else
LLVM_SRC_DIR := src/llvm-$(LLVM_VER)
CLANG_CMAKE_DEP = build/llvm-$(LLVM_VER)/bin/llvm-config
LLVM_CONFIG = ../llvm-$(LLVM_VER)/bin/llvm-config
JULIA_LDFLAGS += -Lbuild/clang-$(LLVM_VER)/lib
LLVM_HEADER_DIRS = src/llvm-$(LLVM_VER)/include build/llvm-$(LLVM_VER)/include
CXXJL_CPPFLAGS += -Isrc/clang-$(LLVM_VER)/lib -Ibuild/clang-$(LLVM_VER)/include \
-Isrc/clang-$(LLVM_VER)/include
JULIA_LDFLAGS += -L$(BASE_JULIA_BIN)/../lib -L$(BASE_JULIA_BIN)/../lib/julia
endif # PREBUILD_CI_BINARIES

CXX_LLVM_VER := $(LLVM_VER)
ifeq ($(CXX_LLVM_VER),svn)
CXX_LLVM_VER := $(shell $(BASE_JULIA_BIN)/../tools/llvm-config --version)
LIB_DIRS := $(JULIA_BIN)/../lib
LIB_DIRS += $(JULIA_BIN)/../lib/julia
ifneq ($(USEMSVC), 1)
LIB_DIRS += $(JULIA_BIN)
endif
JULIA_LDFLAGS = $(addprefix -L,$(LIB_DIRS))
# JULIA_LDFLAGS += -Lbuild/clang-$(LLVM_VER)/lib

ifneq ($(LLVM_HEADER_DIRS),)
CXXJL_CPPFLAGS += $(addprefix -I,$(LLVM_HEADER_DIRS))
endif

FLAGS = -std=c++11 $(CPPFLAGS) $(CFLAGS) $(CXXJL_CPPFLAGS)

Expand All @@ -149,58 +103,42 @@ else
CPP_STDOUT := $(CPP) -E
endif

ifeq ($(LLVM_USE_CMAKE),1)
LLVM_LIB_NAME := LLVM
else ifeq ($(LLVM_VER),svn)
LLVM_LIB_NAME := LLVM
else
LLVM_LIB_NAME := LLVM-$(CXX_LLVM_VER)
endif
LDFLAGS += -l$(LLVM_LIB_NAME)
LIB_DEPENDENCY += $(JULIA_BIN)/../lib/lib$(LLVM_LIB_NAME).$(SHLIB_EXT)
LLVM_EXTRA_CPPFLAGS = -DLLVM_NDEBUG
JULIA_LIB := $(JULIA_BIN)/../lib/libjulia.$(SHLIB_EXT)
JULIA_DEBUG_LIB := $(JULIA_BIN)/../lib/libjulia-debug.$(SHLIB_EXT)

LIB_DEPENDENCY += $(LIBDIR)/lib$(LLVM_LIB_NAME).$(SHLIB_EXT)
all: usr/lib/libcxxffi.$(SHLIB_EXT) usr/lib/libcxxffi-debug.$(SHLIB_EXT) usr/clang_constants.jl

usr/lib:
@mkdir -p $(CURDIR)/usr/lib/

build:
@mkdir -p $(CURDIR)/build

LLVM_EXTRA_CPPFLAGS =
ifneq ($(LLVM_ASSERTIONS),1)
LLVM_EXTRA_CPPFLAGS += -DLLVM_NDEBUG
endif
usr/lib: usr/src
mkdir $@

build/bootstrap.o: ../src/bootstrap.cpp BuildBootstrap.Makefile $(LIB_DEPENDENCY) | build
usr/lib/bootstrap.o: ../src/bootstrap.cpp BuildBootstrap.Makefile $(LIB_DEPENDENCY) llvm-patched | usr/lib
@$(call PRINT_CC, $(CXX) $(CXX_ABI_SETTING) -fno-rtti -DLIBRARY_EXPORTS -fPIC -O0 -g $(FLAGS) $(LLVM_EXTRA_CPPFLAGS) -c ../src/bootstrap.cpp -o $@)


LINKED_LIBS = $(addprefix -l,$(CLANG_LIBS))
ifeq ($(BUILD_LLDB),1)
LINKED_LIBS += $(LLDB_LIBS)
endif

ifneq (,$(wildcard $(BASE_JULIA_BIN)/../lib/libjulia.$(SHLIB_EXT)))
usr/lib/libcxxffi.$(SHLIB_EXT): build/bootstrap.o $(LIB_DEPENDENCY) | usr/lib
ifneq (,$(wildcard $(JULIA_LIB)))
usr/lib/libcxxffi.$(SHLIB_EXT): usr/lib/bootstrap.o $(LIB_DEPENDENCY) | usr/lib
@$(call PRINT_LINK, $(CXX) -shared -fPIC $(JULIA_LDFLAGS) -ljulia $(LDFLAGS) -o $@ $(WHOLE_ARCHIVE) $(LINKED_LIBS) $(NO_WHOLE_ARCHIVE) $< )
else
usr/lib/libcxxffi.$(SHLIB_EXT):
@echo "Not building release library because corresponding julia RELEASE library does not exist."
@echo "To build, simply run the build again once the library at"
@echo $(build_libdir)/libjulia.$(SHLIB_EXT)
@echo $(JULIA_LIB)
@echo "has been built."
endif

ifneq (,$(wildcard $(BASE_JULIA_BIN)/../lib/libjulia-debug.$(SHLIB_EXT)))
ifneq (,$(wildcard $(JULIA_DEBUG_LIB)))
usr/lib/libcxxffi-debug.$(SHLIB_EXT): build/bootstrap.o $(LIB_DEPENDENCY) | usr/lib
@$(call PRINT_LINK, $(CXX) -shared -fPIC $(JULIA_LDFLAGS) -ljulia-debug $(LDFLAGS) -o $@ $(WHOLE_ARCHIVE) $(LINKED_LIBS) $(NO_WHOLE_ARCHIVE) $< )
else
usr/lib/libcxxffi-debug.$(SHLIB_EXT):
@echo "Not building debug library because corresponding julia DEBUG library does not exist."
@echo "To build, simply run the build again once the library at"
@echo $(build_libdir)/libjulia-debug.$(SHLIB_EXT)
@echo $(JULIA_DEBUG_LIB)
@echo "has been built."
endif

build/clang_constants.jl: ../src/cenumvals.jl.h usr/lib/libcxxffi.$(SHLIB_EXT)
usr/clang_constants.jl: ../src/cenumvals.jl.h usr/lib/libcxxffi.$(SHLIB_EXT)
@$(call PRINT_PERL, $(CPP_STDOUT) $(CXXJL_CPPFLAGS) -DJULIA ../src/cenumvals.jl.h > $@)
30 changes: 9 additions & 21 deletions deps/build.jl
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,11 @@
using Libdl

if haskey(ENV, "PREBUILT_CI_BINARIES") && ENV["PREBUILT_CI_BINARIES"] == "1"
# Try to download pre-built binaries
if !isdir("build") || length(readdir("build")) == 0
os_tag = Sys.isapple() ? "osx" : "linux"
run(`rm -rf build/ src/`)
filename = "llvm-$(os_tag)-$(Base.libllvm_version).tgz"
run(`wget https://s3.amazonaws.com/julia-cxx/$filename`)
run(`tar xzf $filename --strip-components=1`)
end
end

#in case we have specified the path to the julia installation
#that contains the headers etc, use that
BASE_JULIA_BIN = get(ENV, "BASE_JULIA_BIN", Sys.BINDIR)
BASE_JULIA_SRC = get(ENV, "BASE_JULIA_SRC", joinpath(BASE_JULIA_BIN, "..", ".."))
# in case we have specified the path to the julia installation
# that contains the headers etc, use that
BASE_JULIA_BIN = get(ENV, "BASE_JULIA_BIN", Sys.BINDIR) |> normpath
BASE_JULIA_SRC = get(ENV, "BASE_JULIA_SRC", joinpath(BASE_JULIA_BIN, "..", "..")) |> normpath

#write a simple include file with that path
# write a simple include file with that path
println("writing path.jl file")
s = """
const BASE_JULIA_BIN=$(sprint(show, BASE_JULIA_BIN))
Expand All @@ -28,21 +17,20 @@ export BASE_JULIA_SRC

println("Tuning for julia installation at $BASE_JULIA_BIN with sources possibly at $BASE_JULIA_SRC")

# Try to autodetect C++ ABI in use
# try to autodetect C++ ABI in use
llvm_path = Sys.iswindows() ? "LLVM" :
(Sys.isapple() && Base.libllvm_version >= v"3.8") ? "libLLVM" : "libLLVM-$(Base.libllvm_version)"

Sys.isapple() ? "libLLVM" : "libLLVM-$(Base.libllvm_version)"
llvm_lib_path = Libdl.dlpath(llvm_path)
old_cxx_abi = findfirst("_ZN4llvm3sys16getProcessTripleEv", String(open(read, llvm_lib_path))) !== nothing
old_cxx_abi && (ENV["OLD_CXX_ABI"] = "1")

llvm_config_path = joinpath(BASE_JULIA_BIN,"..","tools","llvm-config")
llvm_config_path = joinpath(BASE_JULIA_BIN, "..", "tools", "llvm-config")
if isfile(llvm_config_path)
@info "Building julia source build"
ENV["LLVM_CONFIG"] = llvm_config_path
delete!(ENV,"LLVM_VER")
make = Sys.isbsd() && !Sys.isapple() ? `gmake` : `make`
run(`$make -j$(Sys.CPU_THREADS) -f BuildBootstrap.Makefile BASE_JULIA_BIN=$BASE_JULIA_BIN BASE_JULIA_SRC=$BASE_JULIA_SRC`)
run(`$make all -j$(Sys.CPU_THREADS) -f BuildBootstrap.Makefile BASE_JULIA_BIN=$BASE_JULIA_BIN BASE_JULIA_SRC=$BASE_JULIA_SRC`)
s = s * "\n const IS_BINARYBUILD = false"
else
@info "Building julia binary build"
Expand Down
6 changes: 0 additions & 6 deletions deps/llvm-patches/README
View file

This file was deleted.

Loading

0 comments on commit c5bdd65

Please sign in to comment.