From 89a11e15e7daf565ce8032c4425d8bea60e367f1 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 18 Jun 2024 22:43:43 +0200 Subject: [PATCH] fix(single-binary): bundle ld.so (#2602) * debug * fix copy command/silly muscle memory Signed-off-by: Ettore Di Giacinto * remove tmate * Debugging * Start binary with ld.so if present in libdir Signed-off-by: Ettore Di Giacinto * small refactor Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto --- .github/workflows/release.yaml | 6 +++-- Makefile | 2 +- core/cli/models.go | 1 + pkg/library/dynaload.go | 46 +++++++++++++++++++++++++++++++--- pkg/model/initializers.go | 10 ++++++-- pkg/model/process.go | 4 +-- 6 files changed, 58 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d89d49ba5b71..536f4e84c366 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -102,8 +102,9 @@ jobs: export PATH=/usr/local/cuda/bin:$PATH sudo rm -rf /usr/aarch64-linux-gnu/lib/libstdc++.so.6 sudo cp -rf /usr/aarch64-linux-gnu/lib/libstdc++.so* /usr/aarch64-linux-gnu/lib/libstdc++.so.6 + sudo cp /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 ld.so GO_TAGS=p2p \ - BACKEND_LIBS="./grpc/cmake/cross_build/third_party/re2/libre2.a ./grpc/cmake/cross_build/libgrpc.a ./grpc/cmake/cross_build/libgrpc++.a ./grpc/cmake/cross_build/third_party/protobuf/libprotobuf.a /usr/aarch64-linux-gnu/lib/libc.so.6 /usr/aarch64-linux-gnu/lib/libstdc++.so.6 /usr/aarch64-linux-gnu/lib/libgomp.so.1 /usr/aarch64-linux-gnu/lib/libm.so.6 /usr/aarch64-linux-gnu/lib/libgcc_s.so.1 /usr/aarch64-linux-gnu/lib/libdl.so.2 /usr/aarch64-linux-gnu/lib/libpthread.so.0" \ + BACKEND_LIBS="./grpc/cmake/cross_build/third_party/re2/libre2.a ./grpc/cmake/cross_build/libgrpc.a ./grpc/cmake/cross_build/libgrpc++.a ./grpc/cmake/cross_build/third_party/protobuf/libprotobuf.a /usr/aarch64-linux-gnu/lib/libc.so.6 /usr/aarch64-linux-gnu/lib/libstdc++.so.6 /usr/aarch64-linux-gnu/lib/libgomp.so.1 /usr/aarch64-linux-gnu/lib/libm.so.6 /usr/aarch64-linux-gnu/lib/libgcc_s.so.1 /usr/aarch64-linux-gnu/lib/libdl.so.2 /usr/aarch64-linux-gnu/lib/libpthread.so.0 ./ld.so" \ GOOS=linux \ GOARCH=arm64 \ CMAKE_ARGS="-DProtobuf_INCLUDE_DIRS=$CROSS_STAGING_PREFIX/include -DProtobuf_DIR=$CROSS_STAGING_PREFIX/lib/cmake/protobuf -DgRPC_DIR=$CROSS_STAGING_PREFIX/lib/cmake/grpc -DCMAKE_TOOLCHAIN_FILE=$CMAKE_CROSS_TOOLCHAIN -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++" make dist-cross-linux-arm64 @@ -212,8 +213,9 @@ jobs: export PATH=/usr/local/cuda/bin:$PATH export PATH=/opt/rocm/bin:$PATH source /opt/intel/oneapi/setvars.sh + sudo cp /lib64/ld-linux-x86-64.so.2 ld.so GO_TAGS=p2p \ - BACKEND_LIBS="/usr/lib/x86_64-linux-gnu/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libm.so.6 /usr/lib/x86_64-linux-gnu/libgcc_s.so.1 /usr/lib/x86_64-linux-gnu/libc.so.6 /usr/lib/x86_64-linux-gnu/libgomp.so.1" \ + BACKEND_LIBS="./ld.so /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libm.so.6 /usr/lib/x86_64-linux-gnu/libgcc_s.so.1 /usr/lib/x86_64-linux-gnu/libc.so.6 /usr/lib/x86_64-linux-gnu/libgomp.so.1" \ make -j4 dist - uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index 8ec61e04c163..5a2acafb4300 100644 --- a/Makefile +++ b/Makefile @@ -315,7 +315,7 @@ build: prepare backend-assets grpcs ## Build the project $(info ${GREEN}I LD_FLAGS: ${YELLOW}$(LD_FLAGS)${RESET}) ifneq ($(BACKEND_LIBS),) $(MAKE) backend-assets/lib - cp -r $(BACKEND_LIBS) backend-assets/lib/ + cp $(BACKEND_LIBS) backend-assets/lib/ endif CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o $(BINARY_NAME) ./ diff --git a/core/cli/models.go b/core/cli/models.go index 0e01eddca7ae..a6ba39b9a091 100644 --- a/core/cli/models.go +++ b/core/cli/models.go @@ -57,6 +57,7 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error { if err := json.Unmarshal([]byte(mi.Galleries), &galleries); err != nil { log.Error().Err(err).Msg("unable to load galleries") } + for _, modelName := range mi.ModelArgs { progressBar := progressbar.NewOptions( diff --git a/pkg/library/dynaload.go b/pkg/library/dynaload.go index 57b002591d8c..172e389a2c31 100644 --- a/pkg/library/dynaload.go +++ b/pkg/library/dynaload.go @@ -5,11 +5,20 @@ import ( "os" "path/filepath" "runtime" + + "github.com/rs/zerolog/log" ) +/* + This file contains functions to load libraries from the asset directory to keep the business logic clean. +*/ + +// skipLibraryPath checks if LOCALAI_SKIP_LIBRARY_PATH is set +var skipLibraryPath = os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" + +// LoadExtractedLibs loads the extracted libraries from the asset dir func LoadExtractedLibs(dir string) { - // Skip this if LOCALAI_SKIP_LIBRARY_PATH is set - if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" { + if skipLibraryPath { return } @@ -18,9 +27,38 @@ func LoadExtractedLibs(dir string) { } } +// LoadLDSO checks if there is a ld.so in the asset dir and if so, prefixes the grpc process with it. +// In linux, if we find a ld.so in the asset dir we prefix it to run with the libs exposed in +// LD_LIBRARY_PATH for more compatibility +// If we don't do this, we might run into stack smash +// See also: https://stackoverflow.com/questions/847179/multiple-glibc-libraries-on-a-single-host/851229#851229 +// In this case, we expect a ld.so in the lib asset dir. +// If that's present, we use it to run the grpc backends as supposedly built against +// that specific version of ld.so +func LoadLDSO(assetDir string, args []string, grpcProcess string) ([]string, string) { + if skipLibraryPath { + return args, grpcProcess + } + + if runtime.GOOS != "linux" { + return args, grpcProcess + } + + // Check if there is a ld.so file in the assetDir, if it does, we need to run the grpc process with it + ldPath := filepath.Join(assetDir, "backend-assets", "lib", "ld.so") + if _, err := os.Stat(ldPath); err == nil { + log.Debug().Msgf("ld.so found") + // We need to run the grpc process with the ld.so + args = append(args, grpcProcess) + grpcProcess = ldPath + } + + return args, grpcProcess +} + +// LoadExternal sets the LD_LIBRARY_PATH to include the given directory func LoadExternal(dir string) { - // Skip this if LOCALAI_SKIP_LIBRARY_PATH is set - if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" { + if skipLibraryPath { return } diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index 7572735e32bb..5c01e3668102 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -11,6 +11,7 @@ import ( "time" grpc "github.com/go-skynet/LocalAI/pkg/grpc" + "github.com/go-skynet/LocalAI/pkg/library" "github.com/go-skynet/LocalAI/pkg/xsysinfo" "github.com/klauspost/cpuid/v2" "github.com/phayes/freeport" @@ -326,8 +327,13 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string return "", fmt.Errorf("failed allocating free ports: %s", err.Error()) } - // Make sure the process is executable - if err := ml.startProcess(grpcProcess, o.model, serverAddress); err != nil { + args := []string{} + + // Load the ld.so if it exists + args, grpcProcess = library.LoadLDSO(o.assetDir, args, grpcProcess) + + // Make sure the process is executable in any circumstance + if err := ml.startProcess(grpcProcess, o.model, serverAddress, args...); err != nil { return "", err } diff --git a/pkg/model/process.go b/pkg/model/process.go index b7180f5d8948..58def58c2373 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -69,7 +69,7 @@ func (ml *ModelLoader) GetGRPCPID(id string) (int, error) { return strconv.Atoi(p.PID) } -func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string) error { +func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string, args ...string) error { // Make sure the process is executable if err := os.Chmod(grpcProcess, 0700); err != nil { return err @@ -82,7 +82,7 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string grpcControlProcess := process.New( process.WithTemporaryStateDir(), process.WithName(grpcProcess), - process.WithArgs("--addr", serverAddress), + process.WithArgs(append(args, []string{"--addr", serverAddress}...)...), process.WithEnvironment(os.Environ()...), )