Part 2 of #3933: failing make install-arrow (#3944)

Bears-R-Us · Dec 16, 2024 · 2cd4bf1 · 2cd4bf1
1 parent 0270c50
commit 2cd4bf1
Show file tree

Hide file tree

Showing 2 changed files with 63 additions and 30 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -75,7 +75,7 @@ jobs:
     - name: Install dependencies
       run: |
         apt-get update && apt-get install -y -V ca-certificates lsb-release wget
-        make install-arrow
+        make install-arrow-quick
         apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev
         make install-iconv
         echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
@@ -110,21 +110,24 @@ jobs:
     - name: Install dependencies
       run: |
         apt-get update && apt-get install -y -V ca-certificates lsb-release wget build-essential ninja-build cmake
-        apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libcurl4-openssl-dev libidn2-dev
+        apt-get update && apt-get install -y -V libhdf5-dev hdf5-tools libzmq3-dev python3-pip libcurl4-openssl-dev libidn2-dev
+        apt-get update -y && apt-get -y -V upgrade 
         echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
     - name: Check python version
       run: |
         python3 --version
+
     - name: Install Chapel frontend bindings
       run: |
         (cd $CHPL_HOME/tools/chapel-py && python3 -m pip install .)
-    - name: Make install-arrow
+    - name: Make install-arrow-quick
       run: |
-        make install-arrow
-    - name: Make install-arrow OS=unrecognized
+        make install-arrow-quick
+    - name: Make install-arrow
       run: |
+        apt-get remove -y apache-arrow-apt-source
         make arrow-clean
-        make install-arrow OS=unrecognized              
+        make install-arrow    
     - name: Make install-hdf5
       run: |
         make install-hdf5
@@ -155,7 +158,7 @@ jobs:
     - name: Install dependencies
       run: |
         apt-get update && apt-get install -y -V ca-certificates lsb-release wget
-        make install-arrow
+        make install-arrow-quick
         apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev
         make install-iconv
         
@@ -186,7 +189,7 @@ jobs:
     - name: Install dependencies
       run: |
         apt-get update && apt-get install -y -V ca-certificates lsb-release wget
-        make install-arrow
+        make install-arrow-quick
         apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev
         make install-iconv
         echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
@@ -225,7 +228,7 @@ jobs:
     - name: Install dependencies
       run: |
         apt-get update && apt-get install -y -V ca-certificates lsb-release wget
-        make install-arrow
+        make install-arrow-quick
         apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev
         make install-iconv
         echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
@@ -270,7 +273,7 @@ jobs:
     - name: Install dependencies
       run: |
         apt-get update && apt-get install -y -V ca-certificates lsb-release wget
-        make install-arrow
+        make install-arrow-quick
         apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev 
         pip install pytest-benchmark==4.0.0
         make install-iconv

diff --git a/Makefile b/Makefile
@@ -190,13 +190,6 @@ OS_CODENAME :=$(shell lsb_release --codename --short)
 #   System release, for example, 22 extracted from 22.04
 OS_RELEASE :=$(shell lsb_release -rs | cut -d'.' -f1)
 
-ARROW_VER := 18.1.0
-ARROW_NAME_VER := apache-arrow-$(ARROW_VER)
-ARROW_FULL_NAME_VER := arrow-apache-arrow-$(ARROW_VER)
-ARROW_BUILD_DIR := $(DEP_BUILD_DIR)/$(ARROW_FULL_NAME_VER)
-ARROW_INSTALL_DIR := $(DEP_INSTALL_DIR)/arrow-install
-
-NUM_CORES := $(shell nproc --all)
 
 ifeq ($(OS_FINAL),$(filter $(OS_FINAL), almalinux almalinux-rc))
     ARROW_LINK := "https://apache.jfrog.io/ui/native/arrow/${OS_FINAL}/${OS_RELEASE}/apache-arrow-release-latest.rpm"
@@ -210,11 +203,10 @@ else ifeq ($(OS_FINAL), centos-rc)
     ARROW_LINK := "https://apache.jfrog.io/ui/native/arrow/centos-rc/9-stream/apache-arrow-release-latest.rpm"
 else ifeq ($(OS_FINAL),$(filter $(OS_FINAL), ubuntu ubuntu-rc debian debian-rc))
     ARROW_LINK := "https://apache.jfrog.io/artifactory/arrow/${OS_FINAL}/apache-arrow-apt-source-latest-${OS_CODENAME}.deb"
-else
-    ARROW_LINK := https://github.com/apache/arrow/archive/refs/tags/$(ARROW_NAME_VER).tar.gz
 endif
 
-install-arrow:
+
+install-arrow-quick:
 	@echo "Installing Apache Arrow/Parquet"
 	@echo "from build directory: ${DEP_BUILD_DIR}"
 	mkdir -p ${DEP_BUILD_DIR}
@@ -239,20 +231,58 @@ install-arrow:
 			cd $(DEP_BUILD_DIR) && apt install -y -V ./apache-arrow*
         endif
     else
+		@echo "make install-arrow-quick does not support ${OS}.  Please use make install-arrow instead."
+    endif
+
+
+ARROW_VER := 18.1.0
+ARROW_NAME_VER := apache-arrow-$(ARROW_VER)
+ARROW_FULL_NAME_VER := arrow-apache-arrow-$(ARROW_VER)
+ARROW_BUILD_DIR := $(DEP_BUILD_DIR)/$(ARROW_FULL_NAME_VER)
+ARROW_INSTALL_DIR := $(DEP_INSTALL_DIR)/arrow-install
+ARROW_SOURCE_LINK := https://github.com/apache/arrow/archive/refs/tags/$(ARROW_NAME_VER).tar.gz
+
+NUM_CORES := $(shell nproc --all)
+
+install-arrow:
+	@echo "Installing Apache Arrow/Parquet"
+	@echo "from build directory: ${DEP_BUILD_DIR}"
+    ifneq ($(shell id -u), 0)
+		sudo rm -rf $(ARROW_INSTALL_DIR)
+    else
+		rm -rf $(ARROW_INSTALL_DIR)
+    endif
+	mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)
+
+    #   If the BUILD_DIR does not contain the apache-arrow file, use wget to fetch it
+    ifeq (,$(wildcard ${DEP_BUILD_DIR}/apache-arrow*))
+		cd $(DEP_BUILD_DIR) && wget $(ARROW_SOURCE_LINK) && tar -xvf $(ARROW_NAME_VER).tar.gz
+		#mkdir -p $(ARROW_BUILD_DIR)/cpp/build-release
+		#cd $(DEP_BUILD_DIR) && tar -xvf $(ARROW_NAME_VER).tar.gz
+    #   If the tar.gz file exists, unzip it
+    else ifneq (,$(wildcard ${DEP_BUILD_DIR}/$(ARROW_NAME_VER).tar.gz))
 		cd $(DEP_BUILD_DIR) && tar -xvf $(ARROW_NAME_VER).tar.gz
-		mkdir -p $(ARROW_BUILD_DIR)/cpp/build-release
-		cd $(ARROW_BUILD_DIR)/cpp/build-release && cmake .. -DCMAKE_INSTALL_PREFIX=$(ARROW_INSTALL_DIR) -DARROW_DEPENDENCY_SOURCE=AUTO -DCMAKE_BUILD_TYPE=Release -DARROW_PARQUET=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON $(ARROW_OPTIONS) && make -j$(NUM_CORES)
-        #   If not root, use sudo
-        ifneq ($(shell id -u), 0)
-			cd $(ARROW_BUILD_DIR)/cpp/build-release && sudo make install
-        else
-			cd $(ARROW_BUILD_DIR)/cpp/build-release && make install
-        endif
+    endif
+
+	mkdir -p $(ARROW_BUILD_DIR)/cpp/build-release
+	cd $(ARROW_BUILD_DIR)/cpp/build-release && cmake -S $(ARROW_BUILD_DIR)/cpp .. -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=$(ARROW_INSTALL_DIR) -DARROW_DEPENDENCY_SOURCE=AUTO -DCMAKE_BUILD_TYPE=Release -DARROW_PARQUET=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON $(ARROW_OPTIONS) && make -j$(NUM_CORES)
+    #   If not root, use sudo
+    ifneq ($(shell id -u), 0)
+		cd $(ARROW_BUILD_DIR)/cpp/build-release && sudo make install
+    else
+		cd $(ARROW_BUILD_DIR)/cpp/build-release && make install
     endif
 
 arrow-clean:
-	rm -rf $(DEP_BUILD_DIR)/apache-arrow*
-	rm -rf $(DEP_BUILD_DIR)/arrow-apache-arrow*	
+    #   If not root, use sudo
+    ifneq ($(shell id -u), 0)
+		sudo rm -rf $(DEP_BUILD_DIR)/apache-arrow*
+		sudo rm -rf $(DEP_BUILD_DIR)/arrow-apache-arrow*	
+    else
+		rm -rf $(DEP_BUILD_DIR)/apache-arrow*
+		rm -rf $(DEP_BUILD_DIR)/arrow-apache-arrow*	
+    endif
+
 
 ICONV_VER := 1.17
 ICONV_NAME_VER := libiconv-$(ICONV_VER)