Skip to content

After fixing the setting ENABLE_HDFS=false, need 'hdfs' (your linux command not found) is still reported. #763

After fixing the setting ENABLE_HDFS=false, need 'hdfs' (your linux command not found) is still reported.

After fixing the setting ENABLE_HDFS=false, need 'hdfs' (your linux command not found) is still reported. #763

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Integration Test
on:
push:
branches: [master,dev-*]
pull_request:
branches: [master,dev-*]
#concurrency:
# group: test-${{ github.head_ref || github.run_id }}
# cancel-in-progress: true
env:
KIND_CONFIG_PATH: './linkis-dist/helm/scripts/resources/kind-cluster.yaml'
KIND_CLUSTER_NAME: 'test-helm'
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
jobs:
integration-test:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [16.0.0]
kubernetes-version:
# - 'kindest/node:v1.21.10'
- 'kindest/node:v1.23.4'
timeout-minutes: 90
env:
TAG: ${{ github.sha }}
SKIP_TEST: true
HUB: ghcr.io/apache/linkis
LINKIS_VERSION: 1.5.0
steps:
- name: Free up disk space
run: |
# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
# du -sh /* 2> /dev/null | sort -rh 2> /dev/null | head
# du -h -d2 /usr 2> /dev/null | sort -rh 2> /dev/null | head
echo $JAVA_HOME
echo "Check free disk space before cleanup."
df -h
echo "Removing non-essential tools and libraries."
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo rm -rf /opt/ghc
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/share/boost
# delete libraries for Android (12G), PowerShell (1.3G), Swift (1.7G)
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/local/share/powershell
sudo rm -rf /usr/share/swift
echo "Check free disk space after cleanup."
df -h
echo $JAVA_HOME
- name: Prune docker images
run: |
echo "Pruning docker images on GH action runner node"
docker image prune -a -f
docker system df
df -h
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: true
- name: Set up JDK 8
uses: actions/setup-java@v2
with:
distribution: 'adopt'
java-version: 8
- name: Cache local Maven repository
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v2
with:
node-version: ${{ matrix.node-version }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Build frontend by node.js
run: |
cd linkis-web
sed -i "/VUE_APP_MN_CONFIG_PREFIX/d" .env
npm install
npm run build
- name: Build backend by maven
run: |
./mvnw install -Pdocker -Dmysql.connector.scope=compile -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.web=true -Dlinkis.build.ldh=true
- name: Set up chart-testing
uses: ./.github/actions/chart-testing-action
- name: Create Kind cluster
uses: ./.github/actions/kind-action
with:
config: ${{ env.KIND_CONFIG_PATH }}
node_image: ${{ matrix.kubernetes-version }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
- name: Start Linkis Service
run: |
docker tag linkis:${{ env.LINKIS_VERSION }} linkis:dev
docker tag linkis-web:${{ env.LINKIS_VERSION }} linkis-web:dev
docker tag linkis-ldh:${{ env.LINKIS_VERSION }} linkis-ldh:dev
# for debug Download the image directly first, then adjust it to active build
#ROOT_DIR=./linkis-dist/
#MIRRORS="ghcr.io"
#TAG="dev"
#docker pull ${MIRRORS}/apache/linkis/linkis-ldh:${TAG}
#docker pull ${MIRRORS}/apache/linkis/linkis:${TAG}
#docker pull ${MIRRORS}/apache/linkis/linkis-web:${TAG}
#docker tag ${MIRRORS}/apache/linkis/linkis:${TAG} linkis:dev
#docker tag ${MIRRORS}/apache/linkis/linkis-web:${TAG} linkis-web:dev
#docker tag ${MIRRORS}/apache/linkis/linkis-ldh:${TAG} linkis-ldh:dev
#show image list
docker image ls
bash ./linkis-dist/helm/scripts/install-mysql.sh false\
&& bash ./linkis-dist/helm/scripts/install-ldh.sh true \
&& bash ./linkis-dist/helm/scripts/install-charts-with-ldh.sh linkis linkis-demo true
n=0
sleep 60
while (($n<10))
do
kubectl get pods -A
n=$((n+1))
sleep 20
done
#show linkis pod logs
#POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-linkismanager -o jsonpath='{.items[0].metadata.name}'`
#kubectl logs -n linkis ${POD_NAME} -f --tail=10000
#POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-engineconnmanager -o jsonpath='{.items[0].metadata.name}'`
#kubectl logs -n linkis ${POD_NAME} -f --tail=10000
shell: bash
- name: Linkis-Cli Test
run: |
# Enable port-forward
bash ./linkis-dist/helm/scripts/remote-proxy.sh start
# Show port-forward list
bash ./linkis-dist/helm/scripts/remote-proxy.sh list
# Check if the web service is available
curl http://127.0.0.1:8088/indexhtml
# Execute test by linkis-cli
POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-mg-gateway -o jsonpath='{.items[0].metadata.name}'`
kubectl exec -it -n linkis ${POD_NAME} -- bash -c " \
sh /opt/linkis/bin/linkis-cli -engineType shell-1 -codeType shell -code \"pwd\" ";
kubectl exec -it -n linkis ${POD_NAME} -- bash -c " \
sh /opt/linkis/bin/linkis-cli -engineType python-python2 -codeType python -code 'print(\"hello\")' "
#todo
#kubectl exec -it -n linkis ${POD_NAME} -- bash -c " \
#sh /opt/linkis/bin/linkis-cli -engineType hive-3.1.3 -codeType hql -code 'show databases' "
#kubectl exec -it -n linkis ${POD_NAME} -- bash -c " \
#sh /opt/linkis/bin/linkis-cli -engineType spark-3.2.1 -codeType sql -code 'show databases' "
shell: bash