Merge pull request #462 from WeBankFinTech/dev-0.11.0

Dev 0.11.0
apache · Jan 19, 2021 · c5dbcb1 · c5dbcb1
2 parents 4ac1a51 + fea04cf
commit c5dbcb1
Show file tree

Hide file tree

Showing 240 changed files with 5,417 additions and 412 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,8 @@
 .idea
 .DS_Store
 
+target/
+
 assembly/target
 
 core/cloudModule/target/
@@ -65,6 +67,7 @@ ujes/definedEngines/python/entrance/target/
 ujes/definedEngines/spark/engine/target/
 ujes/definedEngines/spark/enginemanager/target/
 ujes/definedEngines/spark/entrance/target/
+ujes/definedEngines/presto/entrance/target/
 ujes/engine/target/
 ujes/enginemanager/target/
 ujes/entrance/target/

diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -21,8 +21,8 @@
     <parent>
         <groupId>com.webank.wedatasphere.linkis</groupId>
         <artifactId>linkis</artifactId>
-        <version>0.10.0</version>
-	<relativePath>../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/assembly/public-module/pom.xml b/assembly/public-module/pom.xml
@@ -18,8 +18,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/assembly/src/main/assembly/assembly.xml b/assembly/src/main/assembly/assembly.xml
@@ -196,6 +196,28 @@
       </includes>
     </fileSet>-->
 
+    <!-- elasticsearch -->
+    <fileSet>
+      <directory>
+        ${project.parent.basedir}/ujes/definedEngines/es/entrance/target/
+      </directory>
+      <outputDirectory>share/linkis/ujes/es</outputDirectory>
+      <includes>
+        <include>*.zip</include>
+      </includes>
+    </fileSet>
+
+    <!-- presto -->
+    <fileSet>
+      <directory>
+        ${project.parent.basedir}/ujes/definedEngines/presto/entrance/target/
+      </directory>
+      <outputDirectory>share/linkis/ujes/presto</outputDirectory>
+      <includes>
+        <include>*.zip</include>
+      </includes>
+    </fileSet>
+
     <!--resourcemanager-->
     <fileSet>
       <directory>

diff --git a/bml/bml-engine-hook/pom.xml b/bml/bml-engine-hook/pom.xml
@@ -5,8 +5,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlHookConf.scala b/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlHookConf.scala
@@ -1,11 +1,12 @@
 package com.webank.wedatasphere.linkis.bml.conf
 
+import com.webank.wedatasphere.linkis.bml.utils.BmlHookUtils
 import com.webank.wedatasphere.linkis.common.conf.CommonVars
 
 /**
   * created by cooperyang on 2019/9/23
   * Description:
   */
 object BmlHookConf {
-  val WORK_DIR_STR = CommonVars("wds.linkis.bml.work.dir", "user.dir")
+  val WORK_DIR_STR = CommonVars("wds.linkis.bml.work.dir", BmlHookUtils.getCurrentWorkDir)
 }
diff --git a/...hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala b/...hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala
@@ -4,6 +4,7 @@ import java.io.File
 import java.util
 
 import com.webank.wedatasphere.linkis.bml.client.{BmlClient, BmlClientFactory}
+import com.webank.wedatasphere.linkis.bml.conf.BmlHookConf
 import com.webank.wedatasphere.linkis.bml.exception.BmlHookDownloadException
 import com.webank.wedatasphere.linkis.bml.utils.BmlHookUtils
 import com.webank.wedatasphere.linkis.common.exception.ErrorException
@@ -42,12 +43,13 @@ class BmlEnginePreExecuteHook extends EnginePreExecuteHook with Logging{
   val pathType:String = "file://"
 
   override def callPreExecuteHook(engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String = {
-    val workDir = BmlHookUtils.getCurrentWorkDir
+    val workDir = BmlHookConf.WORK_DIR_STR.getValue
     val jobId = engineExecutorContext.getJobId
+    var hookCode = code
     executeRequest match {
       case resourceExecuteRequest:ResourceExecuteRequest => val resources = resourceExecuteRequest.resources
-        if (null == resources) return code
-        resources foreach {
+        if (null == resources) return hookCode
+        val resourcePaths = resources map {
           case resource:util.Map[String, Object] => val fileName = resource.get(FILE_NAME_STR).toString
             val resourceId = resource.get(RESOURCE_ID_STR).toString
             val version = resource.get(VERSION_STR).toString
@@ -65,13 +67,23 @@ class BmlEnginePreExecuteHook extends EnginePreExecuteHook with Logging{
             }
             if (response.isSuccess){
               logger.info(s"for job $jobId resourceId $resourceId version $version download to path $fullPath ok")
+              fullPath
             }else{
               logger.warn(s"for job $jobId resourceId $resourceId version $version download to path $fullPath Failed")
+              null
             }
-          case _ => logger.warn("job resource cannot download")
+          case _ =>
+            logger.warn("job resource cannot download")
+            null
         }
+        hookCode = if (StringUtils.isNotBlank(hookCode)) hookCode else executeRequest.code
+        hookCode = callResourcesDownloadedHook(resourcePaths.toArray, engineExecutorContext, executeRequest, hookCode)
       case _ =>
     }
-    if (StringUtils.isNotBlank(code)) code else executeRequest.code
+    if (StringUtils.isNotBlank(hookCode)) hookCode else executeRequest.code
+  }
+
+  def callResourcesDownloadedHook(resourcePaths: Array[String], engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String = {
+    code
   }
 }
diff --git a/...ml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/utils/BmlHookUtils.scala b/...ml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/utils/BmlHookUtils.scala
@@ -1,7 +1,5 @@
 package com.webank.wedatasphere.linkis.bml.utils
 
-import com.webank.wedatasphere.linkis.common.utils.Utils
-
 /**
   * created by cooperyang on 2019/9/24
   * Description:

diff --git a/bml/bmlclient/pom.xml b/bml/bmlclient/pom.xml
@@ -5,8 +5,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/bml/bmlcommon/pom.xml b/bml/bmlcommon/pom.xml
@@ -5,8 +5,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/bml/bmlserver/pom.xml b/bml/bmlserver/pom.xml
@@ -5,7 +5,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/bml/bmlserver/pom_k8s.xml b/bml/bmlserver/pom_k8s.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/conf/config.sh b/conf/config.sh
@@ -132,4 +132,4 @@ MDM_PORT=9118
 ## java application default jvm memory
 export SERVER_HEAP_SIZE="512M"
 
-LINKIS_VERSION=0.10.0
+LINKIS_VERSION=0.11.0
diff --git a/contextservice/cs-cache/pom.xml b/contextservice/cs-cache/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-client/pom.xml b/contextservice/cs-client/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-common/pom.xml b/contextservice/cs-common/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-highavailable/pom.xml b/contextservice/cs-highavailable/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-listener/pom.xml b/contextservice/cs-listener/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-persistence/pom.xml b/contextservice/cs-persistence/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-search/pom.xml b/contextservice/cs-search/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-server/pom.xml b/contextservice/cs-server/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-server/pom_k8s.xml b/contextservice/cs-server/pom_k8s.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/contextservice/cs-ujes-client/pom.xml b/contextservice/cs-ujes-client/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
+        <version>0.11.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/core/cloudModule/pom.xml b/core/cloudModule/pom.xml
@@ -21,8 +21,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -132,6 +132,10 @@
             <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
             <version>2.0.0.RELEASE</version>
             <exclusions>
+                <exclusion>
+                    <artifactId>jsr311-api</artifactId>
+                    <groupId>javax.ws.rs</groupId>
+                </exclusion>
                 <exclusion>
                     <artifactId>jackson-databind</artifactId>
                     <groupId>com.fasterxml.jackson.core</groupId>

diff --git a/core/cloudMybatis/pom.xml b/core/cloudMybatis/pom.xml
@@ -21,8 +21,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>jar</packaging>

diff --git a/core/cloudProtocol/pom.xml b/core/cloudProtocol/pom.xml
@@ -21,8 +21,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/...col/src/main/scala/com/webank/wedatasphere/linkis/protocol/engine/ResponseNewEngine.scala b/...col/src/main/scala/com/webank/wedatasphere/linkis/protocol/engine/ResponseNewEngine.scala
@@ -32,6 +32,11 @@ trait ResponseEngine extends RetryableProtocol {
   * @param pid
   */
 case class ResponseEnginePid(port: Int, pid: String) extends RetryableProtocol
+/**
+  * engine send to engineManager
+  * @param host
+  */
+case class ResponseEngineHost(port: Int, host: String) extends RetryableProtocol
 /**
   * engine send to engineManager
   * @param port

diff --git a/core/cloudRPC/pom.xml b/core/cloudRPC/pom.xml
@@ -21,8 +21,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/BaseRPCSender.scala b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/BaseRPCSender.scala
@@ -91,7 +91,7 @@ private[rpc] class BaseRPCSender extends Sender with Logging {
     val msg = RPCProduct.getRPCProduct.toMessage(message)
     msg.data("duration", timeout.toMillis)
     BaseRPCSender.addInstanceInfo(msg.getData)
-    val response = getRPC.receiveAndReply(msg)
+    val response = getRPC.receiveAndReplyInMills(msg)
     RPCConsumer.getRPCConsumer.toObject(response)
   }
 

diff --git a/core/common/pom.xml b/core/common/pom.xml
@@ -21,8 +21,8 @@
     <parent>
         <artifactId>linkis</artifactId>
         <groupId>com.webank.wedatasphere.linkis</groupId>
-        <version>0.10.0</version>
-	<relativePath>../../pom.xml</relativePath>
+        <version>0.11.0</version>
+        <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>