update pom version to 0.6.3

edp963 · Mar 18, 2020 · 625b1a0 · 625b1a0
1 parent f3f14b7
commit 625b1a0
Show file tree

Hide file tree

Showing 44 changed files with 122 additions and 101 deletions.
diff --git a/common/pom.xml b/common/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/dbdriver/dbpool/LICENSE-3RD-PARTY b/dbdriver/dbpool/LICENSE-3RD-PARTY
@@ -35,7 +35,7 @@ List of third-party dependencies grouped by their license type.
         * Lang (commons-lang:commons-lang:2.3 - http://jakarta.apache.org/commons/lang/)
         * Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/)
         * Commons Net (commons-net:commons-net:3.1 - http://commons.apache.org/net/)
-        * wormhole-util (edp.wormhole:wormhole-util:0.6.2 - no url defined)
+        * wormhole-util (edp.wormhole:wormhole-util:0.6.3 - no url defined)
         * Metrics Core (io.dropwizard.metrics:metrics-core:3.1.2 - http://metrics.codahale.com/metrics-core/)
         * The Netty Project (io.netty:netty:3.6.2.Final - http://netty.io/)
         * Netty/All-in-One (io.netty:netty-all:4.0.23.Final - http://netty.io/netty-all/)
@@ -91,12 +91,12 @@ List of third-party dependencies grouped by their license type.
         * Phoenix Query Server (org.apache.phoenix:phoenix-server:4.7.0-HBase-1.1 - http://www.apache.org/phoenix/phoenix-server/)
         * Phoenix Query Server Client (org.apache.phoenix:phoenix-server-client:4.7.0-HBase-1.1 - http://www.apache.org/phoenix/phoenix-server-client/)
         * Apache Thrift (org.apache.thrift:libthrift:0.9.0 - http://thrift.apache.org)
-        * Apache Twill API (org.apache.twill:twill-api:0.6.2-incubating - http://twill.incubator.apache.org/twill-api)
-        * Apache Twill common library (org.apache.twill:twill-common:0.6.2-incubating - http://twill.incubator.apache.org/twill-common)
-        * Apache Twill core library (org.apache.twill:twill-core:0.6.2-incubating - http://twill.incubator.apache.org/twill-core)
-        * Apache Twill discovery service API (org.apache.twill:twill-discovery-api:0.6.2-incubating - http://twill.incubator.apache.org/twill-discovery-api)
-        * Apache Twill discovery service implementations (org.apache.twill:twill-discovery-core:0.6.2-incubating - http://twill.incubator.apache.org/twill-discovery-core)
-        * Apache Twill ZooKeeper client library (org.apache.twill:twill-zookeeper:0.6.2-incubating - http://twill.incubator.apache.org/twill-zookeeper)
+        * Apache Twill API (org.apache.twill:twill-api:0.6.3-incubating - http://twill.incubator.apache.org/twill-api)
+        * Apache Twill common library (org.apache.twill:twill-common:0.6.3-incubating - http://twill.incubator.apache.org/twill-common)
+        * Apache Twill core library (org.apache.twill:twill-core:0.6.3-incubating - http://twill.incubator.apache.org/twill-core)
+        * Apache Twill discovery service API (org.apache.twill:twill-discovery-api:0.6.3-incubating - http://twill.incubator.apache.org/twill-discovery-api)
+        * Apache Twill discovery service implementations (org.apache.twill:twill-discovery-core:0.6.3-incubating - http://twill.incubator.apache.org/twill-discovery-core)
+        * Apache Twill ZooKeeper client library (org.apache.twill:twill-zookeeper:0.6.3-incubating - http://twill.incubator.apache.org/twill-zookeeper)
         * zookeeper (org.apache.zookeeper:zookeeper:3.4.6 - no url defined)
         * Jackson (org.codehaus.jackson:jackson-core-asl:1.9.2 - http://jackson.codehaus.org)
         * JAX-RS provider for JSON content type (org.codehaus.jackson:jackson-jaxrs:1.9.13 - http://jackson.codehaus.org)

diff --git a/dbdriver/dbpool/pom.xml b/dbdriver/dbpool/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/dbdriver/hbaseconnection/pom.xml b/dbdriver/hbaseconnection/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/dbdriver/kuduconnection/pom.xml b/dbdriver/kuduconnection/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/dbdriver/phoenix-jdbc/pom.xml b/dbdriver/phoenix-jdbc/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/dbdriver/redispool/pom.xml b/dbdriver/redispool/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/dbdriver/sql4es2/pom.xml b/dbdriver/sql4es2/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/dbdriver/sql4es5/pom.xml b/dbdriver/sql4es5/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/dev/lic/licenses.xml b/dev/lic/licenses.xml
@@ -923,7 +923,7 @@
     <dependency>
       <groupId>edp.wormhole</groupId>
       <artifactId>wormhole-es-hadoop</artifactId>
-      <version>0.6.2</version>
+      <version>0.6.3</version>
       <licenses>
         <license>
           <name>Apache License, Version 2.0</name>
@@ -1032,7 +1032,7 @@
     <dependency>
       <groupId>edp.wormhole</groupId>
       <artifactId>wormhole-util</artifactId>
-      <version>0.6.2</version>
+      <version>0.6.3</version>
       <licenses>
         <license>
           <name>Apache License, Version 2.0</name>
@@ -1080,7 +1080,7 @@
     <dependency>
       <groupId>edp.wormhole</groupId>
       <artifactId>wormhole-kafka-010</artifactId>
-      <version>0.6.2</version>
+      <version>0.6.3</version>
       <licenses>
         <license>
           <name>Apache License, Version 2.0</name>
@@ -1699,7 +1699,7 @@
     <dependency>
       <groupId>org.apache.twill</groupId>
       <artifactId>twill-zookeeper</artifactId>
-      <version>0.6.2-incubating</version>
+      <version>0.6.3-incubating</version>
       <licenses>
         <license>
           <name>The Apache Software License, Version 2.0</name>
@@ -2389,7 +2389,7 @@
     <dependency>
       <groupId>org.apache.twill</groupId>
       <artifactId>twill-core</artifactId>
-      <version>0.6.2-incubating</version>
+      <version>0.6.3-incubating</version>
       <licenses>
         <license>
           <name>The Apache Software License, Version 2.0</name>
@@ -2487,7 +2487,7 @@
     <dependency>
       <groupId>edp.wormhole</groupId>
       <artifactId>wormhole-zookeeper</artifactId>
-      <version>0.6.2</version>
+      <version>0.6.3</version>
       <licenses>
         <license>
           <name>Apache License, Version 2.0</name>
@@ -2815,7 +2815,7 @@
     <dependency>
       <groupId>edp.wormhole</groupId>
       <artifactId>wormhole-dbpool</artifactId>
-      <version>0.6.2</version>
+      <version>0.6.3</version>
       <licenses>
         <license>
           <name>Apache License, Version 2.0</name>
@@ -2855,7 +2855,7 @@
     <dependency>
       <groupId>edp.wormhole</groupId>
       <artifactId>wormhole-rider-server_1.2</artifactId>
-      <version>0.6.2</version>
+      <version>0.6.3</version>
       <licenses>
         <license>
           <name>Apache License, Version 2.0</name>
@@ -3047,7 +3047,7 @@
     <dependency>
       <groupId>org.apache.twill</groupId>
       <artifactId>twill-api</artifactId>
-      <version>0.6.2-incubating</version>
+      <version>0.6.3-incubating</version>
       <licenses>
         <license>
           <name>The Apache Software License, Version 2.0</name>
@@ -3440,7 +3440,7 @@
     <dependency>
       <groupId>org.apache.twill</groupId>
       <artifactId>twill-common</artifactId>
-      <version>0.6.2-incubating</version>
+      <version>0.6.3-incubating</version>
       <licenses>
         <license>
           <name>The Apache Software License, Version 2.0</name>
@@ -3706,7 +3706,7 @@
     <dependency>
       <groupId>edp.wormhole</groupId>
       <artifactId>wormhole-hadoop</artifactId>
-      <version>0.6.2</version>
+      <version>0.6.3</version>
       <licenses>
         <license>
           <name>Apache License, Version 2.0</name>
@@ -3830,7 +3830,7 @@
     <dependency>
       <groupId>org.apache.twill</groupId>
       <artifactId>twill-discovery-api</artifactId>
-      <version>0.6.2-incubating</version>
+      <version>0.6.3-incubating</version>
       <licenses>
         <license>
           <name>The Apache Software License, Version 2.0</name>
@@ -4177,7 +4177,7 @@
     <dependency>
       <groupId>org.apache.twill</groupId>
       <artifactId>twill-discovery-core</artifactId>
-      <version>0.6.2-incubating</version>
+      <version>0.6.3-incubating</version>
       <licenses>
         <license>
           <name>The Apache Software License, Version 2.0</name>
@@ -4418,7 +4418,7 @@
     <dependency>
       <groupId>edp.wormhole</groupId>
       <artifactId>wormhole-ums_1.3</artifactId>
-      <version>0.6.2</version>
+      <version>0.6.3</version>
       <licenses>
         <license>
           <name>Apache License, Version 2.0</name>

diff --git a/docs/_config.yml b/docs/_config.yml
@@ -12,6 +12,6 @@ include:
   - _static
   - _modules
 
-WH_VERSION_SHORT: 0.6.2
+WH_VERSION_SHORT: 0.6.3
 SCALA_BINARY_VERSION: "${scala.binary.version}"
 SCALA_VERSION: "${scala.binary.version}.8"
diff --git a/docs/deployment.md b/docs/deployment.md
@@ -32,11 +32,11 @@ mysql-connector-java-{your-db-version}.jar
 
 #### 下载安装包
 
-**下载 wormhole-0.6.2.tar.gz 包 (链接：https://pan.baidu.com/s/1womXy7Ylr1oaO_u3VqeT4g  提取码：nthu)，或者自编译**
+**下载 wormhole-0.6.3.tar.gz 包 (链接：https://pan.baidu.com/s/1womXy7Ylr1oaO_u3VqeT4g  提取码：nthu)，或者自编译**
 
 ```
-下载wormhole-0.6.2.tar.gz安装包
-tar -xvf wormhole-0.6.2.tar.gz
+下载wormhole-0.6.3.tar.gz安装包
+tar -xvf wormhole-0.6.3.tar.gz
 或者自编译，生成的tar包在 wormhole/target
 git clone -b 0.6 https://github.com/edp963/wormhole.git
 cd wormhole

diff --git a/docs/faq.md b/docs/faq.md
@@ -9,6 +9,10 @@ description: Wormhole Concept page
 {:toc}
 ### Stream 
 
+#### Stream启动时提示resource is not enough
+
+Stream启动时提示resource is not enough，说明project分配的资源不足，需要admin用户登陆后，增大project分配的memory和cpu资源
+
 #### Spark Stream 一直处于 starting 状态
 
 1. 在页面上查看stream日志，根据日志上的错误信息排查问题，一般是spark配置、目录权限、用户权限等问题。如果是权限问题，请按照部署文档说明执行deploy.sh脚本或根据提示手动修复。
@@ -29,6 +33,8 @@ description: Wormhole Concept page
 
 5. 重启wormhole服务，重启stream。
 
+6. 如果yarn上该stream为running状态，wormhole页面显示仍为starting状态，可检查yarn集群的时钟和wormhole部署机器的时钟是否同步，如果时钟不同步，状态更新可能会存在问题。
+
 **若以上步骤仍不能解决问题，请及时反馈~~**
 
 **注意: Flow suspending状态代表挂起状态，标识Flow信息已注册到Stream中，Stream目前处于非running状态。Stream状态正常后Flow状态会自动切换到running或failed状态。具体请查看Stream/Flow部分文档。**
@@ -37,7 +43,7 @@ description: Wormhole Concept page
 #### CDH版Spark消费Kafka报错
 
 1. 错误信息
- 
+
   ```
   Exception in thread "streaming-start" java.lang.NoSuchMethodError: org.apache.kafka.clients.consumer.KafkaConsumer.subscribe(Ljava/util/Collection;)V
 	at org.apache.spark.streaming.kafka010.Subscribe.onStart(ConsumerStrategy.scala:85)
@@ -92,7 +98,7 @@ description: Wormhole Concept page
 
 ### Flow
 
-#### Flow running状态，sink端接收不到数据
+#### Spark Flow running状态，sink端接收不到数据
 
 1. 检查Yarn上对应Stream的driver/executor日志，看有没有错误信息。
 
@@ -102,5 +108,18 @@ description: Wormhole Concept page
 
 4. flow transformation配置后可选择在日志上sample show几条数据，查看是否因为逻辑问题导致。
 
-
+   备注：sink的数据库需要创建ums_id_ （long型），ums_active_ （int型）和ums_ts_（datetime型）三个系统字段，用来做幂等用。 
+
+#### Flink Flow一直处于starting状态
+
+1. 检查Yarn上对应Stream的job日志，看有没有错误信息，检查是否有task生成，当没有有效数据的时候，不会形成task，flow的状态会一直为starting
+2. 检查是否有有效数据，查看flow消费的offset范围，检查offset范围内的测试数据消息key是否与flow sourcenamespace匹配。若flow sourcenamespace为`kafka.kafka01.source.user.*.*.*`，则生成kafka消息时，key应设置为`data_increment_data.kafka.kafka01.source.user.*.*.*`。key和namespace匹配时数据才为有效数据
+
+### Job
+
+#### Job创建时，version选择失败
 
+1. 检查wormhole application.log中是否有异常
+2. 检查wormhole部署的机器是否配置了HADOOP_HOME环境变量
+3. 检查hdfs上是否有该namespace对应的数据
+4. 检查hdfs上该namespace对应的数据，第五位是否为数字
diff --git a/docs/user-guide.md b/docs/user-guide.md
@@ -201,8 +201,8 @@ Wormhole 0.6.1及之后版本支持用户自定义sink
 
 - 替换线上包
 
-- - 如果使用的是sparkx，将生成的wormhole/sparkx/target目录下的wormhole-ums_1.3-sparkx_2.2-0.6.2-jar-with-dependencies替换到线上wormhole app/目录下的该文件
-  - 如果使用的是flinkx，则将wormhole/flinkx/target目录下wormhole-ums_1.3-flinkx_1.5.1-0.6.2-jar-with-dependencies替换线上文件
+- - 如果使用的是sparkx，将生成的wormhole/sparkx/target目录下的wormhole-ums_1.3-sparkx_2.2-0.6.3-jar-with-dependencies替换到线上wormhole app/目录下的该文件
+  - 如果使用的是flinkx，则将wormhole/flinkx/target目录下wormhole-ums_1.3-flinkx_1.5.1-0.6.3-jar-with-dependencies替换线上文件
 
 （2）在用户项目中建立customer sink class流程
 
@@ -222,7 +222,7 @@ Wormhole 0.6.1及之后版本支持用户自定义sink
 
      <artifactId>wormhole-sinks</artifactId>
 
-     <version>0.6.2</version>
+     <version>0.6.3</version>
 
   </dependency>
 
@@ -234,7 +234,7 @@ Wormhole 0.6.1及之后版本支持用户自定义sink
 
             <artifactId>wormhole-ums_1.3-flinkx_1.5.1</artifactId>
 
-            <version>0.6.2</version>
+            <version>0.6.3</version>
 
  </dependency>
 
@@ -263,7 +263,7 @@ Wormhole 0.6.1及之后版本支持用户自定义sink
   <dependency>
      <groupId>edp.wormhole</groupId>
      <artifactId>wormhole-sparkxinterface</artifactId>
-     <version>0.6.2</version>
+     <version>0.6.3</version>
   </dependency>
   ```
 
@@ -440,7 +440,7 @@ Java程序：
   <dependency>
      <groupId>edp.wormhole</groupId>
      <artifactId>wormhole-flinkxinterface</artifactId>
-     <version>0.6.2</version>
+     <version>0.6.3</version>
   </dependency>
   ```
 
@@ -613,6 +613,8 @@ Flink中通过Transformation Config可选择对流处理中异常信息的处理
 
 首先使用 hdfslog Stream 将源数据备份到 Hdfs，Flow 出错或需要重算时，可配置 Job 重算。具体配置可参考Stream 和 Flow。Job中source端可选择数据的版本信息，将该版本的数据重算。
 
+Job配置中version为namespace的第五层，表示数据的版本。使用 hdfslog Stream 将源数据备份到 Hdfs时，改层需要为数字，配置job时，可根据不同的版本进行数据重算。
+
 Job中Spark SQL表名为“increment”。例如：
 
 `select key, value from increment;`

diff --git a/externalclient/hadoop/LICENSE-3RD-PARTY b/externalclient/hadoop/LICENSE-3RD-PARTY
@@ -69,7 +69,7 @@ List of third-party dependencies grouped by their license type.
         * Jetty Server (org.mortbay.jetty:jetty:6.1.26 - http://www.eclipse.org/jetty/jetty-parent/project/modules/jetty)
         * Jetty Utilities (org.mortbay.jetty:jetty-util:6.1.26 - http://www.eclipse.org/jetty/jetty-parent/project/jetty-util)
         * scalatest (org.scalatest:scalatest_2.11:2.2.5 - http://www.scalatest.org)
-        * Snappy for Java (org.xerial.snappy:snappy-java:1.0.6.2 - http://code.google.com/p/snappy-java/)
+        * Snappy for Java (org.xerial.snappy:snappy-java:1.0.6.3 - http://code.google.com/p/snappy-java/)
         * Xerces2 Java Parser (xerces:xercesImpl:2.9.1 - http://xerces.apache.org/xerces2-j)
         * XML Commons External Components XML APIs (xml-apis:xml-apis:1.3.04 - http://xml.apache.org/commons/components/external/)
 

diff --git a/externalclient/hadoop/pom.xml b/externalclient/hadoop/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/externalclient/zookeeper/pom.xml b/externalclient/zookeeper/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>edp.wormhole</groupId>
         <artifactId>wormhole-parent</artifactId>
-        <version>0.6.2</version>
+        <version>0.6.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>