diff --git a/common/pom.xml b/common/pom.xml index 715c5882e..1f3981145 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../pom.xml diff --git a/dbdriver/dbpool/LICENSE-3RD-PARTY b/dbdriver/dbpool/LICENSE-3RD-PARTY index 3c9ef64c0..9ab37602c 100644 --- a/dbdriver/dbpool/LICENSE-3RD-PARTY +++ b/dbdriver/dbpool/LICENSE-3RD-PARTY @@ -35,7 +35,7 @@ List of third-party dependencies grouped by their license type. * Lang (commons-lang:commons-lang:2.3 - http://jakarta.apache.org/commons/lang/) * Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/) * Commons Net (commons-net:commons-net:3.1 - http://commons.apache.org/net/) - * wormhole-util (edp.wormhole:wormhole-util:0.6.2 - no url defined) + * wormhole-util (edp.wormhole:wormhole-util:0.6.3 - no url defined) * Metrics Core (io.dropwizard.metrics:metrics-core:3.1.2 - http://metrics.codahale.com/metrics-core/) * The Netty Project (io.netty:netty:3.6.2.Final - http://netty.io/) * Netty/All-in-One (io.netty:netty-all:4.0.23.Final - http://netty.io/netty-all/) @@ -91,12 +91,12 @@ List of third-party dependencies grouped by their license type. * Phoenix Query Server (org.apache.phoenix:phoenix-server:4.7.0-HBase-1.1 - http://www.apache.org/phoenix/phoenix-server/) * Phoenix Query Server Client (org.apache.phoenix:phoenix-server-client:4.7.0-HBase-1.1 - http://www.apache.org/phoenix/phoenix-server-client/) * Apache Thrift (org.apache.thrift:libthrift:0.9.0 - http://thrift.apache.org) - * Apache Twill API (org.apache.twill:twill-api:0.6.2-incubating - http://twill.incubator.apache.org/twill-api) - * Apache Twill common library (org.apache.twill:twill-common:0.6.2-incubating - http://twill.incubator.apache.org/twill-common) - * Apache Twill core library (org.apache.twill:twill-core:0.6.2-incubating - http://twill.incubator.apache.org/twill-core) - * Apache Twill discovery service API (org.apache.twill:twill-discovery-api:0.6.2-incubating - http://twill.incubator.apache.org/twill-discovery-api) - * Apache Twill discovery service implementations (org.apache.twill:twill-discovery-core:0.6.2-incubating - http://twill.incubator.apache.org/twill-discovery-core) - * Apache Twill ZooKeeper client library (org.apache.twill:twill-zookeeper:0.6.2-incubating - http://twill.incubator.apache.org/twill-zookeeper) + * Apache Twill API (org.apache.twill:twill-api:0.6.3-incubating - http://twill.incubator.apache.org/twill-api) + * Apache Twill common library (org.apache.twill:twill-common:0.6.3-incubating - http://twill.incubator.apache.org/twill-common) + * Apache Twill core library (org.apache.twill:twill-core:0.6.3-incubating - http://twill.incubator.apache.org/twill-core) + * Apache Twill discovery service API (org.apache.twill:twill-discovery-api:0.6.3-incubating - http://twill.incubator.apache.org/twill-discovery-api) + * Apache Twill discovery service implementations (org.apache.twill:twill-discovery-core:0.6.3-incubating - http://twill.incubator.apache.org/twill-discovery-core) + * Apache Twill ZooKeeper client library (org.apache.twill:twill-zookeeper:0.6.3-incubating - http://twill.incubator.apache.org/twill-zookeeper) * zookeeper (org.apache.zookeeper:zookeeper:3.4.6 - no url defined) * Jackson (org.codehaus.jackson:jackson-core-asl:1.9.2 - http://jackson.codehaus.org) * JAX-RS provider for JSON content type (org.codehaus.jackson:jackson-jaxrs:1.9.13 - http://jackson.codehaus.org) diff --git a/dbdriver/dbpool/pom.xml b/dbdriver/dbpool/pom.xml index bb6fb6894..2cea04e8f 100644 --- a/dbdriver/dbpool/pom.xml +++ b/dbdriver/dbpool/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/dbdriver/hbaseconnection/pom.xml b/dbdriver/hbaseconnection/pom.xml index bd74f8f6e..37a3e68bd 100644 --- a/dbdriver/hbaseconnection/pom.xml +++ b/dbdriver/hbaseconnection/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/dbdriver/kuduconnection/pom.xml b/dbdriver/kuduconnection/pom.xml index 893515e45..c05ebb074 100644 --- a/dbdriver/kuduconnection/pom.xml +++ b/dbdriver/kuduconnection/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/dbdriver/phoenix-jdbc/pom.xml b/dbdriver/phoenix-jdbc/pom.xml index 959116c5e..ee51f6cfd 100644 --- a/dbdriver/phoenix-jdbc/pom.xml +++ b/dbdriver/phoenix-jdbc/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/dbdriver/redispool/pom.xml b/dbdriver/redispool/pom.xml index df469d37c..d2ff22ea0 100644 --- a/dbdriver/redispool/pom.xml +++ b/dbdriver/redispool/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/dbdriver/sql4es2/pom.xml b/dbdriver/sql4es2/pom.xml index 78d4d9e99..ccf850ca2 100644 --- a/dbdriver/sql4es2/pom.xml +++ b/dbdriver/sql4es2/pom.xml @@ -4,7 +4,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/dbdriver/sql4es5/pom.xml b/dbdriver/sql4es5/pom.xml index 5c9ba75af..193063c35 100644 --- a/dbdriver/sql4es5/pom.xml +++ b/dbdriver/sql4es5/pom.xml @@ -4,7 +4,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/dev/lic/licenses.xml b/dev/lic/licenses.xml index c7c294b34..e9a503f29 100644 --- a/dev/lic/licenses.xml +++ b/dev/lic/licenses.xml @@ -923,7 +923,7 @@ edp.wormhole wormhole-es-hadoop - 0.6.2 + 0.6.3 Apache License, Version 2.0 @@ -1032,7 +1032,7 @@ edp.wormhole wormhole-util - 0.6.2 + 0.6.3 Apache License, Version 2.0 @@ -1080,7 +1080,7 @@ edp.wormhole wormhole-kafka-010 - 0.6.2 + 0.6.3 Apache License, Version 2.0 @@ -1699,7 +1699,7 @@ org.apache.twill twill-zookeeper - 0.6.2-incubating + 0.6.3-incubating The Apache Software License, Version 2.0 @@ -2389,7 +2389,7 @@ org.apache.twill twill-core - 0.6.2-incubating + 0.6.3-incubating The Apache Software License, Version 2.0 @@ -2487,7 +2487,7 @@ edp.wormhole wormhole-zookeeper - 0.6.2 + 0.6.3 Apache License, Version 2.0 @@ -2815,7 +2815,7 @@ edp.wormhole wormhole-dbpool - 0.6.2 + 0.6.3 Apache License, Version 2.0 @@ -2855,7 +2855,7 @@ edp.wormhole wormhole-rider-server_1.2 - 0.6.2 + 0.6.3 Apache License, Version 2.0 @@ -3047,7 +3047,7 @@ org.apache.twill twill-api - 0.6.2-incubating + 0.6.3-incubating The Apache Software License, Version 2.0 @@ -3440,7 +3440,7 @@ org.apache.twill twill-common - 0.6.2-incubating + 0.6.3-incubating The Apache Software License, Version 2.0 @@ -3706,7 +3706,7 @@ edp.wormhole wormhole-hadoop - 0.6.2 + 0.6.3 Apache License, Version 2.0 @@ -3830,7 +3830,7 @@ org.apache.twill twill-discovery-api - 0.6.2-incubating + 0.6.3-incubating The Apache Software License, Version 2.0 @@ -4177,7 +4177,7 @@ org.apache.twill twill-discovery-core - 0.6.2-incubating + 0.6.3-incubating The Apache Software License, Version 2.0 @@ -4418,7 +4418,7 @@ edp.wormhole wormhole-ums_1.3 - 0.6.2 + 0.6.3 Apache License, Version 2.0 diff --git a/docs/_config.yml b/docs/_config.yml index c4f923491..bbe8ae530 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -12,6 +12,6 @@ include: - _static - _modules -WH_VERSION_SHORT: 0.6.2 +WH_VERSION_SHORT: 0.6.3 SCALA_BINARY_VERSION: "${scala.binary.version}" SCALA_VERSION: "${scala.binary.version}.8" diff --git a/docs/deployment.md b/docs/deployment.md index cb1b3ad13..7e4e69268 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -32,11 +32,11 @@ mysql-connector-java-{your-db-version}.jar #### 下载安装包 -**下载 wormhole-0.6.2.tar.gz 包 (链接：https://pan.baidu.com/s/1womXy7Ylr1oaO_u3VqeT4g 提取码：nthu)，或者自编译** +**下载 wormhole-0.6.3.tar.gz 包 (链接：https://pan.baidu.com/s/1womXy7Ylr1oaO_u3VqeT4g 提取码：nthu)，或者自编译** ``` -下载wormhole-0.6.2.tar.gz安装包 -tar -xvf wormhole-0.6.2.tar.gz +下载wormhole-0.6.3.tar.gz安装包 +tar -xvf wormhole-0.6.3.tar.gz 或者自编译，生成的tar包在 wormhole/target git clone -b 0.6 https://github.com/edp963/wormhole.git cd wormhole diff --git a/docs/faq.md b/docs/faq.md index 50d4f80fe..4f0454c02 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -9,6 +9,10 @@ description: Wormhole Concept page {:toc} ### Stream +#### Stream启动时提示resource is not enough + +Stream启动时提示resource is not enough，说明project分配的资源不足，需要admin用户登陆后，增大project分配的memory和cpu资源 + #### Spark Stream 一直处于 starting 状态 1. 在页面上查看stream日志，根据日志上的错误信息排查问题，一般是spark配置、目录权限、用户权限等问题。如果是权限问题，请按照部署文档说明执行deploy.sh脚本或根据提示手动修复。 @@ -29,6 +33,8 @@ description: Wormhole Concept page 5. 重启wormhole服务，重启stream。 +6. 如果yarn上该stream为running状态，wormhole页面显示仍为starting状态，可检查yarn集群的时钟和wormhole部署机器的时钟是否同步，如果时钟不同步，状态更新可能会存在问题。 + **若以上步骤仍不能解决问题，请及时反馈~~** **注意: Flow suspending状态代表挂起状态，标识Flow信息已注册到Stream中，Stream目前处于非running状态。Stream状态正常后Flow状态会自动切换到running或failed状态。具体请查看Stream/Flow部分文档。** @@ -37,7 +43,7 @@ description: Wormhole Concept page #### CDH版Spark消费Kafka报错 1. 错误信息 - + ``` Exception in thread "streaming-start" java.lang.NoSuchMethodError: org.apache.kafka.clients.consumer.KafkaConsumer.subscribe(Ljava/util/Collection;)V at org.apache.spark.streaming.kafka010.Subscribe.onStart(ConsumerStrategy.scala:85) @@ -92,7 +98,7 @@ description: Wormhole Concept page ### Flow -#### Flow running状态，sink端接收不到数据 +#### Spark Flow running状态，sink端接收不到数据 1. 检查Yarn上对应Stream的driver/executor日志，看有没有错误信息。 @@ -102,5 +108,18 @@ description: Wormhole Concept page 4. flow transformation配置后可选择在日志上sample show几条数据，查看是否因为逻辑问题导致。 - + 备注：sink的数据库需要创建ums_id_ （long型），ums_active_ （int型）和ums_ts_（datetime型）三个系统字段，用来做幂等用。 + +#### Flink Flow一直处于starting状态 + +1. 检查Yarn上对应Stream的job日志，看有没有错误信息，检查是否有task生成，当没有有效数据的时候，不会形成task，flow的状态会一直为starting +2. 检查是否有有效数据，查看flow消费的offset范围，检查offset范围内的测试数据消息key是否与flow sourcenamespace匹配。若flow sourcenamespace为`kafka.kafka01.source.user.*.*.*`，则生成kafka消息时，key应设置为`data_increment_data.kafka.kafka01.source.user.*.*.*`。key和namespace匹配时数据才为有效数据 + +### Job + +#### Job创建时，version选择失败 +1. 检查wormhole application.log中是否有异常 +2. 检查wormhole部署的机器是否配置了HADOOP_HOME环境变量 +3. 检查hdfs上是否有该namespace对应的数据 +4. 检查hdfs上该namespace对应的数据，第五位是否为数字 \ No newline at end of file diff --git a/docs/user-guide.md b/docs/user-guide.md index b082d9a83..6f41a0e14 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -201,8 +201,8 @@ Wormhole 0.6.1及之后版本支持用户自定义sink - 替换线上包 -- - 如果使用的是sparkx，将生成的wormhole/sparkx/target目录下的wormhole-ums_1.3-sparkx_2.2-0.6.2-jar-with-dependencies替换到线上wormhole app/目录下的该文件 - - 如果使用的是flinkx，则将wormhole/flinkx/target目录下wormhole-ums_1.3-flinkx_1.5.1-0.6.2-jar-with-dependencies替换线上文件 +- - 如果使用的是sparkx，将生成的wormhole/sparkx/target目录下的wormhole-ums_1.3-sparkx_2.2-0.6.3-jar-with-dependencies替换到线上wormhole app/目录下的该文件 + - 如果使用的是flinkx，则将wormhole/flinkx/target目录下wormhole-ums_1.3-flinkx_1.5.1-0.6.3-jar-with-dependencies替换线上文件 （2）在用户项目中建立customer sink class流程 @@ -222,7 +222,7 @@ Wormhole 0.6.1及之后版本支持用户自定义sink ​ wormhole-sinks -​ 0.6.2 +​ 0.6.3 @@ -234,7 +234,7 @@ Wormhole 0.6.1及之后版本支持用户自定义sink ​ wormhole-ums_1.3-flinkx_1.5.1 -​ 0.6.2 +​ 0.6.3 @@ -263,7 +263,7 @@ Wormhole 0.6.1及之后版本支持用户自定义sink edp.wormhole wormhole-sparkxinterface - 0.6.2 + 0.6.3 ``` @@ -440,7 +440,7 @@ Java程序： edp.wormhole wormhole-flinkxinterface - 0.6.2 + 0.6.3 ``` @@ -613,6 +613,8 @@ Flink中通过Transformation Config可选择对流处理中异常信息的处理 首先使用 hdfslog Stream 将源数据备份到 Hdfs，Flow 出错或需要重算时，可配置 Job 重算。具体配置可参考Stream 和 Flow。Job中source端可选择数据的版本信息，将该版本的数据重算。 +Job配置中version为namespace的第五层，表示数据的版本。使用 hdfslog Stream 将源数据备份到 Hdfs时，改层需要为数字，配置job时，可根据不同的版本进行数据重算。 + Job中Spark SQL表名为“increment”。例如： `select key, value from increment;` diff --git a/externalclient/hadoop/LICENSE-3RD-PARTY b/externalclient/hadoop/LICENSE-3RD-PARTY index 361d9ab67..f84140d8c 100644 --- a/externalclient/hadoop/LICENSE-3RD-PARTY +++ b/externalclient/hadoop/LICENSE-3RD-PARTY @@ -69,7 +69,7 @@ List of third-party dependencies grouped by their license type. * Jetty Server (org.mortbay.jetty:jetty:6.1.26 - http://www.eclipse.org/jetty/jetty-parent/project/modules/jetty) * Jetty Utilities (org.mortbay.jetty:jetty-util:6.1.26 - http://www.eclipse.org/jetty/jetty-parent/project/jetty-util) * scalatest (org.scalatest:scalatest_2.11:2.2.5 - http://www.scalatest.org) - * Snappy for Java (org.xerial.snappy:snappy-java:1.0.6.2 - http://code.google.com/p/snappy-java/) + * Snappy for Java (org.xerial.snappy:snappy-java:1.0.6.3 - http://code.google.com/p/snappy-java/) * Xerces2 Java Parser (xerces:xercesImpl:2.9.1 - http://xerces.apache.org/xerces2-j) * XML Commons External Components XML APIs (xml-apis:xml-apis:1.3.04 - http://xml.apache.org/commons/components/external/) diff --git a/externalclient/hadoop/pom.xml b/externalclient/hadoop/pom.xml index 0bdf1324e..ff6813178 100644 --- a/externalclient/hadoop/pom.xml +++ b/externalclient/hadoop/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/externalclient/zookeeper/pom.xml b/externalclient/zookeeper/pom.xml index 872dcc351..9f3bcac3f 100644 --- a/externalclient/zookeeper/pom.xml +++ b/externalclient/zookeeper/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/flinkx/LICENSE-3RD-PARTY b/flinkx/LICENSE-3RD-PARTY index 27f2c8d92..48c939fa7 100644 --- a/flinkx/LICENSE-3RD-PARTY +++ b/flinkx/LICENSE-3RD-PARTY @@ -18,13 +18,13 @@ List of third-party dependencies grouped by their license type. * Metrics Core Library (com.yammer.metrics:metrics-core:2.2.0 - http://metrics.codahale.com/metrics-core/) * Apache Commons CLI (commons-cli:commons-cli:1.3.1 - http://commons.apache.org/proper/commons-cli/) * Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/) - * wormhole-dbpool (edp.wormhole:wormhole-dbpool:0.6.2 - no url defined) - * wormhole-flinkxinterface (edp.wormhole:wormhole-flinkxinterface:0.6.2 - no url defined) - * wormhole-interface (edp.wormhole:wormhole-interface:0.6.2 - no url defined) - * wormhole-kafka-010 (edp.wormhole:wormhole-kafka-010:0.6.2 - no url defined) - * wormhole-ums_1.3 (edp.wormhole:wormhole-ums_1.3:0.6.2 - no url defined) - * wormhole-util (edp.wormhole:wormhole-util:0.6.2 - no url defined) - * wormhole-zookeeper (edp.wormhole:wormhole-zookeeper:0.6.2 - no url defined) + * wormhole-dbpool (edp.wormhole:wormhole-dbpool:0.6.3 - no url defined) + * wormhole-flinkxinterface (edp.wormhole:wormhole-flinkxinterface:0.6.3 - no url defined) + * wormhole-interface (edp.wormhole:wormhole-interface:0.6.3 - no url defined) + * wormhole-kafka-010 (edp.wormhole:wormhole-kafka-010:0.6.3 - no url defined) + * wormhole-ums_1.3 (edp.wormhole:wormhole-ums_1.3:0.6.3 - no url defined) + * wormhole-util (edp.wormhole:wormhole-util:0.6.3 - no url defined) + * wormhole-zookeeper (edp.wormhole:wormhole-zookeeper:0.6.3 - no url defined) * The Netty Project (io.netty:netty:3.7.0.Final - http://netty.io/) * force-shading (org.apache.flink:force-shading:1.5.1 - https://www.apache.org/force-shading/) * Javassist (org.javassist:javassist:3.18.2-GA - http://www.javassist.org/) diff --git a/flinkx/pom.xml b/flinkx/pom.xml index 94bde9e13..3f0d29e3d 100644 --- a/flinkx/pom.xml +++ b/flinkx/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../pom.xml diff --git a/interface/flinkxinterface/pom.xml b/interface/flinkxinterface/pom.xml index 0497b7795..8d156adb7 100644 --- a/interface/flinkxinterface/pom.xml +++ b/interface/flinkxinterface/pom.xml @@ -6,7 +6,7 @@ wormhole-parent edp.wormhole - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/interface/publicinterface/pom.xml b/interface/publicinterface/pom.xml index a4b701c5b..ba390ac1e 100644 --- a/interface/publicinterface/pom.xml +++ b/interface/publicinterface/pom.xml @@ -6,7 +6,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/interface/sparkxinterface/pom.xml b/interface/sparkxinterface/pom.xml index fbbc63d78..c0149222d 100644 --- a/interface/sparkxinterface/pom.xml +++ b/interface/sparkxinterface/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/kafkatool/kafka010-2/pom.xml b/kafkatool/kafka010-2/pom.xml index 8608bd33e..c8139f335 100644 --- a/kafkatool/kafka010-2/pom.xml +++ b/kafkatool/kafka010-2/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/kafkatool/kafka010/LICENSE-3RD-PARTY b/kafkatool/kafka010/LICENSE-3RD-PARTY index 1efa78956..8d84b4d65 100644 --- a/kafkatool/kafka010/LICENSE-3RD-PARTY +++ b/kafkatool/kafka010/LICENSE-3RD-PARTY @@ -9,7 +9,7 @@ List of third-party dependencies grouped by their license type. * Jackson-core (com.fasterxml.jackson.core:jackson-core:2.3.1 - http://wiki.fasterxml.com/JacksonHome) * jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.3.1 - http://wiki.fasterxml.com/JacksonHome) * Metrics Core Library (com.yammer.metrics:metrics-core:2.2.0 - http://metrics.codahale.com/metrics-core/) - * wormhole-util (edp.wormhole:wormhole-util:0.6.2 - no url defined) + * wormhole-util (edp.wormhole:wormhole-util:0.6.3 - no url defined) * The Netty Project (io.netty:netty:3.7.0.Final - http://netty.io/) * Joda-Time (joda-time:joda-time:2.9.8 - http://www.joda.org/joda-time/) * Apache Log4j (log4j:log4j:1.2.17 - http://logging.apache.org/log4j/1.2/) diff --git a/kafkatool/kafka010/pom.xml b/kafkatool/kafka010/pom.xml index cd16cd41d..f205cc2ad 100644 --- a/kafkatool/kafka010/pom.xml +++ b/kafkatool/kafka010/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/kafkatool/kafka08/LICENSE-3RD-PARTY b/kafkatool/kafka08/LICENSE-3RD-PARTY index 2e36cf1ae..0926d954c 100644 --- a/kafkatool/kafka08/LICENSE-3RD-PARTY +++ b/kafkatool/kafka08/LICENSE-3RD-PARTY @@ -9,7 +9,7 @@ List of third-party dependencies grouped by their license type. * Jackson-core (com.fasterxml.jackson.core:jackson-core:2.3.1 - http://wiki.fasterxml.com/JacksonHome) * jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.3.1 - http://wiki.fasterxml.com/JacksonHome) * Metrics Core Library (com.yammer.metrics:metrics-core:2.2.0 - http://metrics.codahale.com/metrics-core/) - * wormhole-util (edp.wormhole:wormhole-util:0.6.2 - no url defined) + * wormhole-util (edp.wormhole:wormhole-util:0.6.3 - no url defined) * The Netty Project (io.netty:netty:3.7.0.Final - http://netty.io/) * Joda-Time (joda-time:joda-time:2.9.8 - http://www.joda.org/joda-time/) * Apache Log4j (log4j:log4j:1.2.17 - http://logging.apache.org/log4j/1.2/) diff --git a/kafkatool/kafka08/pom.xml b/kafkatool/kafka08/pom.xml index 7722a44a3..c21f34857 100644 --- a/kafkatool/kafka08/pom.xml +++ b/kafkatool/kafka08/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/pom.xml b/pom.xml index 9f972bc83..8259c8c33 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 edp.wormhole - 0.6.2 + 0.6.3 wormhole-parent pom @@ -59,7 +59,7 @@ 2.3.3 2.4.3 1.3 - 0.6.2 + 0.6.3 UTF-8 UTF-8 1.8 @@ -233,7 +233,7 @@ <!––> <!–com.holdenkarau–> <!–spark-testing-base_2.11–> - <!–2.0.0-preview_0.6.2-preview–> + <!–2.0.0-preview_0.6.3-preview–> <!––> <!– ======= flink ========= –> diff --git a/reporter/pom.xml b/reporter/pom.xml index 70ae7cf64..966fe92e4 100644 --- a/reporter/pom.xml +++ b/reporter/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../pom.xml diff --git a/rider/bin/start.sh b/rider/bin/start.sh index c656e859d..574ce499c 100644 --- a/rider/bin/start.sh +++ b/rider/bin/start.sh @@ -16,4 +16,4 @@ PORT=`grep port $WORMHOLE_HOME/conf/application.conf | head -1 | cut -d = -f2 | echo "wormholeServer port: $PORT" echo "[WormholeServer] is starting..." -java -Xmx4G -DWORMHOLE_HOME=$WORMHOLE_HOME -cp $WORMHOLE_HOME/lib/wormhole-rider-server_1.3-0.6.2.jar:$WORMHOLE_HOME/lib/* edp.rider.RiderStarter & +java -Xmx4G -DWORMHOLE_HOME=$WORMHOLE_HOME -cp $WORMHOLE_HOME/lib/wormhole-rider-server_1.3-0.6.3.jar:$WORMHOLE_HOME/lib/* edp.rider.RiderStarter & diff --git a/rider/rider-assembly/pom.xml b/rider/rider-assembly/pom.xml index e241fdf29..441ffe1fc 100644 --- a/rider/rider-assembly/pom.xml +++ b/rider/rider-assembly/pom.xml @@ -7,12 +7,12 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml wormhole - 0.6.2 + 0.6.3 pom @@ -26,12 +26,12 @@ edp.wormhole wormhole-rider-server_${ums.version} - 0.6.2 + 0.6.3 - + diff --git a/rider/rider-server/LICENSE-3RD-PARTY b/rider/rider-server/LICENSE-3RD-PARTY index e1b9ceaa3..e798e4325 100644 --- a/rider/rider-server/LICENSE-3RD-PARTY +++ b/rider/rider-server/LICENSE-3RD-PARTY @@ -28,10 +28,10 @@ List of third-party dependencies grouped by their license type. * HikariCP (com.zaxxer:HikariCP:2.5.1 - https://github.com/brettwooldridge/HikariCP) * Apache Commons Codec (commons-codec:commons-codec:1.10 - http://commons.apache.org/proper/commons-codec/) * Commons Logging (commons-logging:commons-logging:1.1.3 - http://commons.apache.org/proper/commons-logging/) - * wormhole-kafka-010 (edp.wormhole:wormhole-kafka-010:0.6.2 - no url defined) - * wormhole-ums_1.3 (edp.wormhole:wormhole-ums_1.3:0.6.2 - no url defined) - * wormhole-util (edp.wormhole:wormhole-util:0.6.2 - no url defined) - * wormhole-zookeeper (edp.wormhole:wormhole-zookeeper:0.6.2 - no url defined) + * wormhole-kafka-010 (edp.wormhole:wormhole-kafka-010:0.6.3 - no url defined) + * wormhole-ums_1.3 (edp.wormhole:wormhole-ums_1.3:0.6.3 - no url defined) + * wormhole-util (edp.wormhole:wormhole-util:0.6.3 - no url defined) + * wormhole-zookeeper (edp.wormhole:wormhole-zookeeper:0.6.3 - no url defined) * The Netty Project (io.netty:netty:3.7.0.Final - http://netty.io/) * spray-json (io.spray:spray-json_2.11:1.3.3 - https://github.com/spray/spray-json) * swagger-annotations (io.swagger:swagger-annotations:1.5.12 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) diff --git a/rider/rider-server/pom.xml b/rider/rider-server/pom.xml index eeed01cec..7701c8164 100644 --- a/rider/rider-server/pom.xml +++ b/rider/rider-server/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/rider/rider-server/src/main/scala/edp/rider/common/RiderConfig.scala b/rider/rider-server/src/main/scala/edp/rider/common/RiderConfig.scala index ebd7da643..1c65f4351 100644 --- a/rider/rider-server/src/main/scala/edp/rider/common/RiderConfig.scala +++ b/rider/rider-server/src/main/scala/edp/rider/common/RiderConfig.scala @@ -232,10 +232,10 @@ object RiderConfig { lazy val appTags = getStringConfig("spark.app.tags", "wormhole") lazy val wormholeClientLogPath = getStringConfig("spark.wormhole.client.log.path", s"${RiderConfig.riderRootPath}/logs") - lazy val wormholeJarPath = getStringConfig("spark.wormhole.jar.path", s"${RiderConfig.riderRootPath}/app/wormhole-ums_1.3-sparkx_2.2-0.6.2-jar-with-dependencies.jar") - lazy val wormholeKafka08JarPath = getStringConfig("spark.wormhole.kafka08.jar.path", s"${RiderConfig.riderRootPath}/app/wormhole-ums_1.3-sparkx_2.2-0.6.2-jar-with-dependencies-kafka08.jar") + lazy val wormholeJarPath = getStringConfig("spark.wormhole.jar.path", s"${RiderConfig.riderRootPath}/app/wormhole-ums_1.3-sparkx_2.2-0.6.3-jar-with-dependencies.jar") + lazy val wormholeKafka08JarPath = getStringConfig("spark.wormhole.kafka08.jar.path", s"${RiderConfig.riderRootPath}/app/wormhole-ums_1.3-sparkx_2.2-0.6.3-jar-with-dependencies-kafka08.jar") lazy val kafka08StreamNames = getStringConfig("spark.wormhole.kafka08.streams", "") - lazy val sparkxInterfaceJarPath = getStringConfig("spark.wormhole.sparkxinterface.jar.path", s"${RiderConfig.riderRootPath}/app/wormhole-sparkxinterface-0.6.2.jar") + lazy val sparkxInterfaceJarPath = getStringConfig("spark.wormhole.sparkxinterface.jar.path", s"${RiderConfig.riderRootPath}/app/wormhole-sparkxinterface-0.6.3.jar") lazy val rm1Url = config.getString("spark.yarn.rm1.http.url") lazy val rm2Url = getStringConfig("spark.yarn.rm2.http.url", rm1Url) lazy val kafkaSessionTimeOut = getIntConfig("spark.kafka.session.timeout", 30000) @@ -351,7 +351,7 @@ object RiderConfig { lazy val defaultFlinkConfig = FlinkDefaultConfig("", FlinkResourceConfig(2, 6, 1, 2), "") - lazy val flink = RiderFlink(config.getString("flink.home"), config.getString("flink.yarn.queue.name"), getBooleanConfig("flink.feedback.enabled", false), getIntConfig("flink.feedback.state.count", 100), getIntConfig("flink.feedback.interval", 30), 1, getStringConfig("flink.wormhole.jar.path", s"${RiderConfig.riderRootPath}/app/wormhole-ums_1.3-flinkx_1.7.2-0.6.2-jar-with-dependencies.jar"), getStringConfig("flink.wormhole.client.log.path", s"$riderRootPath/logs/flows"), getIntConfig("spark.kafka.session.timeout", 30000), getIntConfig("spark.kafka.group.max.session.timeout.ms", 60000)) + lazy val flink = RiderFlink(config.getString("flink.home"), config.getString("flink.yarn.queue.name"), getBooleanConfig("flink.feedback.enabled", false), getIntConfig("flink.feedback.state.count", 100), getIntConfig("flink.feedback.interval", 30), 1, getStringConfig("flink.wormhole.jar.path", s"${RiderConfig.riderRootPath}/app/wormhole-ums_1.3-flinkx_1.7.2-0.6.3-jar-with-dependencies.jar"), getStringConfig("flink.wormhole.client.log.path", s"$riderRootPath/logs/flows"), getIntConfig("spark.kafka.session.timeout", 30000), getIntConfig("spark.kafka.group.max.session.timeout.ms", 60000)) // lazy val flinkCheckpoint = FlinkCheckpoint(getBooleanConfig("flink.checkpoint.enable", false), getIntConfig("flink.checkpoint.interval", 60000), getStringConfig("flink.stateBackend", "")) diff --git a/sinks/pom.xml b/sinks/pom.xml index 27ef58220..eed4cd29d 100644 --- a/sinks/pom.xml +++ b/sinks/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../pom.xml diff --git a/sparkextension/spark_extension_2_2/pom.xml b/sparkextension/spark_extension_2_2/pom.xml index 83fe2c9e5..21fbddfab 100644 --- a/sparkextension/spark_extension_2_2/pom.xml +++ b/sparkextension/spark_extension_2_2/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/sparkextension/spark_extension_2_3/pom.xml b/sparkextension/spark_extension_2_3/pom.xml index 33a7aa81a..dd243f559 100644 --- a/sparkextension/spark_extension_2_3/pom.xml +++ b/sparkextension/spark_extension_2_3/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/sparkextension/spark_extension_2_4/pom.xml b/sparkextension/spark_extension_2_4/pom.xml index 966391406..a2d244643 100644 --- a/sparkextension/spark_extension_2_4/pom.xml +++ b/sparkextension/spark_extension_2_4/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../../pom.xml diff --git a/sparkx/LICENSE-3RD-PARTY b/sparkx/LICENSE-3RD-PARTY index 6f990a633..9d456cb5e 100644 --- a/sparkx/LICENSE-3RD-PARTY +++ b/sparkx/LICENSE-3RD-PARTY @@ -43,13 +43,13 @@ List of third-party dependencies grouped by their license type. * Commons Lang (commons-lang:commons-lang:2.6 - http://commons.apache.org/lang/) * Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/) * Commons Net (commons-net:commons-net:2.2 - http://commons.apache.org/net/) - * wormhole-dbpool (edp.wormhole:wormhole-dbpool:0.6.2 - no url defined) - * wormhole-es-hadoop (edp.wormhole:wormhole-es-hadoop:0.6.2 - no url defined) - * wormhole-hadoop (edp.wormhole:wormhole-hadoop:0.6.2 - no url defined) - * wormhole-kafka-010 (edp.wormhole:wormhole-kafka-010:0.6.2 - no url defined) - * wormhole-ums_1.3 (edp.wormhole:wormhole-ums_1.3:0.6.2 - no url defined) - * wormhole-util (edp.wormhole:wormhole-util:0.6.2 - no url defined) - * wormhole-zookeeper (edp.wormhole:wormhole-zookeeper:0.6.2 - no url defined) + * wormhole-dbpool (edp.wormhole:wormhole-dbpool:0.6.3 - no url defined) + * wormhole-es-hadoop (edp.wormhole:wormhole-es-hadoop:0.6.3 - no url defined) + * wormhole-hadoop (edp.wormhole:wormhole-hadoop:0.6.3 - no url defined) + * wormhole-kafka-010 (edp.wormhole:wormhole-kafka-010:0.6.3 - no url defined) + * wormhole-ums_1.3 (edp.wormhole:wormhole-ums_1.3:0.6.3 - no url defined) + * wormhole-util (edp.wormhole:wormhole-util:0.6.3 - no url defined) + * wormhole-zookeeper (edp.wormhole:wormhole-zookeeper:0.6.3 - no url defined) * Metrics Core (io.dropwizard.metrics:metrics-core:3.1.2 - http://metrics.codahale.com/metrics-core/) * Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:3.1.2 - http://metrics.codahale.com/metrics-graphite/) * Jackson Integration for Metrics (io.dropwizard.metrics:metrics-json:3.1.2 - http://metrics.codahale.com/metrics-json/) @@ -148,12 +148,12 @@ List of third-party dependencies grouped by their license type. * Apache Tephra Core (org.apache.tephra:tephra-core:0.9.0-incubating - http://tephra.incubator.apache.org/tephra-core) * Apache Tephra HBase 1.1 Compatibility (org.apache.tephra:tephra-hbase-compat-1.1:0.9.0-incubating - http://tephra.incubator.apache.org/tephra-hbase-compat-1.1-base/tephra-hbase-compat-1.1) * Apache Thrift (org.apache.thrift:libthrift:0.9.0 - http://thrift.apache.org) - * Apache Twill API (org.apache.twill:twill-api:0.6.2-incubating - http://twill.incubator.apache.org/twill-api) - * Apache Twill common library (org.apache.twill:twill-common:0.6.2-incubating - http://twill.incubator.apache.org/twill-common) - * Apache Twill core library (org.apache.twill:twill-core:0.6.2-incubating - http://twill.incubator.apache.org/twill-core) - * Apache Twill discovery service API (org.apache.twill:twill-discovery-api:0.6.2-incubating - http://twill.incubator.apache.org/twill-discovery-api) - * Apache Twill discovery service implementations (org.apache.twill:twill-discovery-core:0.6.2-incubating - http://twill.incubator.apache.org/twill-discovery-core) - * Apache Twill ZooKeeper client library (org.apache.twill:twill-zookeeper:0.6.2-incubating - http://twill.incubator.apache.org/twill-zookeeper) + * Apache Twill API (org.apache.twill:twill-api:0.6.3-incubating - http://twill.incubator.apache.org/twill-api) + * Apache Twill common library (org.apache.twill:twill-common:0.6.3-incubating - http://twill.incubator.apache.org/twill-common) + * Apache Twill core library (org.apache.twill:twill-core:0.6.3-incubating - http://twill.incubator.apache.org/twill-core) + * Apache Twill discovery service API (org.apache.twill:twill-discovery-api:0.6.3-incubating - http://twill.incubator.apache.org/twill-discovery-api) + * Apache Twill discovery service implementations (org.apache.twill:twill-discovery-core:0.6.3-incubating - http://twill.incubator.apache.org/twill-discovery-core) + * Apache Twill ZooKeeper client library (org.apache.twill:twill-zookeeper:0.6.3-incubating - http://twill.incubator.apache.org/twill-zookeeper) * Apache XBean :: ASM 5 shaded (repackaged) (org.apache.xbean:xbean-asm5-shaded:4.4 - http://geronimo.apache.org/maven/xbean/4.4/xbean-asm5-shaded) * zookeeper (org.apache.zookeeper:zookeeper:3.4.6 - no url defined) * Jackson (org.codehaus.jackson:jackson-core-asl:1.9.13 - http://jackson.codehaus.org) diff --git a/sparkx/pom.xml b/sparkx/pom.xml index 920648b4c..e8b72ce75 100644 --- a/sparkx/pom.xml +++ b/sparkx/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../pom.xml diff --git a/swifts/pom.xml b/swifts/pom.xml index da10be47e..410191dd7 100644 --- a/swifts/pom.xml +++ b/swifts/pom.xml @@ -5,7 +5,7 @@ wormhole-parent edp.wormhole - 0.6.2 + 0.6.3 ../pom.xml 4.0.0 diff --git a/ums/LICENSE-3RD-PARTY b/ums/LICENSE-3RD-PARTY index dc6056881..eee0db0c6 100644 --- a/ums/LICENSE-3RD-PARTY +++ b/ums/LICENSE-3RD-PARTY @@ -7,7 +7,7 @@ List of third-party dependencies grouped by their license type. * Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.3.0 - http://wiki.fasterxml.com/JacksonHome) * Jackson-core (com.fasterxml.jackson.core:jackson-core:2.3.1 - http://wiki.fasterxml.com/JacksonHome) * jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.3.1 - http://wiki.fasterxml.com/JacksonHome) - * wormhole-util (edp.wormhole:wormhole-util:0.6.2 - no url defined) + * wormhole-util (edp.wormhole:wormhole-util:0.6.3 - no url defined) * Joda-Time (joda-time:joda-time:2.9.8 - http://www.joda.org/joda-time/) * Apache Log4j (log4j:log4j:1.2.17 - http://logging.apache.org/log4j/1.2/) * Joda-Convert (org.joda:joda-convert:1.7 - http://www.joda.org/joda-convert/) diff --git a/ums/pom.xml b/ums/pom.xml index 9fd4ea77f..fb335296b 100644 --- a/ums/pom.xml +++ b/ums/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../pom.xml diff --git a/util/pom.xml b/util/pom.xml index ebd0b9059..f6b52ca59 100644 --- a/util/pom.xml +++ b/util/pom.xml @@ -7,7 +7,7 @@ edp.wormhole wormhole-parent - 0.6.2 + 0.6.3 ../pom.xml