From f04bd8a41fd2fb4020087ac775734c1e49ead96f Mon Sep 17 00:00:00 2001 From: Anirudh Date: Sun, 24 Sep 2017 11:20:17 -0700 Subject: [PATCH 1/3] Update POMs --- assembly/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- conf/kubernetes-resource-staging-server.yaml | 2 +- conf/kubernetes-shuffle-service.yaml | 2 +- core/pom.xml | 2 +- examples/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/flume-assembly/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml | 2 +- external/java8-tests/pom.xml | 2 +- external/kafka-0-10-assembly/pom.xml | 2 +- external/kafka-0-10-sql/pom.xml | 2 +- external/kafka-0-10/pom.xml | 2 +- external/kafka-0-8-assembly/pom.xml | 2 +- external/kafka-0-8/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- resource-managers/kubernetes/core/pom.xml | 2 +- resource-managers/kubernetes/docker-minimal-bundle/pom.xml | 2 +- .../kubernetes/integration-tests-spark-jobs-helpers/pom.xml | 2 +- .../kubernetes/integration-tests-spark-jobs/pom.xml | 2 +- resource-managers/kubernetes/integration-tests/pom.xml | 2 +- resource-managers/mesos/pom.xml | 2 +- resource-managers/yarn/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- 43 files changed, 43 insertions(+), 43 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 29ab454fee253..52b81e3130ef6 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index f437084ec6dd5..4a5b5a55840ac 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index b80c1df889731..6adb201aeedc1 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index ae9ec59180486..e10f8cf347f19 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 60549568be0ef..321dedd6b1871 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 764be312470cf..0c7e01d1d8dde 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index d0b543137781f..3d8d25bbd1ba5 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/conf/kubernetes-resource-staging-server.yaml b/conf/kubernetes-resource-staging-server.yaml index 80d59b8091903..2d6e619cdb37b 100644 --- a/conf/kubernetes-resource-staging-server.yaml +++ b/conf/kubernetes-resource-staging-server.yaml @@ -32,7 +32,7 @@ spec: name: spark-resource-staging-server-config containers: - name: spark-resource-staging-server - image: kubespark/spark-resource-staging-server:v2.2.0-kubernetes-0.3.0 + image: kubespark/spark-resource-staging-server:v2.2.0-kubernetes-0.4.0 resources: requests: cpu: 100m diff --git a/conf/kubernetes-shuffle-service.yaml b/conf/kubernetes-shuffle-service.yaml index 8ab0b362ea32e..8fe2008638530 100644 --- a/conf/kubernetes-shuffle-service.yaml +++ b/conf/kubernetes-shuffle-service.yaml @@ -38,7 +38,7 @@ spec: # This is an official image that is built # from the dockerfiles/shuffle directory # in the spark distribution. - image: kubespark/spark-shuffle:v2.2.0-kubernetes-0.3.0 + image: kubespark/spark-shuffle:v2.2.0-kubernetes-0.4.0 imagePullPolicy: IfNotPresent volumeMounts: - mountPath: '/tmp' diff --git a/core/pom.xml b/core/pom.xml index a152b27284abf..d9706395bb05c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/examples/pom.xml b/examples/pom.xml index 3f7814d060526..25f3188c97ad7 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index f90eb57c64dfe..e62d606f2ce66 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index 8592d85ccbd3f..2ad920b0060e0 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index ef0231990c3c9..894604ce32501 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 606116ad0860c..2e48a86408b7c 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/java8-tests/pom.xml b/external/java8-tests/pom.xml index 586f87d312ec0..b78a5e32c5476 100644 --- a/external/java8-tests/pom.xml +++ b/external/java8-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index 39e1339bbf2a1..7c7ee1411230e 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 68ee5be2889b2..e700967c20c37 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index 23e4eef8253b2..b61f4292d5d1e 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml index aa36d4a713e1e..c18d0ba6b5156 100644 --- a/external/kafka-0-8-assembly/pom.xml +++ b/external/kafka-0-8-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml index af7de9380e271..ca66cd3cfea57 100644 --- a/external/kafka-0-8/pom.xml +++ b/external/kafka-0-8/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index ced0e89bdff16..e629143dcb96d 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 0ae897f62dda3..863c2eac4d8a9 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index 2744a9fdea489..39febb0210873 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index e59db233b54d8..2bee93dfa3d02 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 9df0c2d79a5fb..c38d90d930955 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index d120f8f3fc4f0..858ebc30ecf51 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 755f1cfe954f0..7dd0bffb2948b 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/pom.xml b/pom.xml index d90ad23752f54..2abe588a36db8 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index c042a69c14644..e859618ea5278 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index dfbf8adc5bc7b..a740ae33a5bc6 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../../pom.xml diff --git a/resource-managers/kubernetes/docker-minimal-bundle/pom.xml b/resource-managers/kubernetes/docker-minimal-bundle/pom.xml index 585d8d474bba5..b20909a409898 100644 --- a/resource-managers/kubernetes/docker-minimal-bundle/pom.xml +++ b/resource-managers/kubernetes/docker-minimal-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests-spark-jobs-helpers/pom.xml b/resource-managers/kubernetes/integration-tests-spark-jobs-helpers/pom.xml index 2e877daf95c63..b4a14e186e14a 100644 --- a/resource-managers/kubernetes/integration-tests-spark-jobs-helpers/pom.xml +++ b/resource-managers/kubernetes/integration-tests-spark-jobs-helpers/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml b/resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml index 276bbabdf32e1..14cfebac9bcac 100644 --- a/resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml +++ b/resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index fc28026de1e43..af4c05104480a 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index db492f57120e5..b8d0ed88500ac 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index a59cf8c5eb5ac..f3b5d650f40f0 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 60fed70731bfd..444b82da596fc 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 4b55cc859005c..0ac7ab0073e33 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index caa5da8b4d8b2..0694c8e0e5fec 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index df9535de6677b..810de105f4b6d 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index e5ffe7af8c044..1e6a50159553e 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 7ca0a9bb2d3a3..71d48190e556a 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.2.0-k8s-0.4.0-SNAPSHOT + 2.2.0-k8s-0.4.0 ../pom.xml From e00b2a943aaeb058e9f8de92e61ba1b740071b27 Mon Sep 17 00:00:00 2001 From: foxish Date: Sun, 24 Sep 2017 14:00:23 -0700 Subject: [PATCH 2/3] Update extensions/v1beta1.Deployment to apps --- conf/kubernetes-resource-staging-server.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/kubernetes-resource-staging-server.yaml b/conf/kubernetes-resource-staging-server.yaml index 2d6e619cdb37b..697ec1721c427 100644 --- a/conf/kubernetes-resource-staging-server.yaml +++ b/conf/kubernetes-resource-staging-server.yaml @@ -15,7 +15,7 @@ # limitations under the License. # --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1beta1 kind: Deployment metadata: name: spark-resource-staging-server From 55a57cb0ab302d24b43f8697b51cc76d48200050 Mon Sep 17 00:00:00 2001 From: foxish Date: Sun, 24 Sep 2017 15:38:13 -0700 Subject: [PATCH 3/3] Modified defaults on rss and ss --- conf/kubernetes-resource-staging-server.yaml | 2 +- conf/kubernetes-shuffle-service.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/kubernetes-resource-staging-server.yaml b/conf/kubernetes-resource-staging-server.yaml index 697ec1721c427..18b446574fdaa 100644 --- a/conf/kubernetes-resource-staging-server.yaml +++ b/conf/kubernetes-resource-staging-server.yaml @@ -39,7 +39,7 @@ spec: memory: 256Mi limits: cpu: 100m - memory: 256Mi + memory: 1Gi volumeMounts: - name: resource-staging-server-properties mountPath: '/etc/spark-resource-staging-server' diff --git a/conf/kubernetes-shuffle-service.yaml b/conf/kubernetes-shuffle-service.yaml index 8fe2008638530..6f02106ff06c5 100644 --- a/conf/kubernetes-shuffle-service.yaml +++ b/conf/kubernetes-shuffle-service.yaml @@ -32,7 +32,7 @@ spec: volumes: - name: temp-volume hostPath: - path: '/var/tmp' # change this path according to your cluster configuration. + path: '/tmp' # change this path according to your cluster configuration. containers: - name: shuffle # This is an official image that is built