From 32513d55976867332aa08db6765adbdd0a39690f Mon Sep 17 00:00:00 2001 From: mycFelix Date: Mon, 20 Feb 2017 16:05:07 +0800 Subject: [PATCH 1/5] upgrade yarn scheduler driver memory to ByteAmount --- .../twitter/heron/common/basics/TypeUtils.java | 11 +++++++++++ .../heron/scheduler/yarn/HeronMasterDriver.java | 2 -- .../twitter/heron/scheduler/yarn/YarnContext.java | 8 +++++--- .../com/twitter/heron/scheduler/yarn/YarnKey.java | 15 ++++----------- .../heron/scheduler/yarn/YarnLauncher.java | 5 +++-- .../java/com/twitter/heron/spi/common/Config.java | 8 ++++++++ 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/heron/common/src/java/com/twitter/heron/common/basics/TypeUtils.java b/heron/common/src/java/com/twitter/heron/common/basics/TypeUtils.java index 5dcc1a413ef..52f1e63fb6f 100644 --- a/heron/common/src/java/com/twitter/heron/common/basics/TypeUtils.java +++ b/heron/common/src/java/com/twitter/heron/common/basics/TypeUtils.java @@ -87,6 +87,17 @@ public static ByteAmount getByteAmount(Object o) { } } + public static ByteAmount getByteAmountMB(Object o) { + if (o != null && o instanceof ByteAmount) { + return (ByteAmount) o; + } + try { + return ByteAmount.fromMegabytes(getLong(o)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Don't know how to convert " + o + " to ByteAmount", e); + } + } + public static Boolean getBoolean(Object o) { if (o instanceof Boolean) { return (Boolean) o; diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/HeronMasterDriver.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/HeronMasterDriver.java index 1a2d1814f44..d8afb84ae4b 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/HeronMasterDriver.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/HeronMasterDriver.java @@ -99,9 +99,7 @@ */ @Unit public class HeronMasterDriver { - static final int TM_MEM_SIZE_MB = 1024; static final int TMASTER_CONTAINER_ID = 0; - static final int MB = 1024 * 1024; private static final Logger LOG = Logger.getLogger(HeronMasterDriver.class.getName()); private final String topologyPackageName; private final String heronCorePackageName; diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java index c63659608c7..cf00c091d7d 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java @@ -14,6 +14,8 @@ package com.twitter.heron.scheduler.yarn; +import com.twitter.heron.common.basics.ByteAmount; +import com.twitter.heron.common.basics.TypeUtils; import com.twitter.heron.spi.common.Config; import com.twitter.heron.spi.common.Context; @@ -27,8 +29,8 @@ public static String heronYarnQueue(Config cfg) { YarnKey.HERON_SCHEDULER_YARN_QUEUE.getDefaultString()); } - public static int heronDriverMemoryMb(Config cfg) { - return cfg.getIntegerValue(YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.value(), - YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.getDefaultInt()); + public static ByteAmount heronDriverMemoryMb(Config cfg) { + return cfg.getByteAmountValueMB(YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.value(), + (ByteAmount) YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.getDefault()); } } diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnKey.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnKey.java index 46b85a39e89..7f6683a0c41 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnKey.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnKey.java @@ -14,6 +14,7 @@ package com.twitter.heron.scheduler.yarn; +import com.twitter.heron.common.basics.ByteAmount; import com.twitter.heron.spi.common.Key; /** @@ -23,7 +24,7 @@ public enum YarnKey { // yarn queue for submitting and launching the topology HERON_SCHEDULER_YARN_QUEUE("heron.scheduler.yarn.queue", "default"), // the amount of memory topology's driver (yarn application master) needs - YARN_SCHEDULER_DRIVER_MEMORY_MB("heron.scheduler.yarn.driver.memory.mb", 2048); + YARN_SCHEDULER_DRIVER_MEMORY_MB("heron.scheduler.yarn.driver.memory.mb", ByteAmount.fromMegabytes(2048)); private final String value; private final Key.Type type; @@ -35,9 +36,9 @@ public enum YarnKey { this.defaultValue = defaultValue; } - YarnKey(String value, Integer defaultValue) { + YarnKey(String value, ByteAmount defaultValue) { this.value = value; - this.type = Key.Type.INTEGER; + this.type = Key.Type.BYTE_AMOUNT; this.defaultValue = defaultValue; } @@ -56,12 +57,4 @@ public String getDefaultString() { } return (String) this.defaultValue; } - - public int getDefaultInt() { - if (type != Key.Type.INTEGER) { - throw new IllegalAccessError(String.format( - "Config Key %s is type %s, getDefaultInt() not supported", this.name(), this.type)); - } - return (Integer) this.defaultValue; - } } diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java index 54c1e5aceda..9e74cee83fe 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java @@ -34,6 +34,7 @@ import org.apache.reef.tang.annotations.Unit; import org.apache.reef.tang.exceptions.InjectionException; +import com.twitter.heron.common.basics.ByteAmount; import com.twitter.heron.scheduler.yarn.HeronMasterDriver.ContainerAllocationHandler; import com.twitter.heron.scheduler.yarn.HeronMasterDriver.FailedContainerHandler; import com.twitter.heron.scheduler.yarn.HeronMasterDriver.HeronSchedulerLauncher; @@ -63,7 +64,7 @@ public class YarnLauncher implements ILauncher { private String role; private String env; private String queue; - private int driverMemory; + private ByteAmount driverMemory; private ArrayList libJars = new ArrayList<>(); @Override @@ -161,7 +162,7 @@ Configuration getHMDriverConf() { .set(HeronDriverConfiguration.HTTP_PORT, 0) .set(HeronDriverConfiguration.VERBOSE, false) .set(YarnDriverConfiguration.QUEUE, queue) - .set(DriverConfiguration.DRIVER_MEMORY, driverMemory) + .set(DriverConfiguration.DRIVER_MEMORY, driverMemory.asMegabytes()) .build(); } diff --git a/heron/spi/src/java/com/twitter/heron/spi/common/Config.java b/heron/spi/src/java/com/twitter/heron/spi/common/Config.java index 817cb3ad40d..64973b2da10 100644 --- a/heron/spi/src/java/com/twitter/heron/spi/common/Config.java +++ b/heron/spi/src/java/com/twitter/heron/spi/common/Config.java @@ -240,6 +240,14 @@ public ByteAmount getByteAmountValue(Key key) { return TypeUtils.getByteAmount(value); } + public ByteAmount getByteAmountValueMB(String key, ByteAmount defaultValue) { + Object value = get(key); + if (value != null) { + return TypeUtils.getByteAmountMB(value); + } + return defaultValue; + } + DryRunFormatType getDryRunFormatType(Key key) { return (DryRunFormatType) get(key); } From 085e878569b9a2b6e5702cc563ff7088797a97c5 Mon Sep 17 00:00:00 2001 From: mycFelix Date: Tue, 21 Feb 2017 08:57:23 +0800 Subject: [PATCH 2/5] remove unused import --- .../src/java/com/twitter/heron/scheduler/yarn/YarnContext.java | 1 - .../src/java/com/twitter/heron/scheduler/yarn/YarnKey.java | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java index cf00c091d7d..2eab3f987ae 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java @@ -15,7 +15,6 @@ package com.twitter.heron.scheduler.yarn; import com.twitter.heron.common.basics.ByteAmount; -import com.twitter.heron.common.basics.TypeUtils; import com.twitter.heron.spi.common.Config; import com.twitter.heron.spi.common.Context; diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnKey.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnKey.java index 7f6683a0c41..ad1a3db28dc 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnKey.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnKey.java @@ -24,7 +24,8 @@ public enum YarnKey { // yarn queue for submitting and launching the topology HERON_SCHEDULER_YARN_QUEUE("heron.scheduler.yarn.queue", "default"), // the amount of memory topology's driver (yarn application master) needs - YARN_SCHEDULER_DRIVER_MEMORY_MB("heron.scheduler.yarn.driver.memory.mb", ByteAmount.fromMegabytes(2048)); + YARN_SCHEDULER_DRIVER_MEMORY_MB("heron.scheduler.yarn.driver.memory.mb", + ByteAmount.fromMegabytes(2048)); private final String value; private final Key.Type type; From afe436777b800ed8ab214d7aca18666a212670e9 Mon Sep 17 00:00:00 2001 From: mycFelix Date: Wed, 22 Feb 2017 10:34:28 +0800 Subject: [PATCH 3/5] adding enum ByteAmountUnit --- .../heron/common/basics/ByteAmountUnit.java | 21 +++++++++++++++++++ .../heron/common/basics/TypeUtils.java | 16 +++++++++++--- .../heron/scheduler/yarn/YarnContext.java | 7 ++++--- .../heron/scheduler/yarn/YarnLauncher.java | 2 +- .../com/twitter/heron/spi/common/Config.java | 19 +++++++++-------- 5 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 heron/common/src/java/com/twitter/heron/common/basics/ByteAmountUnit.java diff --git a/heron/common/src/java/com/twitter/heron/common/basics/ByteAmountUnit.java b/heron/common/src/java/com/twitter/heron/common/basics/ByteAmountUnit.java new file mode 100644 index 00000000000..b14b3c362b6 --- /dev/null +++ b/heron/common/src/java/com/twitter/heron/common/basics/ByteAmountUnit.java @@ -0,0 +1,21 @@ +// Copyright 2017 Twitter. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.twitter.heron.common.basics; + +public enum ByteAmountUnit { + BYTE, + MB, + GB +} diff --git a/heron/common/src/java/com/twitter/heron/common/basics/TypeUtils.java b/heron/common/src/java/com/twitter/heron/common/basics/TypeUtils.java index 52f1e63fb6f..63443b2d281 100644 --- a/heron/common/src/java/com/twitter/heron/common/basics/TypeUtils.java +++ b/heron/common/src/java/com/twitter/heron/common/basics/TypeUtils.java @@ -81,18 +81,28 @@ public static ByteAmount getByteAmount(Object o) { return (ByteAmount) o; } try { - return ByteAmount.fromBytes(getLong(o)); + return getByteAmount(o, ByteAmountUnit.BYTE); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Don't know how to convert " + o + " to ByteAmount", e); } } - public static ByteAmount getByteAmountMB(Object o) { + public static ByteAmount getByteAmount(Object o, ByteAmountUnit unit) { if (o != null && o instanceof ByteAmount) { return (ByteAmount) o; } try { - return ByteAmount.fromMegabytes(getLong(o)); + long amount = getLong(o); + switch (unit) { + case BYTE: + return ByteAmount.fromBytes(amount); + case MB: + return ByteAmount.fromMegabytes(amount); + case GB: + return ByteAmount.fromGigabytes(amount); + default: + return ByteAmount.fromBytes(amount); + } } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Don't know how to convert " + o + " to ByteAmount", e); } diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java index 2eab3f987ae..e5b4ed6afd0 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnContext.java @@ -15,6 +15,7 @@ package com.twitter.heron.scheduler.yarn; import com.twitter.heron.common.basics.ByteAmount; +import com.twitter.heron.common.basics.ByteAmountUnit; import com.twitter.heron.spi.common.Config; import com.twitter.heron.spi.common.Context; @@ -28,8 +29,8 @@ public static String heronYarnQueue(Config cfg) { YarnKey.HERON_SCHEDULER_YARN_QUEUE.getDefaultString()); } - public static ByteAmount heronDriverMemoryMb(Config cfg) { - return cfg.getByteAmountValueMB(YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.value(), - (ByteAmount) YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.getDefault()); + public static ByteAmount heronDriverMemory(Config cfg) { + return cfg.getByteAmountValue(YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.value(), + (ByteAmount) YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.getDefault(), ByteAmountUnit.MB); } } diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java index 9e74cee83fe..33c3cc00f91 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java @@ -76,7 +76,7 @@ public void initialize(Config config, Config runtime) { role = Context.role(config); env = Context.environ(config); queue = YarnContext.heronYarnQueue(config); - driverMemory = YarnContext.heronDriverMemoryMb(config); + driverMemory = YarnContext.heronDriverMemory(config); try { // In addition to jar for REEF's driver implementation, jar for packing and state manager diff --git a/heron/spi/src/java/com/twitter/heron/spi/common/Config.java b/heron/spi/src/java/com/twitter/heron/spi/common/Config.java index 64973b2da10..e1e1b14c160 100644 --- a/heron/spi/src/java/com/twitter/heron/spi/common/Config.java +++ b/heron/spi/src/java/com/twitter/heron/spi/common/Config.java @@ -21,6 +21,7 @@ import java.util.logging.Logger; import com.twitter.heron.common.basics.ByteAmount; +import com.twitter.heron.common.basics.ByteAmountUnit; import com.twitter.heron.common.basics.DryRunFormatType; import com.twitter.heron.common.basics.PackageType; import com.twitter.heron.common.basics.TypeUtils; @@ -28,12 +29,12 @@ /** * Config is an Immutable Map of <String, Object> The get/set API that uses Key objects * should be favored over Strings. Usage of the String API should be refactored out. - * + *

* A newly created Config object holds configs that might include wildcard tokens, like * ${HERON_HOME}/bin, ${HERON_LIB}/packing/*. Token substitution can be done by converting that * config to a local or cluster config by using the {@code Config.toLocalMode} or * {@code Config.toClusterMode} methods. - * + *

* Local mode is for a config to be used to run Heron locally, where HERON_HOME might be an install * dir on the local host (e.g. HERON_HOME=/usr/bin/heron). Cluster mode is to be used when building * configs for a remote process run on a service, where all directories might be relative to the @@ -103,11 +104,11 @@ private static Config expand(Config config) { * Recursively expand each config value until token substitution is exhausted. We must recurse * to handle the case where field expansion requires multiple iterations, due to new tokens being * introduced as we replace. For example: - * - * ${HERON_BIN}/heron-executor gets expanded to - * ${HERON_HOME}/bin/heron-executor gets expanded to - * /usr/local/heron/bin/heron-executor - * + *

+ * ${HERON_BIN}/heron-executor gets expanded to + * ${HERON_HOME}/bin/heron-executor gets expanded to + * /usr/local/heron/bin/heron-executor + *

* If break logic is when another round does not reduce the number of tokens, since it means we * couldn't find a valid replacement. */ @@ -240,10 +241,10 @@ public ByteAmount getByteAmountValue(Key key) { return TypeUtils.getByteAmount(value); } - public ByteAmount getByteAmountValueMB(String key, ByteAmount defaultValue) { + public ByteAmount getByteAmountValue(String key, ByteAmount defaultValue, ByteAmountUnit unit) { Object value = get(key); if (value != null) { - return TypeUtils.getByteAmountMB(value); + return TypeUtils.getByteAmount(defaultValue, unit); } return defaultValue; } From 8e98314bf93dad45e3e7748f85d4581a6d9fe7ee Mon Sep 17 00:00:00 2001 From: mycFelix Date: Wed, 22 Feb 2017 10:59:38 +0800 Subject: [PATCH 4/5] fix typo --- heron/spi/src/java/com/twitter/heron/spi/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heron/spi/src/java/com/twitter/heron/spi/common/Config.java b/heron/spi/src/java/com/twitter/heron/spi/common/Config.java index e1e1b14c160..bbae1b6a198 100644 --- a/heron/spi/src/java/com/twitter/heron/spi/common/Config.java +++ b/heron/spi/src/java/com/twitter/heron/spi/common/Config.java @@ -244,7 +244,7 @@ public ByteAmount getByteAmountValue(Key key) { public ByteAmount getByteAmountValue(String key, ByteAmount defaultValue, ByteAmountUnit unit) { Object value = get(key); if (value != null) { - return TypeUtils.getByteAmount(defaultValue, unit); + return TypeUtils.getByteAmount(value, unit); } return defaultValue; } From 55f03767c76859085355b19a9e61eecad4a3e95b Mon Sep 17 00:00:00 2001 From: mycFelix Date: Thu, 23 Feb 2017 11:17:47 +0800 Subject: [PATCH 5/5] cast to int --- .../src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java index 33c3cc00f91..05870b109dd 100644 --- a/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java +++ b/heron/schedulers/src/java/com/twitter/heron/scheduler/yarn/YarnLauncher.java @@ -162,7 +162,7 @@ Configuration getHMDriverConf() { .set(HeronDriverConfiguration.HTTP_PORT, 0) .set(HeronDriverConfiguration.VERBOSE, false) .set(YarnDriverConfiguration.QUEUE, queue) - .set(DriverConfiguration.DRIVER_MEMORY, driverMemory.asMegabytes()) + .set(DriverConfiguration.DRIVER_MEMORY, (int) driverMemory.asMegabytes()) .build(); }