Skip to content

Commit

Permalink
Merge pull request #663 from mpryahin/master
Browse files Browse the repository at this point in the history 
Support for huge memory units
  • Loading branch information
@havocp
havocp authored Feb 17, 2020
2 parents e3ec7d3 + 89afa77 commit d0271d4
Show file tree
Hide file tree
Showing 6 changed files with 140 additions and 60 deletions.
47 changes: 40 additions & 7 deletions config/src/main/java/com/typesafe/config/ConfigMemorySize.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,21 @@
*/
package com.typesafe.config;

import java.math.BigInteger;

/**
* An immutable class representing an amount of memory. Use
* static factory methods such as {@link
* ConfigMemorySize#ofBytes(long)} to create instances.
* ConfigMemorySize#ofBytes(BigInteger)} to create instances.
*
* @since 1.3.0
*/
public final class ConfigMemorySize {
private final long bytes;

private ConfigMemorySize(long bytes) {
if (bytes < 0)
private BigInteger bytes;

private ConfigMemorySize(BigInteger bytes) {
if (bytes.signum() < 0)
throw new IllegalArgumentException("Attempt to construct ConfigMemorySize with negative number: " + bytes);
this.bytes = bytes;
}
Expand All @@ -26,16 +29,46 @@ private ConfigMemorySize(long bytes) {
* @param bytes a number of bytes
* @return an instance representing the number of bytes
*/
public static ConfigMemorySize ofBytes(long bytes) {
public static ConfigMemorySize ofBytes(BigInteger bytes) {
return new ConfigMemorySize(bytes);
}

/**
* Constructs a ConfigMemorySize representing the given
* number of bytes.
* @param bytes a number of bytes
* @return an instance representing the number of bytes
*/
public static ConfigMemorySize ofBytes(long bytes) {
return new ConfigMemorySize(BigInteger.valueOf(bytes));
}

/**
* Gets the size in bytes.
*
* @since 1.3.0
* @return how many bytes
* @exception IllegalArgumentException when memory value
* in bytes doesn't fit in a long value. Consider using
* {@link #toBytesBigInteger} in this case.
*/
public long toBytes() {
if (bytes.bitLength() < 64)
return bytes.longValue();
else
throw new IllegalArgumentException(
"size-in-bytes value is out of range for a 64-bit long: '" + bytes + "'");
}

/**
* Gets the size in bytes. The behavior of this method
* is the same as that of the {@link #toBytes()} method,
* except that the number of bytes returned as a
* BigInteger value. Use it when memory value in bytes
* doesn't fit in a long value.
* @return how many bytes
*/
public BigInteger toBytesBigInteger() {
return bytes;
}

Expand All @@ -47,7 +80,7 @@ public String toString() {
@Override
public boolean equals(Object other) {
if (other instanceof ConfigMemorySize) {
return ((ConfigMemorySize)other).bytes == this.bytes;
return ((ConfigMemorySize)other).bytes.equals(this.bytes);
} else {
return false;
}
Expand All @@ -56,7 +89,7 @@ public boolean equals(Object other) {
@Override
public int hashCode() {
// in Java 8 this can become Long.hashCode(bytes)
return Long.valueOf(bytes).hashCode();
return bytes.hashCode();
}

}
Expand Down
98 changes: 62 additions & 36 deletions config/src/main/java/com/typesafe/config/impl/SimpleConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import com.typesafe.config.Config;
import com.typesafe.config.ConfigException;
Expand Down Expand Up @@ -282,20 +283,54 @@ public Object getAnyRef(String path) {

@Override
public Long getBytes(String path) {
Long size = null;
BigInteger bytes = getBytesBigInteger(path);
ConfigValue v = find(path, ConfigValueType.STRING);
return toLong(bytes, v.origin(), path);
}

private BigInteger getBytesBigInteger(String path) {
BigInteger bytes;
ConfigValue v = find(path, ConfigValueType.STRING);
try {
size = getLong(path);
bytes = BigInteger.valueOf(getLong(path));
} catch (ConfigException.WrongType e) {
ConfigValue v = find(path, ConfigValueType.STRING);
size = parseBytes((String) v.unwrapped(),
v.origin(), path);
bytes = parseBytes((String) v.unwrapped(),
v.origin(), path);
}
if (bytes.signum() < 0)
throw new ConfigException.BadValue(v.origin(), path,
"Attempt to construct memory size with negative number: " + bytes);
return bytes;
}

private List<BigInteger> getBytesListBigInteger(String path){
List<BigInteger> result = new ArrayList<>();
List<? extends ConfigValue> list = getList(path);

for (ConfigValue v : list) {
BigInteger bytes;
if (v.valueType() == ConfigValueType.NUMBER) {
bytes = BigInteger.valueOf(((Number) v.unwrapped()).longValue());
} else if (v.valueType() == ConfigValueType.STRING) {
String s = (String) v.unwrapped();
bytes = parseBytes(s, v.origin(), path);
} else {
throw new ConfigException.WrongType(v.origin(), path,
"memory size string or number of bytes", v.valueType()
.name());
}
if (bytes.signum() < 0)
throw new ConfigException.BadValue(v.origin(), path,
"Attempt to construct ConfigMemorySize with negative number: " + bytes);

result.add(bytes);
}
return size;
return result;
}

@Override
public ConfigMemorySize getMemorySize(String path) {
return ConfigMemorySize.ofBytes(getBytes(path));
return ConfigMemorySize.ofBytes(getBytesBigInteger(path));
}

@Deprecated
Expand Down Expand Up @@ -482,32 +517,27 @@ public List<? extends Object> getAnyRefList(String path) {

@Override
public List<Long> getBytesList(String path) {
List<Long> l = new ArrayList<Long>();
List<? extends ConfigValue> list = getList(path);
for (ConfigValue v : list) {
if (v.valueType() == ConfigValueType.NUMBER) {
l.add(((Number) v.unwrapped()).longValue());
} else if (v.valueType() == ConfigValueType.STRING) {
String s = (String) v.unwrapped();
Long n = parseBytes(s, v.origin(), path);
l.add(n);
} else {
throw new ConfigException.WrongType(v.origin(), path,
"memory size string or number of bytes", v.valueType()
.name());
}
ConfigValue v = find(path, ConfigValueType.LIST);
return getBytesListBigInteger(path).stream()
.map(bytes -> toLong(bytes, v.origin(), path))
.collect(Collectors.toList());
}

private Long toLong(BigInteger value, ConfigOrigin originForException,
String pathForException){
if (value.bitLength() < 64) {
return value.longValue();
} else {
throw new ConfigException.BadValue(originForException, pathForException,
"size-in-bytes value is out of range for a 64-bit long: '" + value + "'");
}
return l;
}

@Override
public List<ConfigMemorySize> getMemorySizeList(String path) {
List<Long> list = getBytesList(path);
List<ConfigMemorySize> builder = new ArrayList<ConfigMemorySize>();
for (Long v : list) {
builder.add(ConfigMemorySize.ofBytes(v));
}
return builder;
return getBytesListBigInteger(path).stream()
.map(ConfigMemorySize::ofBytes)
.collect(Collectors.toList());
}

@Override
Expand Down Expand Up @@ -848,12 +878,12 @@ static MemoryUnit parseUnit(String unit) {
* @throws ConfigException
* if string is invalid
*/
public static long parseBytes(String input, ConfigOrigin originForException,
String pathForException) {
public static BigInteger parseBytes(String input, ConfigOrigin originForException,
String pathForException) {
String s = ConfigImplUtil.unicodeTrim(input);
String unitString = getUnits(s);
String numberString = ConfigImplUtil.unicodeTrim(s.substring(0,
s.length() - unitString.length()));
s.length() - unitString.length()));

// this would be caught later anyway, but the error message
// is more helpful if we check it here.
Expand All @@ -880,11 +910,7 @@ public static long parseBytes(String input, ConfigOrigin originForException,
BigDecimal resultDecimal = (new BigDecimal(units.bytes)).multiply(new BigDecimal(numberString));
result = resultDecimal.toBigInteger();
}
if (result.bitLength() < 64)
return result.longValue();
else
throw new ConfigException.BadValue(originForException, pathForException,
"size-in-bytes value is out of range for a 64-bit long: '" + input + "'");
return result;
} catch (NumberFormatException e) {
throw new ConfigException.BadValue(originForException, pathForException,
"Could not parse size-in-bytes number '" + numberString + "'");
Expand Down
2 changes: 2 additions & 0 deletions config/src/test/resources/test01.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@
"megsList" : [1M, 1024K, 1048576],
"megAsNumber" : 1048576,
"halfMeg" : 0.5M
"yottabyte" : 1YB
"yottabyteList" : [1YB, 0.5YB]
},

"system" : {
Expand Down
8 changes: 8 additions & 0 deletions config/src/test/scala/com/typesafe/config/impl/ConfigMemorySizeTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
*/
package com.typesafe.config.impl

import java.math.BigInteger

import org.junit.Assert._
import org.junit._
import com.typesafe.config.ConfigMemorySize
Expand All @@ -22,4 +24,10 @@ class ConfigMemorySizeTest extends TestUtils {
val kilobyte = ConfigMemorySize.ofBytes(1024)
assertEquals(1024, kilobyte.toBytes)
}

@Test
def testGetBytes() {
val yottabyte = ConfigMemorySize.ofBytes(new BigInteger("1000000000000000000000000"))
assertEquals(new BigInteger("1000000000000000000000000"), yottabyte.toBytesBigInteger)
}
}
5 changes: 5 additions & 0 deletions config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
*/
package com.typesafe.config.impl

import java.math.BigInteger
import java.time.temporal.{ ChronoUnit, TemporalUnit }

import org.junit.Assert._
Expand Down Expand Up @@ -831,6 +832,10 @@ class ConfigTest extends TestUtils {
assertEquals(Seq(1024 * 1024L, 1024 * 1024L, 1024L * 1024L),
conf.getMemorySizeList("memsizes.megsList").asScala.map(_.toBytes))
assertEquals(512 * 1024L, conf.getMemorySize("memsizes.halfMeg").toBytes)

assertEquals(new BigInteger("1000000000000000000000000"), conf.getMemorySize("memsizes.yottabyte").toBytesBigInteger)
assertEquals(Seq(new BigInteger("1000000000000000000000000"), new BigInteger("500000000000000000000000")),
conf.getMemorySizeList("memsizes.yottabyteList").asScala.map(_.toBytesBigInteger))
}

@Test
Expand Down
40 changes: 23 additions & 17 deletions config/src/test/scala/com/typesafe/config/impl/UnitParserTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
*/
package com.typesafe.config.impl

import java.math.BigInteger
import java.time.{ LocalDate, Period }
import java.time.temporal.ChronoUnit

Expand Down Expand Up @@ -89,10 +90,10 @@ class UnitParserTest extends TestUtils {

@Test
def parseMemorySizeInBytes(): Unit = {
def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
def parseMem(s: String): BigInteger = SimpleConfig.parseBytes(s, fakeOrigin(), "test")

assertEquals(Long.MaxValue, parseMem(s"${Long.MaxValue} bytes"))
assertEquals(Long.MinValue, parseMem(s"${Long.MinValue} bytes"))
assertEquals(BigInteger.valueOf(Long.MaxValue), parseMem(s"${Long.MaxValue} bytes"))
assertEquals(BigInteger.valueOf(Long.MinValue), parseMem(s"${Long.MinValue} bytes"))

val oneMebis = List("1048576", "1048576b", "1048576bytes", "1048576byte",
"1048576 b", "1048576 bytes",
Expand All @@ -104,7 +105,7 @@ class UnitParserTest extends TestUtils {

for (s <- oneMebis) {
val result = parseMem(s)
assertEquals(1024 * 1024, result)
assertEquals(BigInteger.valueOf(1024 * 1024), result)
}

val oneMegas = List("1000000", "1000000b", "1000000bytes", "1000000byte",
Expand All @@ -117,24 +118,24 @@ class UnitParserTest extends TestUtils {

for (s <- oneMegas) {
val result = parseMem(s)
assertEquals(1000 * 1000, result)
assertEquals(BigInteger.valueOf(1000 * 1000), result)
}

var result = 1024L * 1024 * 1024
for (unit <- Seq("tebi", "pebi", "exbi")) {
var result = BigInteger.valueOf(1024L * 1024 * 1024)
for (unit <- Seq("tebi", "pebi", "exbi", "zebi", "yobi")) {
val first = unit.substring(0, 1).toUpperCase()
result = result * 1024
result = result.multiply(BigInteger.valueOf(1024))
assertEquals(result, parseMem("1" + first))
assertEquals(result, parseMem("1" + first + "i"))
assertEquals(result, parseMem("1" + first + "iB"))
assertEquals(result, parseMem("1" + unit + "byte"))
assertEquals(result, parseMem("1" + unit + "bytes"))
}

result = 1000L * 1000 * 1000
for (unit <- Seq("tera", "peta", "exa")) {
result = BigInteger.valueOf(1000L * 1000 * 1000)
for (unit <- Seq("tera", "peta", "exa", "zetta", "yotta")) {
val first = unit.substring(0, 1).toUpperCase()
result = result * 1000
result = result.multiply(BigInteger.valueOf(1000))
assertEquals(result, parseMem("1" + first + "B"))
assertEquals(result, parseMem("1" + unit + "byte"))
assertEquals(result, parseMem("1" + unit + "bytes"))
Expand All @@ -156,19 +157,25 @@ class UnitParserTest extends TestUtils {
// later on we'll want to check this with BigInteger version of getBytes
@Test
def parseHugeMemorySizes(): Unit = {
def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
def parseMem(s: String): Long = ConfigFactory.parseString(s"v = $s").getBytes("v")
def assertOutOfRange(s: String): Unit = {
val fail = intercept[ConfigException.BadValue] {
parseMem(s)
}
assertTrue("number was too big", fail.getMessage.contains("out of range"))
}

def assertNegativeNumber(s: String): Unit = {
val fail = intercept[ConfigException.BadValue] {
parseMem(s)
}
assertTrue("number was negative", fail.getMessage.contains("negative number"))
}

import java.math.BigInteger
assertOutOfRange(s"${BigInteger.valueOf(Long.MaxValue).add(BigInteger.valueOf(1)).toString} bytes")
assertOutOfRange(s"${BigInteger.valueOf(Long.MinValue).subtract(BigInteger.valueOf(1)).toString} bytes")
assertNegativeNumber(s"${BigInteger.valueOf(Long.MinValue).subtract(BigInteger.valueOf(1)).toString} bytes")

var result = 1024L * 1024 * 1024
for (unit <- Seq("zebi", "yobi")) {
val first = unit.substring(0, 1).toUpperCase()
assertOutOfRange("1" + first)
Expand All @@ -177,17 +184,16 @@ class UnitParserTest extends TestUtils {
assertOutOfRange("1" + unit + "byte")
assertOutOfRange("1" + unit + "bytes")
assertOutOfRange("1.1" + first)
assertOutOfRange("-1" + first)
assertNegativeNumber("-1" + first)
}

result = 1000L * 1000 * 1000
for (unit <- Seq("zetta", "yotta")) {
val first = unit.substring(0, 1).toUpperCase()
assertOutOfRange("1" + first + "B")
assertOutOfRange("1" + unit + "byte")
assertOutOfRange("1" + unit + "bytes")
assertOutOfRange("1.1" + first + "B")
assertOutOfRange("-1" + first + "B")
assertNegativeNumber("-1" + first + "B")
}

assertOutOfRange("1000 exabytes")
Expand Down

0 comments on commit d0271d4

Please sign in to comment.