diff --git a/java/jmh/README.md b/java/jmh/README.md index d9ba1e60ba5..a21adbd2a80 100644 --- a/java/jmh/README.md +++ b/java/jmh/README.md @@ -6,10 +6,10 @@ These are micro-benchmarks for RocksJava functionality, using [JMH (Java Microbe **Note**: This uses a specific build of RocksDB that is set in the `` element of the `dependencies` section of the `pom.xml` file. If you are testing local changes you should build and install a SNAPSHOT version of rocksdbjni, and update the `pom.xml` of rocksdbjni-jmh file to test with this. -For instance, this is how to install the OSX jar you just built for 8.11.0 +For instance, this is how to install the OSX jar you just built for 9.8.0 ```bash -$ mvn install:install-file -Dfile=./java/target/rocksdbjni-8.11.0-SNAPSHOT-osx.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=8.11.0-SNAPSHOT -Dpackaging=jar +$ mvn install:install-file -Dfile=./java/target/rocksdbjni-9.8.0-osx.jar -DgroupId=org.rocksdb -DartifactId=rocksdbjni -Dversion=9.8.0-SNAPSHOT -Dpackaging=jar ``` ```bash @@ -22,3 +22,30 @@ $ java -jar target/rocksdbjni-jmh-1.0-SNAPSHOT-benchmarks.jar ``` NOTE: you can append `-help` to the command above to see all of the JMH runtime options. + +#### Before (multi) + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (multiGetSize) (nthMissingKey) (valueSize) Mode Cnt Score Error Units +MultiNotFoundBenchmarks.multiNotFoundBBEvens no_column_family 100000 16 1000 2 16 thrpt 15 607.494 ± 1.560 ops/s +MultiNotFoundBenchmarks.multiNotFoundBBOdds no_column_family 100000 16 1000 2 16 thrpt 15 531.760 ± 1.968 ops/s +MultiNotFoundBenchmarks.multiNotFoundListEvens no_column_family 100000 16 1000 2 16 thrpt 15 914.955 ± 2.927 ops/s +MultiNotFoundBenchmarks.multiNotFoundListOdds no_column_family 100000 16 1000 2 16 thrpt 15 711.232 ± 2.201 ops/s + +#### Before (single) + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (nthMissingKey) (valueSize) Mode Cnt Score Error Units +GetNotFoundBenchmarks.getNotFoundEven no_column_family 100000 12 2 16 thrpt 15 291802.037 ± 1082.526 ops/s +GetNotFoundBenchmarks.getNotFoundOdd no_column_family 100000 12 2 16 thrpt 15 405500.054 ± 15590.921 ops/s + +#### After (single) +`getNotFoundEven` should be fixed, BUT why should `getNotFoundOdd` be faster than before ? Pushing the exception stack (try) seems mad... + +Benchmark (columnFamilyTestType) (keyCount) (keySize) (nthMissingKey) (valueSize) Mode Cnt Score Error Units +GetNotFoundBenchmarks.getNotFoundEven no_column_family 100000 12 2 16 thrpt 15 850779.017 ± 5680.454 ops/s +GetNotFoundBenchmarks.getNotFoundOdd no_column_family 100000 12 2 16 thrpt 15 771496.888 ± 7186.347 ops/s +Benchmark (columnFamilyTestType) (keyCount) (keySize) (nthMissingKey) (valueSize) Mode Cnt Score Error Units +GetNotFoundBenchmarks.getNotFoundEven no_column_family 100000 12 2 16 thrpt 25 846618.159 ± 4648.865 ops/s +GetNotFoundBenchmarks.getNotFoundOdd no_column_family 100000 12 2 16 thrpt 25 767894.897 ± 2713.019 ops/s + + + diff --git a/java/jmh/pom.xml b/java/jmh/pom.xml index 8400e1e67ee..bc557c66a1e 100644 --- a/java/jmh/pom.xml +++ b/java/jmh/pom.xml @@ -50,7 +50,7 @@ org.rocksdb rocksdbjni - 9.0.0 + 9.8.0-SNAPSHOT diff --git a/java/jmh/src/main/java/org/rocksdb/jmh/GetNotFoundBenchmarks.java b/java/jmh/src/main/java/org/rocksdb/jmh/GetNotFoundBenchmarks.java new file mode 100644 index 00000000000..fa1038800ee --- /dev/null +++ b/java/jmh/src/main/java/org/rocksdb/jmh/GetNotFoundBenchmarks.java @@ -0,0 +1,190 @@ +/** + * Copyright (c) 2011-present, Facebook, Inc. All rights reserved. + * This source code is licensed under both the GPLv2 (found in the + * COPYING file in the root directory) and Apache 2.0 License + * (found in the LICENSE.Apache file in the root directory). + */ +package org.rocksdb.jmh; + +import static org.rocksdb.util.KVUtils.ba; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import org.rocksdb.*; +import org.rocksdb.util.FileUtils; + +@State(Scope.Benchmark) +public class GetNotFoundBenchmarks { + @Param({"no_column_family", "20_column_families"}) String columnFamilyTestType; + + @Param({"1000", "100000"}) int keyCount; + + @Param({"12"}) int keySize; + + @Param({"16"}) int valueSize; + + /** + * Don't create every n-th key + * Usually, just use "2" for making the half-present array + * 0 means no missing keys + */ + @Param({"2", "16", "0"}) int nthMissingKey; + + Path dbDir; + DBOptions options; + ReadOptions readOptions; + int cfs = 0; // number of column families + private AtomicInteger cfHandlesIdx; + ColumnFamilyHandle[] cfHandles; + RocksDB db; + private final AtomicInteger keyIndex = new AtomicInteger(); + private ByteBuffer keyBuf; + private ByteBuffer valueBuf; + private byte[] keyArr; + private byte[] valueArr; + + @Setup(Level.Trial) + public void setup() throws IOException, RocksDBException { + RocksDB.loadLibrary(); + + dbDir = Files.createTempDirectory("rocksjava-get-benchmarks"); + + options = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + readOptions = new ReadOptions(); + + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + + if ("20_column_families".equals(columnFamilyTestType)) { + cfs = 20; + } + + if (cfs > 0) { + cfHandlesIdx = new AtomicInteger(1); + for (int i = 1; i <= cfs; i++) { + cfDescriptors.add(new ColumnFamilyDescriptor(ba("cf" + i))); + } + } + + final List cfHandlesList = new ArrayList<>(cfDescriptors.size()); + db = RocksDB.open(options, dbDir.toAbsolutePath().toString(), cfDescriptors, cfHandlesList); + cfHandles = cfHandlesList.toArray(new ColumnFamilyHandle[0]); + + // store initial data for retrieving via get + keyArr = new byte[keySize]; + valueArr = new byte[valueSize]; + Arrays.fill(keyArr, (byte) 0x30); + Arrays.fill(valueArr, (byte) 0x30); + for (int i = 0; i <= cfs; i++) { + for (int j = 0; j < keyCount; j++) { + if (nthMissingKey <= 0 || j % nthMissingKey != 0) { + final byte[] keyPrefix = ba("key" + j); + final byte[] valuePrefix = ba("value" + j); + System.arraycopy(keyPrefix, 0, keyArr, 0, keyPrefix.length); + System.arraycopy(valuePrefix, 0, valueArr, 0, valuePrefix.length); + db.put(cfHandles[i], keyArr, valueArr); + } + } + } + + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + + keyBuf = ByteBuffer.allocateDirect(keySize); + valueBuf = ByteBuffer.allocateDirect(valueSize); + Arrays.fill(keyArr, (byte) 0x30); + Arrays.fill(valueArr, (byte) 0x30); + keyBuf.put(keyArr); + keyBuf.flip(); + valueBuf.put(valueArr); + valueBuf.flip(); + } + + @TearDown(Level.Trial) + public void cleanup() throws IOException { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + db.close(); + options.close(); + readOptions.close(); + FileUtils.delete(dbDir); + } + + private ColumnFamilyHandle getColumnFamily() { + if (cfs == 0) { + return cfHandles[0]; + } else if (cfs == 1) { + return cfHandles[1]; + } else { + int idx = cfHandlesIdx.getAndIncrement(); + if (idx > cfs) { + cfHandlesIdx.set(1); // doesn't ensure a perfect distribution, but it's ok + idx = 0; + } + return cfHandles[idx]; + } + } + + /** + * Takes the next position in the index. + */ + private int next(final int increment) { + int idx; + int nextIdx; + while (true) { + idx = keyIndex.get(); + nextIdx = idx + increment; + if (nextIdx >= keyCount) { + nextIdx = nextIdx % keyCount; + } + + if (keyIndex.compareAndSet(idx, nextIdx)) { + break; + } + } + return idx; + } + + private int next() { + return next(1); + } + + // String -> byte[] + private byte[] getKeyArr(final int increment) { + final int MAX_LEN = 9; // key100000 + final int keyIdx = next(increment); + final byte[] keyPrefix = ba("key" + keyIdx); + System.arraycopy(keyPrefix, 0, keyArr, 0, keyPrefix.length); + Arrays.fill(keyArr, keyPrefix.length, MAX_LEN, (byte) 0x30); + return keyArr; + } + + /** + * Get even values, these should not be present + * @throws RocksDBException + */ + @Benchmark + public void getNotFoundEven(Blackhole blackhole) throws RocksDBException { + blackhole.consume(db.get(getColumnFamily(), getKeyArr(2))); + } + + /** + * Get odd values 1, 3, 5, these should be present + * @throws RocksDBException + */ + @Benchmark + public void getNotFoundOdd(Blackhole blackhole) throws RocksDBException { + next(); + blackhole.consume(db.get(getColumnFamily(), getKeyArr(2))); + } +} \ No newline at end of file diff --git a/java/jmh/src/main/java/org/rocksdb/jmh/MultiNotFoundBenchmarks.java b/java/jmh/src/main/java/org/rocksdb/jmh/MultiNotFoundBenchmarks.java new file mode 100644 index 00000000000..c579ce07330 --- /dev/null +++ b/java/jmh/src/main/java/org/rocksdb/jmh/MultiNotFoundBenchmarks.java @@ -0,0 +1,276 @@ +package org.rocksdb.jmh; + +import static org.rocksdb.util.KVUtils.ba; +import static org.rocksdb.util.KVUtils.keys; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.rocksdb.*; +import org.rocksdb.util.FileUtils; + +@State(Scope.Thread) +public class MultiNotFoundBenchmarks { + @Param({"no_column_family", "20_column_families"}) String columnFamilyTestType; + + @Param({"100000"}) int keyCount; + + /** + * Don't create every n-th key + * Usually, just use "2" for making the + */ + @Param({"2", "16"}) int nthMissingKey; + + @Param({ + "10", + "100", + "1000", + "10000", + }) + int multiGetSize; + + @Param({"16"}) int valueSize; + + @Param({"16"}) int keySize; // big enough + + Path dbDir; + DBOptions options; + int cfs = 0; // number of column families + private AtomicInteger cfHandlesIdx; + ColumnFamilyHandle[] cfHandles; + RocksDB db; + private final AtomicInteger keyIndex = new AtomicInteger(); + + private List defaultCFHandles = new ArrayList<>(); + + @Setup(Level.Trial) + public void setup() throws IOException, RocksDBException { + RocksDB.loadLibrary(); + + dbDir = Files.createTempDirectory("rocksjava-multiget-benchmarks"); + + options = new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true); + + final List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + + if ("20_column_families".equals(columnFamilyTestType)) { + cfs = 20; + } + + if (cfs > 0) { + cfHandlesIdx = new AtomicInteger(1); + for (int i = 1; i <= cfs; i++) { + cfDescriptors.add(new ColumnFamilyDescriptor(ba("cf" + i))); + } + } + + final List cfHandlesList = new ArrayList<>(cfDescriptors.size()); + db = RocksDB.open(options, dbDir.toAbsolutePath().toString(), cfDescriptors, cfHandlesList); + cfHandles = cfHandlesList.toArray(new ColumnFamilyHandle[0]); + + // store initial data for retrieving via get + for (int j = 0; j < keyCount; j++) { + if (j % nthMissingKey != 0) { + final byte[] paddedValue = Arrays.copyOf(ba("value" + j), valueSize); + db.put(ba("key" + j), paddedValue); + } + } + + // store initial data for retrieving via get - column families + for (int i = 0; i < cfs; i++) { + for (int j = 0; j < keyCount; j++) { + if (j % nthMissingKey != 0) { + final byte[] paddedValue = Arrays.copyOf(ba("value" + j), valueSize); + db.put(cfHandles[i], ba("key" + j), paddedValue); + } + } + } + + // build a big list of default column families for efficient passing + final ColumnFamilyHandle defaultCFH = db.getDefaultColumnFamily(); + for (int i = 0; i < keyCount; i++) { + defaultCFHandles.add(defaultCFH); + } + + // list of random cfs + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + } + + @TearDown(Level.Trial) + public void cleanup() throws IOException { + for (final ColumnFamilyHandle cfHandle : cfHandles) { + cfHandle.close(); + } + db.close(); + options.close(); + FileUtils.delete(dbDir); + } + + private int next(final int inc, final int limit) { + int idx; + int nextIdx; + while (true) { + idx = keyIndex.get(); + nextIdx = idx + inc; + if (nextIdx >= limit) { + nextIdx = inc; + } + + if (keyIndex.compareAndSet(idx, nextIdx)) { + break; + } + } + + if (nextIdx >= limit) { + return -1; + } else { + return idx; + } + } + + ByteBuffer keysBuffer; + ByteBuffer valuesBuffer; + + List valueBuffersList; + List keyBuffersList; + + @Setup + public void allocateSliceBuffers() { + keysBuffer = ByteBuffer.allocateDirect(keyCount * keySize); + valuesBuffer = ByteBuffer.allocateDirect(keyCount * valueSize); + valueBuffersList = new ArrayList<>(); + keyBuffersList = new ArrayList<>(); + for (int i = 0; i < keyCount; i++) { + valueBuffersList.add(valuesBuffer.slice(i * valueSize, valueSize)); + keyBuffersList.add(keysBuffer.slice(i * keySize, keySize)); + } + } + + @TearDown + public void freeSliceBuffers() { + valueBuffersList.clear(); + } + + private List filter(List before, final int initial, final int step) { + final List after = new ArrayList<>(before.size()); + for (int i = initial; i < before.size(); i += step) { + after.add(before.get(i)); + } + return after; + } + + private List filterByteBuffers( + List before, final int initial, final int step) { + final List after = new ArrayList<>(before.size()); + for (int i = initial; i < before.size(); i += step) { + after.add(before.get(i)); + } + return after; + } + + /** + * Perform get on the even-numbered keys, which should not exist + * This should be faster (certainly not slower) than the even-numbered keys, + * but has at times been found to be slower due to implementation issues in the not-found path. + * + * @throws RocksDBException + */ + @Benchmark + public void multiNotFoundListEvens() throws RocksDBException { + final int fromKeyIdx = next(2 * multiGetSize, keyCount); + if (fromKeyIdx >= 0) { + final List keys = filter(keys(fromKeyIdx, fromKeyIdx + 2 * multiGetSize), 0, 2); + final List valueResults = db.multiGetAsList(keys); + for (final byte[] result : valueResults) { + if (result != null) + throw new RuntimeException("Test wrongly returned a value"); + } + } + } + + /** + * Perform get on the odd-numbered keys, which should exist + * This is for reference comparison w/ non-existent keys, see above + * + * @throws RocksDBException + */ + @Benchmark + public void multiNotFoundListOdds() throws RocksDBException { + final int fromKeyIdx = next(2 * multiGetSize, keyCount); + if (fromKeyIdx >= 0) { + final List keys = filter(keys(fromKeyIdx, fromKeyIdx + 2 * multiGetSize), 1, 2); + final List valueResults = db.multiGetAsList(keys); + for (final byte[] result : valueResults) { + if (result.length != valueSize) + throw new RuntimeException("Test valueSize assumption wrong"); + } + } + } + + @Benchmark + public List multiNotFoundBBEvens() throws RocksDBException { + final int fromKeyIdx = next(2 * multiGetSize, keyCount); + if (fromKeyIdx >= 0) { + final List keys = + filterByteBuffers(keys(keyBuffersList, fromKeyIdx, fromKeyIdx + 2 * multiGetSize), 0, 2); + final List values = + valueBuffersList.subList(fromKeyIdx, fromKeyIdx + multiGetSize); + final List statusResults = db.multiGetByteBuffers(keys, values); + for (final ByteBufferGetStatus result : statusResults) { + if (result.status.getCode() != Status.Code.NotFound) + throw new RuntimeException("Test status should be NotFound, was: " + result.status); + } + } + return new ArrayList<>(); + } + + @Benchmark + public List multiNotFoundBBOdds() throws RocksDBException { + final int fromKeyIdx = next(2 * multiGetSize, keyCount); + if (fromKeyIdx >= 0) { + final List keys = + filterByteBuffers(keys(keyBuffersList, fromKeyIdx, fromKeyIdx + 2 * multiGetSize), 1, 2); + final List values = + valueBuffersList.subList(fromKeyIdx, fromKeyIdx + multiGetSize); + final List statusResults = db.multiGetByteBuffers(keys, values); + for (final ByteBufferGetStatus result : statusResults) { + if (result.status.getCode() != Status.Code.Ok) + throw new RuntimeException("Test status not OK: " + result.status); + if (result.value.limit() != valueSize) + throw new RuntimeException("Test valueSize assumption wrong"); + } + } + return new ArrayList<>(); + } + + public static void main(final String[] args) throws RunnerException { + final org.openjdk.jmh.runner.options.Options opt = + new OptionsBuilder() + .include(MultiNotFoundBenchmarks.class.getSimpleName()) + .forks(1) + .jvmArgs("-ea") + .warmupIterations(1) + .measurementIterations(2) + .forks(2) + .param("columnFamilyTestType=", "no_column_family") + .param("multiGetSize=", "1000") + .param("keyCount=", "10000") + .param("nthMissingKey=", "2") + .output("jmh_output") + .build(); + + new Runner(opt).run(); + } +}