diff --git a/benchmarks/utilities/csv_exporter.py b/benchmarks/utilities/csv_exporter.py index a9f1131729..3d48adfe17 100644 --- a/benchmarks/utilities/csv_exporter.py +++ b/benchmarks/utilities/csv_exporter.py @@ -39,7 +39,7 @@ json_file_name = os.path.basename(json_file_full_path) - languages = ["csharp", "node", "python", "rust"] + languages = ["csharp", "node", "python", "rust", "java"] language = next( (language for language in languages if language in json_file_name), None ) diff --git a/java/benchmarks/build.gradle b/java/benchmarks/build.gradle index 8d9e500284..5e22fda5c7 100644 --- a/java/benchmarks/build.gradle +++ b/java/benchmarks/build.gradle @@ -18,6 +18,10 @@ dependencies { implementation 'io.lettuce:lettuce-core:6.2.6.RELEASE' implementation 'commons-cli:commons-cli:1.5.0' implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.13.0' + implementation group: 'com.google.code.gson', name: 'gson', version: '2.10.1' + + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' } // Apply a specific Java toolchain to ease working on different environments. diff --git a/java/benchmarks/src/main/java/javababushka/benchmarks/BenchmarkingApp.java b/java/benchmarks/src/main/java/javababushka/benchmarks/BenchmarkingApp.java index 66ea1f5029..814a60d6a2 100644 --- a/java/benchmarks/src/main/java/javababushka/benchmarks/BenchmarkingApp.java +++ b/java/benchmarks/src/main/java/javababushka/benchmarks/BenchmarkingApp.java @@ -2,8 +2,6 @@ import static javababushka.benchmarks.utils.Benchmarking.testClientSetGet; -import java.io.FileWriter; -import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -57,14 +55,6 @@ public static void main(String[] args) { break; } } - - if (runConfiguration.resultsFile.isPresent()) { - try { - runConfiguration.resultsFile.get().close(); - } catch (IOException ioException) { - System.out.println("Error closing results file"); - } - } } private static Options getOptions() { @@ -97,12 +87,7 @@ private static RunConfiguration verifyOptions(CommandLine line) throws ParseExce } if (line.hasOption("resultsFile")) { - try { - runConfiguration.resultsFile = - Optional.of(new FileWriter(line.getOptionValue("resultsFile"))); - } catch (IOException e) { - throw new ParseException("Unable to write to resultsFile."); - } + runConfiguration.resultsFile = Optional.ofNullable(line.getOptionValue("resultsFile")); } if (line.hasOption("dataSize")) { @@ -210,8 +195,8 @@ public boolean isEqual(String other) { public static class RunConfiguration { public String configuration; - public Optional resultsFile; public int dataSize; + public Optional resultsFile; public List concurrentTasks; public ClientName[] clients; public String host; diff --git a/java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java b/java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java index 14818f46b4..60c7b666f3 100644 --- a/java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java +++ b/java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java @@ -1,18 +1,15 @@ package javababushka.benchmarks.utils; -import java.io.FileWriter; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import java.util.stream.Collectors; import javababushka.benchmarks.AsyncClient; import javababushka.benchmarks.BenchmarkingApp; import javababushka.benchmarks.Client; @@ -26,6 +23,8 @@ public class Benchmarking { static final int SIZE_GET_KEYSPACE = 3750000; static final int SIZE_SET_KEYSPACE = 3000000; static final int ASYNC_OPERATION_TIMEOUT_SEC = 1; + // measurements are done in nano-seconds, but it should be converted to seconds later + static final double SECONDS_IN_NANO = 1e-9; private static ChosenAction randomAction() { if (Math.random() > PROB_GET) { @@ -50,7 +49,7 @@ public interface Operation { void go() throws Exception; } - private static Pair getLatency(Map actions) { + public static Pair measurePerformance(Map actions) { var action = randomAction(); long before = System.nanoTime(); try { @@ -84,56 +83,29 @@ private static double stdDeviation(ArrayList latencies, Double avgLatency) // This has the side-effect of sorting each latencies ArrayList public static Map calculateResults( Map> actionLatencies) { - Map results = new HashMap(); + Map results = new HashMap<>(); for (Map.Entry> entry : actionLatencies.entrySet()) { ChosenAction action = entry.getKey(); ArrayList latencies = entry.getValue(); - Double avgLatency = - latencies.stream().collect(Collectors.summingLong(Long::longValue)) - / Double.valueOf(latencies.size()); + double avgLatency = + SECONDS_IN_NANO * latencies.stream().mapToLong(Long::longValue).sum() / latencies.size(); Collections.sort(latencies); results.put( action, new LatencyResults( avgLatency, - percentile(latencies, 50), - percentile(latencies, 90), - percentile(latencies, 99), - stdDeviation(latencies, avgLatency))); + SECONDS_IN_NANO * percentile(latencies, 50), + SECONDS_IN_NANO * percentile(latencies, 90), + SECONDS_IN_NANO * percentile(latencies, 99), + SECONDS_IN_NANO * stdDeviation(latencies, avgLatency))); } return results; } - public static void printResults( - Map calculatedResults, Optional resultsFile) { - if (resultsFile.isPresent()) { - printResults(calculatedResults, resultsFile.get()); - } else { - printResults(calculatedResults); - } - } - - public static void printResults( - Map resultsMap, FileWriter resultsFile) { - for (Map.Entry entry : resultsMap.entrySet()) { - ChosenAction action = entry.getKey(); - LatencyResults results = entry.getValue(); - - try { - resultsFile.write("Avg. time in ms per " + action + ": " + results.avgLatency / 1000000.0); - resultsFile.write(action + " p50 latency in ms: " + results.p50Latency / 1000000.0); - resultsFile.write(action + " p90 latency in ms: " + results.p90Latency / 1000000.0); - resultsFile.write(action + " p99 latency in ms: " + results.p99Latency / 1000000.0); - resultsFile.write(action + " std dev in ms: " + results.stdDeviation / 1000000.0); - } catch (Exception ignored) { - } - } - } - public static void printResults(Map resultsMap) { for (Map.Entry entry : resultsMap.entrySet()) { ChosenAction action = entry.getKey(); @@ -189,9 +161,10 @@ public static void testClientSetGet( "> iteration = %d/%d, client# = %d/%d%n", iterationIncrement + 1, iterations, clientIndex + 1, clientCount); } + + var actions = getActionMap(clients.get(clientIndex), config.dataSize, async); // operate and calculate tik-tok - Pair result = - measurePerformance(clients.get(clientIndex), config.dataSize, async); + Pair result = measurePerformance(actions); actionResults.get(result.getLeft()).add(result.getRight()); iterationIncrement = iterationCounter.getAndIncrement(); @@ -204,6 +177,7 @@ public static void testClientSetGet( "===> concurrentNum = %d, clientNum = %d, tasks = %d%n", concurrentNum, clientCount, tasks.size()); } + long started = System.nanoTime(); tasks.stream() .map(CompletableFuture::runAsync) .forEach( @@ -215,14 +189,25 @@ public static void testClientSetGet( } }); - printResults(calculateResults(actionResults), config.resultsFile); + var calculatedResults = calculateResults(actionResults); + if (config.resultsFile.isPresent()) { + JsonWriter.Write( + calculatedResults, + config.resultsFile.get(), + config.dataSize, + clientCreator.get().getName(), + clientCount, + concurrentNum, + iterationCounter.get() * 1e9 / (System.nanoTime() - started)); + } + printResults(calculatedResults); } } System.out.println(); } - public static Pair measurePerformance( + public static Map getActionMap( Client client, int dataSize, boolean async) { String value = RandomStringUtils.randomAlphanumeric(dataSize); @@ -251,7 +236,6 @@ public static Pair measurePerformance( .asyncSet(generateKeySet(), value) .get(ASYNC_OPERATION_TIMEOUT_SEC, TimeUnit.SECONDS) : () -> ((SyncClient) client).set(generateKeySet(), value)); - - return getLatency(actions); + return actions; } } diff --git a/java/benchmarks/src/main/java/javababushka/benchmarks/utils/JsonWriter.java b/java/benchmarks/src/main/java/javababushka/benchmarks/utils/JsonWriter.java new file mode 100644 index 0000000000..064148de51 --- /dev/null +++ b/java/benchmarks/src/main/java/javababushka/benchmarks/utils/JsonWriter.java @@ -0,0 +1,103 @@ +package javababushka.benchmarks.utils; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import lombok.Builder; +import lombok.Getter; + +public class JsonWriter { + + public static void Write( + Map calculatedResults, + String resultsFile, + int dataSize, + String client, + int clientCount, + int numOfTasks, + double tps) { + + try { + Gson gson = new GsonBuilder().setPrettyPrinting().serializeNulls().create(); + Collection recordings = new ArrayList<>(); + + Path path = Path.of(resultsFile); + if (Files.exists(path)) { + TypeToken> collectionType = new TypeToken<>() {}; + var json = new String(Files.readAllBytes(path)); + recordings = gson.fromJson(json, collectionType); + } + var data = + new Measurements.MeasurementsBuilder() + // TODO: is_cluster + .data_size(dataSize) + .client(client) + .client_count(clientCount) + .num_of_tasks(numOfTasks) + .tps(tps) + .get_existing_average_latency( + calculatedResults.get(ChosenAction.GET_EXISTING).avgLatency) + .get_existing_p50_latency(calculatedResults.get(ChosenAction.GET_EXISTING).p50Latency) + .get_existing_p90_latency(calculatedResults.get(ChosenAction.GET_EXISTING).p90Latency) + .get_existing_p99_latency(calculatedResults.get(ChosenAction.GET_EXISTING).p99Latency) + .get_existing_std_dev(calculatedResults.get(ChosenAction.GET_EXISTING).stdDeviation) + .get_non_existing_average_latency( + calculatedResults.get(ChosenAction.GET_NON_EXISTING).avgLatency) + .get_non_existing_p50_latency( + calculatedResults.get(ChosenAction.GET_NON_EXISTING).p50Latency) + .get_non_existing_p90_latency( + calculatedResults.get(ChosenAction.GET_NON_EXISTING).p90Latency) + .get_non_existing_p99_latency( + calculatedResults.get(ChosenAction.GET_NON_EXISTING).p99Latency) + .get_non_existing_std_dev( + calculatedResults.get(ChosenAction.GET_NON_EXISTING).stdDeviation) + .set_average_latency(calculatedResults.get(ChosenAction.SET).avgLatency) + .set_p50_latency(calculatedResults.get(ChosenAction.SET).p50Latency) + .set_p90_latency(calculatedResults.get(ChosenAction.SET).p90Latency) + .set_p99_latency(calculatedResults.get(ChosenAction.SET).p99Latency) + .set_std_dev(calculatedResults.get(ChosenAction.SET).stdDeviation) + .build(); + + recordings.add(data); + + Files.write(path, gson.toJson(recordings).getBytes()); + } catch (IOException e) { + System.out.printf( + "Failed to write measurement results into a file '%s': %s%n", + resultsFile, e.getMessage()); + e.printStackTrace(); + } + } + + @Getter + @Builder + public static class Measurements { + private String client; + private int client_count; + private int data_size; + private double get_existing_average_latency; + private double get_existing_p50_latency; + private double get_existing_p90_latency; + private double get_existing_p99_latency; + private double get_existing_std_dev; + private double get_non_existing_average_latency; + private double get_non_existing_p50_latency; + private double get_non_existing_p90_latency; + private double get_non_existing_p99_latency; + private double get_non_existing_std_dev; + private boolean is_cluster; + private int num_of_tasks; + private double set_average_latency; + private double set_p50_latency; + private double set_p90_latency; + private double set_p99_latency; + private double set_std_dev; + private double tps; + } +} diff --git a/java/benchmarks/src/main/java/javababushka/benchmarks/utils/LatencyResults.java b/java/benchmarks/src/main/java/javababushka/benchmarks/utils/LatencyResults.java index 8e2baa3a2b..1ae5f1fd2d 100644 --- a/java/benchmarks/src/main/java/javababushka/benchmarks/utils/LatencyResults.java +++ b/java/benchmarks/src/main/java/javababushka/benchmarks/utils/LatencyResults.java @@ -3,13 +3,17 @@ // Raw timing results in nanoseconds public class LatencyResults { public final double avgLatency; - public final long p50Latency; - public final long p90Latency; - public final long p99Latency; + public final double p50Latency; + public final double p90Latency; + public final double p99Latency; public final double stdDeviation; public LatencyResults( - double avgLatency, long p50Latency, long p90Latency, long p99Latency, double stdDeviation) { + double avgLatency, + double p50Latency, + double p90Latency, + double p99Latency, + double stdDeviation) { this.avgLatency = avgLatency; this.p50Latency = p50Latency; this.p90Latency = p90Latency; diff --git a/java/benchmarks/src/test/java/javababushka/benchmarks/jedis/JedisClientIT.java b/java/benchmarks/src/test/java/javababushka/benchmarks/jedis/JedisClientIT.java index d1669d36e0..12f800260f 100644 --- a/java/benchmarks/src/test/java/javababushka/benchmarks/jedis/JedisClientIT.java +++ b/java/benchmarks/src/test/java/javababushka/benchmarks/jedis/JedisClientIT.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import javababushka.benchmarks.utils.Benchmarking; @@ -54,5 +55,17 @@ public void testResourceSetGet() { actions.put( ChosenAction.GET_NON_EXISTING, () -> jedisClient.get(Benchmarking.generateKeyGet())); actions.put(ChosenAction.SET, () -> jedisClient.set(Benchmarking.generateKeySet(), value)); + + Map> latencies = + Map.of( + ChosenAction.GET_EXISTING, new ArrayList<>(), + ChosenAction.GET_NON_EXISTING, new ArrayList<>(), + ChosenAction.SET, new ArrayList<>()); + for (int i = 0; i < iterations; i++) { + var latency = Benchmarking.measurePerformance(actions); + latencies.get(latency.getKey()).add(latency.getValue()); + } + + Benchmarking.printResults(Benchmarking.calculateResults(latencies)); } }