Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add json reporting. #10

Merged
merged 5 commits into from
Oct 20, 2023
Merged

Add json reporting. #10

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b243d11
Add JSON reporting.
Yury-Fridlyand Oct 17, 2023
0bf99f9
Fix for #26.
Yury-Fridlyand Oct 17, 2023
07d7aff
Update java/benchmarks/src/main/java/javababushka/benchmarks/utils/Be…
Yury-Fridlyand Oct 17, 2023
b60cb25
Use `Optional`.
Yury-Fridlyand Oct 19, 2023
e6ef449
Address PR feedback.
Yury-Fridlyand Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion benchmarks/utilities/csv_exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@

json_file_name = os.path.basename(json_file_full_path)

languages = ["csharp", "node", "python", "rust"]
languages = ["csharp", "node", "python", "rust", "java"]
language = next(
(language for language in languages if language in json_file_name), None
)
Expand Down
4 changes: 4 additions & 0 deletions java/benchmarks/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ dependencies {
implementation 'io.lettuce:lettuce-core:6.2.6.RELEASE'
implementation 'commons-cli:commons-cli:1.5.0'
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.13.0'
implementation group: 'com.google.code.gson', name: 'gson', version: '2.10.1'

compileOnly 'org.projectlombok:lombok:1.18.30'
acarbonetto marked this conversation as resolved.
Show resolved Hide resolved
annotationProcessor 'org.projectlombok:lombok:1.18.30'
}

// Apply a specific Java toolchain to ease working on different environments.
Expand Down
22 changes: 3 additions & 19 deletions java/benchmarks/src/main/java/javababushka/benchmarks/BenchmarkingApp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,8 @@

import static javababushka.benchmarks.utils.Benchmarking.testClientSetGet;

import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javababushka.benchmarks.jedis.JedisClient;
Expand Down Expand Up @@ -57,14 +54,6 @@ public static void main(String[] args) {
break;
}
}

if (runConfiguration.resultsFile.isPresent()) {
try {
runConfiguration.resultsFile.get().close();
} catch (IOException ioException) {
System.out.println("Error closing results file");
}
}
}

private static Options getOptions() {
Expand Down Expand Up @@ -97,12 +86,7 @@ private static RunConfiguration verifyOptions(CommandLine line) throws ParseExce
}

if (line.hasOption("resultsFile")) {
try {
runConfiguration.resultsFile =
Optional.of(new FileWriter(line.getOptionValue("resultsFile")));
} catch (IOException e) {
throw new ParseException("Unable to write to resultsFile.");
}
runConfiguration.resultsFile = line.getOptionValue("resultsFile");
}

if (line.hasOption("dataSize")) {
Expand Down Expand Up @@ -210,8 +194,8 @@ public boolean isEqual(String other) {

public static class RunConfiguration {
public String configuration;
public Optional<FileWriter> resultsFile;
public int dataSize;
public String resultsFile;
public List<Integer> concurrentTasks;
public ClientName[] clients;
public String host;
Expand All @@ -222,7 +206,7 @@ public static class RunConfiguration {

public RunConfiguration() {
configuration = "Release";
resultsFile = Optional.empty();
resultsFile = null;
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd rather not support nulls anywhere. If we support null, we need null checkers each and every time we use this varilable.
I'd prefer to use the empty string or Optionals to support stdout.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in b60cb25.

dataSize = 20;
concurrentTasks = List.of(10, 100);
clients =
Expand Down
72 changes: 28 additions & 44 deletions java/benchmarks/src/main/java/javababushka/benchmarks/utils/Benchmarking.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,15 @@
package javababushka.benchmarks.utils;

import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javababushka.benchmarks.AsyncClient;
import javababushka.benchmarks.BenchmarkingApp;
import javababushka.benchmarks.Client;
Expand All @@ -26,6 +23,8 @@ public class Benchmarking {
static final int SIZE_GET_KEYSPACE = 3750000;
static final int SIZE_SET_KEYSPACE = 3000000;
static final int ASYNC_OPERATION_TIMEOUT_SEC = 1;
// measurements are done in nanosec, but it should be converted to seconts later
Yury-Fridlyand marked this conversation as resolved.
Show resolved Hide resolved
static final double SECONDS_IN_NANO = 1e-9;

private static ChosenAction randomAction() {
if (Math.random() > PROB_GET) {
Expand All @@ -50,7 +49,7 @@ public interface Operation {
void go() throws Exception;
}

private static Pair<ChosenAction, Long> getLatency(Map<ChosenAction, Operation> actions) {
public static Pair<ChosenAction, Long> measurePerformance(Map<ChosenAction, Operation> actions) {
var action = randomAction();
long before = System.nanoTime();
try {
Expand Down Expand Up @@ -84,56 +83,29 @@ private static double stdDeviation(ArrayList<Long> latencies, Double avgLatency)
// This has the side-effect of sorting each latencies ArrayList
public static Map<ChosenAction, LatencyResults> calculateResults(
Map<ChosenAction, ArrayList<Long>> actionLatencies) {
Map<ChosenAction, LatencyResults> results = new HashMap<ChosenAction, LatencyResults>();
Map<ChosenAction, LatencyResults> results = new HashMap<>();

for (Map.Entry<ChosenAction, ArrayList<Long>> entry : actionLatencies.entrySet()) {
ChosenAction action = entry.getKey();
ArrayList<Long> latencies = entry.getValue();

Double avgLatency =
latencies.stream().collect(Collectors.summingLong(Long::longValue))
/ Double.valueOf(latencies.size());
double avgLatency =
SECONDS_IN_NANO * latencies.stream().mapToLong(Long::longValue).sum() / latencies.size();

Collections.sort(latencies);
results.put(
action,
new LatencyResults(
avgLatency,
percentile(latencies, 50),
percentile(latencies, 90),
percentile(latencies, 99),
stdDeviation(latencies, avgLatency)));
SECONDS_IN_NANO * percentile(latencies, 50),
SECONDS_IN_NANO * percentile(latencies, 90),
SECONDS_IN_NANO * percentile(latencies, 99),
SECONDS_IN_NANO * stdDeviation(latencies, avgLatency)));
}

return results;
}

public static void printResults(
Map<ChosenAction, LatencyResults> calculatedResults, Optional<FileWriter> resultsFile) {
if (resultsFile.isPresent()) {
printResults(calculatedResults, resultsFile.get());
} else {
printResults(calculatedResults);
}
}

public static void printResults(
Map<ChosenAction, LatencyResults> resultsMap, FileWriter resultsFile) {
for (Map.Entry<ChosenAction, LatencyResults> entry : resultsMap.entrySet()) {
ChosenAction action = entry.getKey();
LatencyResults results = entry.getValue();

try {
resultsFile.write("Avg. time in ms per " + action + ": " + results.avgLatency / 1000000.0);
resultsFile.write(action + " p50 latency in ms: " + results.p50Latency / 1000000.0);
resultsFile.write(action + " p90 latency in ms: " + results.p90Latency / 1000000.0);
resultsFile.write(action + " p99 latency in ms: " + results.p99Latency / 1000000.0);
resultsFile.write(action + " std dev in ms: " + results.stdDeviation / 1000000.0);
} catch (Exception ignored) {
}
}
}

public static void printResults(Map<ChosenAction, LatencyResults> resultsMap) {
for (Map.Entry<ChosenAction, LatencyResults> entry : resultsMap.entrySet()) {
ChosenAction action = entry.getKey();
Expand Down Expand Up @@ -189,9 +161,10 @@ public static void testClientSetGet(
"> iteration = %d/%d, client# = %d/%d%n",
iterationIncrement + 1, iterations, clientIndex + 1, clientCount);
}

var actions = getActionMap(clients.get(clientIndex), config.dataSize, async);
// operate and calculate tik-tok
Pair<ChosenAction, Long> result =
measurePerformance(clients.get(clientIndex), config.dataSize, async);
Pair<ChosenAction, Long> result = measurePerformance(actions);
actionResults.get(result.getLeft()).add(result.getRight());

iterationIncrement = iterationCounter.getAndIncrement();
Expand All @@ -204,6 +177,7 @@ public static void testClientSetGet(
"===> concurrentNum = %d, clientNum = %d, tasks = %d%n",
concurrentNum, clientCount, tasks.size());
}
long started = System.nanoTime();
tasks.stream()
.map(CompletableFuture::runAsync)
.forEach(
Expand All @@ -215,14 +189,25 @@ public static void testClientSetGet(
}
});

printResults(calculateResults(actionResults), config.resultsFile);
var calculatedResults = calculateResults(actionResults);
if (config.resultsFile != null) {
JsonWriter.WriteJson(
calculatedResults,
config.resultsFile,
config.dataSize,
clientCreator.get().getName(),
clientNum,
concurrentNum,
iterationCounter.get() * 1e9 / (System.nanoTime() - started));
}
printResults(calculatedResults);
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we want to print to stdout each time?

}
}

System.out.println();
}

public static Pair<ChosenAction, Long> measurePerformance(
public static Map<ChosenAction, Operation> getActionMap(
Client client, int dataSize, boolean async) {

String value = RandomStringUtils.randomAlphanumeric(dataSize);
Expand Down Expand Up @@ -251,7 +236,6 @@ public static Pair<ChosenAction, Long> measurePerformance(
.asyncSet(generateKeySet(), value)
.get(ASYNC_OPERATION_TIMEOUT_SEC, TimeUnit.SECONDS)
: () -> ((SyncClient) client).set(generateKeySet(), value));

return getLatency(actions);
return actions;
}
}
102 changes: 102 additions & 0 deletions java/benchmarks/src/main/java/javababushka/benchmarks/utils/JsonWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package javababushka.benchmarks.utils;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import lombok.Getter;

public class JsonWriter {

public static void WriteJson(
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Write?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in e6ef449.

Map<ChosenAction, LatencyResults> calculatedResults,
String resultsFile,
int dataSize,
String client,
int clientCount,
int numOfTasks,
double tps) {

try {
Gson gson = new GsonBuilder().setPrettyPrinting().serializeNulls().create();
Collection<Measurements> recordings = new ArrayList<>();

Path path = Path.of(resultsFile);
if (Files.exists(path)) {
TypeToken<Collection<Measurements>> collectionType = new TypeToken<>() {};
var json = new String(Files.readAllBytes(path));
recordings = gson.fromJson(json, collectionType);
}
var data = new Measurements();
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe create a constructor method to do everything below?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in e6ef449.

data.data_size = dataSize;
data.client = client;
data.client_count = clientCount;
data.num_of_tasks = numOfTasks;
data.tps = tps;
// TODO: is_cluster
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should raise a ticket to do this :P


data.get_existing_average_latency =
calculatedResults.get(ChosenAction.GET_EXISTING).avgLatency;
data.get_existing_p50_latency = calculatedResults.get(ChosenAction.GET_EXISTING).p50Latency;
data.get_existing_p90_latency = calculatedResults.get(ChosenAction.GET_EXISTING).p90Latency;
data.get_existing_p99_latency = calculatedResults.get(ChosenAction.GET_EXISTING).p99Latency;
data.get_existing_std_dev = calculatedResults.get(ChosenAction.GET_EXISTING).stdDeviation;

data.get_non_existing_average_latency =
calculatedResults.get(ChosenAction.GET_NON_EXISTING).avgLatency;
data.get_non_existing_p50_latency =
calculatedResults.get(ChosenAction.GET_NON_EXISTING).p50Latency;
data.get_non_existing_p90_latency =
calculatedResults.get(ChosenAction.GET_NON_EXISTING).p90Latency;
data.get_non_existing_p99_latency =
calculatedResults.get(ChosenAction.GET_NON_EXISTING).p99Latency;
data.get_non_existing_std_dev =
calculatedResults.get(ChosenAction.GET_NON_EXISTING).stdDeviation;

data.set_average_latency = calculatedResults.get(ChosenAction.SET).avgLatency;
data.set_p50_latency = calculatedResults.get(ChosenAction.SET).p50Latency;
data.set_p90_latency = calculatedResults.get(ChosenAction.SET).p90Latency;
data.set_p99_latency = calculatedResults.get(ChosenAction.SET).p99Latency;
data.set_std_dev = calculatedResults.get(ChosenAction.SET).stdDeviation;

recordings.add(data);

Files.write(path, gson.toJson(recordings).getBytes());
} catch (IOException e) {
System.out.printf(
"Failed to write measurement results into a file '%s': %s%n",
resultsFile, e.getMessage());
e.printStackTrace();
}
}

@Getter
public static class Measurements {
private String client;
private int client_count;
private int data_size;
private double get_existing_average_latency;
private double get_existing_p50_latency;
private double get_existing_p90_latency;
private double get_existing_p99_latency;
private double get_existing_std_dev;
private double get_non_existing_average_latency;
private double get_non_existing_p50_latency;
private double get_non_existing_p90_latency;
private double get_non_existing_p99_latency;
private double get_non_existing_std_dev;
private boolean is_cluster;
private int num_of_tasks;
private double set_average_latency;
private double set_p50_latency;
private double set_p90_latency;
private double set_p99_latency;
private double set_std_dev;
private double tps;
}
}
12 changes: 8 additions & 4 deletions java/benchmarks/src/main/java/javababushka/benchmarks/utils/LatencyResults.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,17 @@
// Raw timing results in nanoseconds
public class LatencyResults {
public final double avgLatency;
public final long p50Latency;
public final long p90Latency;
public final long p99Latency;
public final double p50Latency;
public final double p90Latency;
public final double p99Latency;
public final double stdDeviation;

public LatencyResults(
double avgLatency, long p50Latency, long p90Latency, long p99Latency, double stdDeviation) {
double avgLatency,
double p50Latency,
double p90Latency,
double p99Latency,
double stdDeviation) {
this.avgLatency = avgLatency;
this.p50Latency = p50Latency;
this.p90Latency = p90Latency;
Expand Down
13 changes: 13 additions & 0 deletions java/benchmarks/src/test/java/javababushka/benchmarks/jedis/JedisClientIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import javababushka.benchmarks.utils.Benchmarking;
Expand Down Expand Up @@ -54,5 +55,17 @@ public void testResourceSetGet() {
actions.put(
ChosenAction.GET_NON_EXISTING, () -> jedisClient.get(Benchmarking.generateKeyGet()));
actions.put(ChosenAction.SET, () -> jedisClient.set(Benchmarking.generateKeySet(), value));

Map<ChosenAction, ArrayList<Long>> latencies =
Map.of(
ChosenAction.GET_EXISTING, new ArrayList<>(),
ChosenAction.GET_NON_EXISTING, new ArrayList<>(),
ChosenAction.SET, new ArrayList<>());
for (int i = 0; i < iterations; i++) {
var latency = Benchmarking.measurePerformance(actions);
latencies.get(latency.getKey()).add(latency.getValue());
}

Benchmarking.printResults(Benchmarking.calculateResults(latencies));
}
}
Loading