From 54be502c57d1c70306e8188420c171ba26fe20c6 Mon Sep 17 00:00:00 2001 From: Alan Paxton Date: Mon, 9 Sep 2024 17:30:04 +0100 Subject: [PATCH 1/4] New Java LoggerFromOptions Logger implementation The LoggerFromOptions class is a Logger implementation which allows the placement of the logger outside the DB. Used as a logger for a ReadOnly DB on a readonly filesystem, for instance. This is the Java manifestation of CreateLoggerFromOptions - created from a DBOptions. --- java/CMakeLists.txt | 2 + java/Makefile | 1 + java/rocksjni/logger_from_options.cc | 95 +++++++++++++++++++ java/rocksjni/options.cc | 6 ++ .../src/main/java/org/rocksdb/LoggerType.java | 3 +- .../org/rocksdb/util/LoggerFromOptions.java | 40 ++++++++ .../rocksdb/util/LoggerFromOptionsTest.java | 71 ++++++++++++++ src.mk | 1 + 8 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 java/rocksjni/logger_from_options.cc create mode 100644 java/src/main/java/org/rocksdb/util/LoggerFromOptions.java create mode 100644 java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index a60847ead37..c7d4e72183f 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -50,6 +50,7 @@ set(JNI_NATIVE_SOURCES rocksjni/jni_perf_context.cc rocksjni/jnicallback.cc rocksjni/loggerjnicallback.cc + rocksjni/logger_from_options.cc rocksjni/lru_cache.cc rocksjni/memory_util.cc rocksjni/memtablejni.cc @@ -298,6 +299,7 @@ set(JAVA_MAIN_CLASSES src/main/java/org/rocksdb/util/BytewiseComparator.java src/main/java/org/rocksdb/util/Environment.java src/main/java/org/rocksdb/util/IntComparator.java + src/main/java/org/rocksdb/util/LoggerFromOptions.java src/main/java/org/rocksdb/util/ReverseBytewiseComparator.java src/main/java/org/rocksdb/util/SizeUnit.java src/main/java/org/rocksdb/util/StdErrLogger.java diff --git a/java/Makefile b/java/Makefile index 5e00921c62b..9ac0d2e5666 100644 --- a/java/Makefile +++ b/java/Makefile @@ -90,6 +90,7 @@ NATIVE_JAVA_CLASSES = \ org.rocksdb.WriteBatchWithIndex\ org.rocksdb.WriteBufferManager\ org.rocksdb.WBWIRocksIterator\ + org.rocksdb.util.LoggerFromOptions \ org.rocksdb.util.StdErrLogger NATIVE_JAVA_TEST_CLASSES = \ diff --git a/java/rocksjni/logger_from_options.cc b/java/rocksjni/logger_from_options.cc new file mode 100644 index 00000000000..3989c624111 --- /dev/null +++ b/java/rocksjni/logger_from_options.cc @@ -0,0 +1,95 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include + +#include + +#include "include/org_rocksdb_util_LoggerFromOptions.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" +#include "util/stderr_logger.h" + +/* + * Create an "independent" logger, such as might be supplied to a readonly DB + * (e.g. in a readonly filesystem) + * + * Class: org_rocksdb_util_LoggerFromOptions + * Method: newLoggerFromOptions + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_org_rocksdb_util_LoggerFromOptions_newLoggerFromOptions( + JNIEnv* env, jclass, jstring jdb_name, jlong joptions_handle) { + auto* db_options = + reinterpret_cast(joptions_handle); + + if (jdb_name == nullptr) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew( + env, + ROCKSDB_NAMESPACE::Status::InvalidArgument("Invalid (null) db name.")); + return 0; + } + jboolean has_exception = JNI_FALSE; + auto db_name = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, jdb_name, &has_exception); // also releases jlog_prefix + if (has_exception == JNI_TRUE) { + return 0; + } + + std::shared_ptr logger; + + ROCKSDB_NAMESPACE::Status s = + ROCKSDB_NAMESPACE::CreateLoggerFromOptions(db_name, *db_options, &logger); + if (!s.ok()) { + ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s); + return 0; + } + + return reinterpret_cast( + new std::shared_ptr(std::move(logger))); +} + +/* + * Class: org_rocksdb_util_LoggerFromOptions + * Method: setInfoLogLevel + * Signature: (JB)V + */ +void Java_org_rocksdb_util_LoggerFromOptions_setInfoLogLevel(JNIEnv* /*env*/, + jclass /*jcls*/, + jlong jhandle, + jbyte jlog_level) { + auto* handle = + reinterpret_cast*>(jhandle); + handle->get()->SetInfoLogLevel( + static_cast(jlog_level)); +} + +/* + * Class: org_rocksdb_util_LoggerFromOptions + * Method: infoLogLevel + * Signature: (J)B + */ +jbyte Java_org_rocksdb_util_LoggerFromOptions_infoLogLevel(JNIEnv* /*env*/, + jclass /*jcls*/, + jlong jhandle) { + auto* handle = + reinterpret_cast*>(jhandle); + return static_cast(handle->get()->GetInfoLogLevel()); +} + +/* + * Class: org_rocksdb_util_StdErrLogger + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_util_StdErrLogger_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* handle = + reinterpret_cast*>(jhandle); + delete handle; // delete std::shared_ptr +} diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index a6120f67dca..bb080515540 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -1101,6 +1101,12 @@ void Java_org_rocksdb_Options_setLogger(JNIEnv* env, jclass, jlong jhandle, *(reinterpret_cast*>( jlogger_handle)); break; + case 0x3: + // LOGGER_FROM_OPTIONS_IMPLEMENTATION is a logger created by RocksDB + options->info_log = + *(reinterpret_cast*>( + jlogger_handle)); + break; default: ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( env, ROCKSDB_NAMESPACE::Status::InvalidArgument( diff --git a/java/src/main/java/org/rocksdb/LoggerType.java b/java/src/main/java/org/rocksdb/LoggerType.java index f5d0b0d954c..8a6ec85ec1b 100644 --- a/java/src/main/java/org/rocksdb/LoggerType.java +++ b/java/src/main/java/org/rocksdb/LoggerType.java @@ -11,7 +11,8 @@ */ public enum LoggerType { JAVA_IMPLEMENTATION((byte) 0x1), - STDERR_IMPLEMENTATION((byte) 0x2); + STDERR_IMPLEMENTATION((byte) 0x2), + FROM_OPTIONS_IMPLEMENTATION((byte) 0x3); private final byte value; diff --git a/java/src/main/java/org/rocksdb/util/LoggerFromOptions.java b/java/src/main/java/org/rocksdb/util/LoggerFromOptions.java new file mode 100644 index 00000000000..ffb9e5ded83 --- /dev/null +++ b/java/src/main/java/org/rocksdb/util/LoggerFromOptions.java @@ -0,0 +1,40 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.rocksdb.*; + +/** + * + */ +public class LoggerFromOptions extends RocksObject implements LoggerInterface { + public LoggerFromOptions(final String dbName, final DBOptions dbOptions) { + super(newLoggerFromOptions(dbName, dbOptions.getNativeHandle())); + } + + @Override + public void setInfoLogLevel(InfoLogLevel logLevel) { + setInfoLogLevel(nativeHandle_, logLevel.getValue()); + } + + @Override + public InfoLogLevel infoLogLevel() { + return InfoLogLevel.getInfoLogLevel(infoLogLevel(nativeHandle_)); + } + + @Override + public LoggerType getLoggerType() { + return LoggerType.FROM_OPTIONS_IMPLEMENTATION; + } + + @Override protected native void disposeInternal(long handle); + + private static native long newLoggerFromOptions(final String dbName, final long dbOptions); + + private static native void setInfoLogLevel(final long handle, final byte logLevel); + private static native byte infoLogLevel(final long handle); +} diff --git a/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java b/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java new file mode 100644 index 00000000000..abb7447db19 --- /dev/null +++ b/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java @@ -0,0 +1,71 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.*; + +public class LoggerFromOptionsTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); + + // The log is meant to show up in a different directory from the DB.. + @Rule public TemporaryFolder logFolder = new TemporaryFolder(); + + @Test + public void openReadOnlyWithLoggerFromOptions() throws RocksDBException, IOException { + // Create the DB and close it again + try (final Options options = new Options().setCreateIfMissing(true); + final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true); + final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { + db.flush(flushOptions); + } + + LoggerFromOptions logger = + new LoggerFromOptions(logFolder.getRoot().toString(), new DBOptions()); + logger.setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL); + + // Expect these configured options to be output in the log + List remainingMatches = new ArrayList<>( + Arrays.asList("Options.max_log_file_size: 1048576", "Options.log_file_time_to_roll: 2048", + "Options.keep_log_file_num: 24", "Options.recycle_log_file_num: 8")); + + // Configure the log in order for the dump of log configuration to the log to be unique + // Open the DB readonly and give it the new logger to use + try (final Options options = new Options() + .setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL) + .setLogger(logger) + .setMaxLogFileSize(1024L * 1024L) + .setRecycleLogFileNum(8) + .setLogFileTimeToRoll(2048) + .setKeepLogFileNum(24); + final RocksDB db = RocksDB.openReadOnly(options, dbFolder.getRoot().getAbsolutePath())) { + assertThat(db.getDBOptions()).isNotNull(); + } + + // Look for evidence that the (readonly) DB has written to the logger we gave it + File[] logFiles = Objects.requireNonNull(logFolder.getRoot().listFiles()); + assertThat(logFiles.length).isEqualTo(1); + File logFile = logFiles[0]; + BufferedReader reader = new BufferedReader(new FileReader(logFile)); + reader.lines().forEach(s -> remainingMatches.removeIf(s::contains)); + assertThat(remainingMatches).as("Not all expected options have been observed in log").isEmpty(); + } +} diff --git a/src.mk b/src.mk index fe6dc248de4..4ab4265626c 100644 --- a/src.mk +++ b/src.mk @@ -689,6 +689,7 @@ JNI_NATIVE_SOURCES = \ java/rocksjni/jni_multiget_helpers.cc \ java/rocksjni/jnicallback.cc \ java/rocksjni/loggerjnicallback.cc \ + java/rocksjni/logger_from_options.cc \ java/rocksjni/lru_cache.cc \ java/rocksjni/memtablejni.cc \ java/rocksjni/memory_util.cc \ From 8bd82694e720bbf5214ef11c44bd7afbfe4ea826 Mon Sep 17 00:00:00 2001 From: Alan Paxton Date: Tue, 10 Sep 2024 10:54:15 +0100 Subject: [PATCH 2/4] Make the API more readable - Use a static method CreateLoggerFromOptions to wrap the now private LoggerFromOptionsconstructor - Class comment - change names of internal methods for readability --- java/rocksjni/logger_from_options.cc | 2 +- .../java/org/rocksdb/util/LoggerFromOptions.java | 14 ++++++++++---- .../org/rocksdb/util/LoggerFromOptionsTest.java | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/java/rocksjni/logger_from_options.cc b/java/rocksjni/logger_from_options.cc index 3989c624111..fa326ea623b 100644 --- a/java/rocksjni/logger_from_options.cc +++ b/java/rocksjni/logger_from_options.cc @@ -22,7 +22,7 @@ * Signature: (J)J */ JNIEXPORT jlong JNICALL -Java_org_rocksdb_util_LoggerFromOptions_newLoggerFromOptions( +Java_org_rocksdb_util_LoggerFromOptions_createLoggerFromOptions( JNIEnv* env, jclass, jstring jdb_name, jlong joptions_handle) { auto* db_options = reinterpret_cast(joptions_handle); diff --git a/java/src/main/java/org/rocksdb/util/LoggerFromOptions.java b/java/src/main/java/org/rocksdb/util/LoggerFromOptions.java index ffb9e5ded83..014ec8f3e6f 100644 --- a/java/src/main/java/org/rocksdb/util/LoggerFromOptions.java +++ b/java/src/main/java/org/rocksdb/util/LoggerFromOptions.java @@ -9,11 +9,17 @@ import org.rocksdb.*; /** - * + * This is a Java wrapper around a native RocksDB logger created by the + * CreateLoggerFromOptions API. */ public class LoggerFromOptions extends RocksObject implements LoggerInterface { - public LoggerFromOptions(final String dbName, final DBOptions dbOptions) { - super(newLoggerFromOptions(dbName, dbOptions.getNativeHandle())); + private LoggerFromOptions(final String dbName, final DBOptions dbOptions) { + super(createLoggerFromOptions(dbName, dbOptions.getNativeHandle())); + } + + static public LoggerFromOptions CreateLoggerFromOptions( + final String dbName, final DBOptions dbOptions) { + return new LoggerFromOptions(dbName, dbOptions); } @Override @@ -33,7 +39,7 @@ public LoggerType getLoggerType() { @Override protected native void disposeInternal(long handle); - private static native long newLoggerFromOptions(final String dbName, final long dbOptions); + private static native long createLoggerFromOptions(final String dbName, final long dbOptions); private static native void setInfoLogLevel(final long handle, final byte logLevel); private static native byte infoLogLevel(final long handle); diff --git a/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java b/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java index abb7447db19..f9797d47e6b 100644 --- a/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java +++ b/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java @@ -39,7 +39,7 @@ public void openReadOnlyWithLoggerFromOptions() throws RocksDBException, IOExcep } LoggerFromOptions logger = - new LoggerFromOptions(logFolder.getRoot().toString(), new DBOptions()); + LoggerFromOptions.CreateLoggerFromOptions(logFolder.getRoot().toString(), new DBOptions()); logger.setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL); // Expect these configured options to be output in the log From e7209275d67eeeccf5ab08061f576a4e76fa4bce Mon Sep 17 00:00:00 2001 From: Alan Paxton Date: Mon, 16 Sep 2024 14:52:33 +0100 Subject: [PATCH 3/4] Review comments for Java LoggerFromOptions Use standard macro for returning C++ pointer as Java JNIEXPORT and JNICALL not necessary in the implementation part of JNI methods, just the (autogenerated) declarations --- java/rocksjni/logger_from_options.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/java/rocksjni/logger_from_options.cc b/java/rocksjni/logger_from_options.cc index fa326ea623b..ca003f42edd 100644 --- a/java/rocksjni/logger_from_options.cc +++ b/java/rocksjni/logger_from_options.cc @@ -21,8 +21,7 @@ * Method: newLoggerFromOptions * Signature: (J)J */ -JNIEXPORT jlong JNICALL -Java_org_rocksdb_util_LoggerFromOptions_createLoggerFromOptions( +jlong Java_org_rocksdb_util_LoggerFromOptions_createLoggerFromOptions( JNIEnv* env, jclass, jstring jdb_name, jlong joptions_handle) { auto* db_options = reinterpret_cast(joptions_handle); @@ -49,7 +48,7 @@ Java_org_rocksdb_util_LoggerFromOptions_createLoggerFromOptions( return 0; } - return reinterpret_cast( + return GET_CPLUSPLUS_POINTER( new std::shared_ptr(std::move(logger))); } From e3e4609d80e2a578b5d22a650089641790b85c19 Mon Sep 17 00:00:00 2001 From: Alan Paxton Date: Wed, 2 Oct 2024 17:09:14 +0100 Subject: [PATCH 4/4] Trim log file options sizes in test --- .../java/org/rocksdb/util/LoggerFromOptionsTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java b/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java index f9797d47e6b..9a34e0f5d94 100644 --- a/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java +++ b/java/src/test/java/org/rocksdb/util/LoggerFromOptionsTest.java @@ -44,18 +44,18 @@ public void openReadOnlyWithLoggerFromOptions() throws RocksDBException, IOExcep // Expect these configured options to be output in the log List remainingMatches = new ArrayList<>( - Arrays.asList("Options.max_log_file_size: 1048576", "Options.log_file_time_to_roll: 2048", - "Options.keep_log_file_num: 24", "Options.recycle_log_file_num: 8")); + Arrays.asList("Options.max_log_file_size: 65536", "Options.log_file_time_to_roll: 2048", + "Options.keep_log_file_num: 8", "Options.recycle_log_file_num: 3")); // Configure the log in order for the dump of log configuration to the log to be unique // Open the DB readonly and give it the new logger to use try (final Options options = new Options() .setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL) .setLogger(logger) - .setMaxLogFileSize(1024L * 1024L) - .setRecycleLogFileNum(8) + .setMaxLogFileSize(1024L * 64L) + .setRecycleLogFileNum(3) .setLogFileTimeToRoll(2048) - .setKeepLogFileNum(24); + .setKeepLogFileNum(8); final RocksDB db = RocksDB.openReadOnly(options, dbFolder.getRoot().getAbsolutePath())) { assertThat(db.getDBOptions()).isNotNull(); }