Skip to content

Commit

Permalink
Add metrics to the database
Browse files Browse the repository at this point in the history
  • Loading branch information
paberr committed Jun 18, 2024
1 parent 21d91b3 commit a8a39ea
Show file tree
Hide file tree
Showing 20 changed files with 1,220 additions and 163 deletions.
33 changes: 27 additions & 6 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

126 changes: 63 additions & 63 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,65 +3,65 @@
## !!! NOTE: If you add a nimiq module here, also add it to log/src/lib.rs !!!
##
members = [
"blockchain",
"blockchain-interface",
"blockchain-proxy",
"bls",
"client",
"collections",
"consensus",
"database",
"database/database-value",
"genesis",
"genesis-builder",
"handel",
"hash",
"hash/hash_derive",
"key-derivation",
"keys",
"lib",
"light-blockchain",
"log",
"macros",
"mempool",
"metrics-server",
"mnemonic",
"network-interface",
"network-libp2p",
"network-mock",
"pow-migration",
"primitives",
"primitives/account",
"primitives/block",
"primitives/mmr",
"primitives/subscription",
"primitives/transaction",
"primitives/trie",
"rpc-client",
"rpc-interface",
"rpc-server",
"serde",
"serde/derive",
"spammer",
"tendermint",
"test-log",
"test-log/proc-macro",
"test-utils",
"time",
"tools",
"transaction-builder",
"utils",
"validator",
"validator-network",
"vrf",
"wallet",
"web-client",
"zkp",
"zkp-circuits",
"zkp-component",
"zkp-component/zkp-test-gen",
"zkp-primitives",
"zkp-primitives/pedersen-generators",
"blockchain",
"blockchain-interface",
"blockchain-proxy",
"bls",
"client",
"collections",
"consensus",
"database",
"database/database-value",
"genesis",
"genesis-builder",
"handel",
"hash",
"hash/hash_derive",
"key-derivation",
"keys",
"lib",
"light-blockchain",
"log",
"macros",
"mempool",
"metrics-server",
"mnemonic",
"network-interface",
"network-libp2p",
"network-mock",
"pow-migration",
"primitives",
"primitives/account",
"primitives/block",
"primitives/mmr",
"primitives/subscription",
"primitives/transaction",
"primitives/trie",
"rpc-client",
"rpc-interface",
"rpc-server",
"serde",
"serde/derive",
"spammer",
"tendermint",
"test-log",
"test-log/proc-macro",
"test-utils",
"time",
"tools",
"transaction-builder",
"utils",
"validator",
"validator-network",
"vrf",
"wallet",
"web-client",
"zkp",
"zkp-circuits",
"zkp-component",
"zkp-component/zkp-test-gen",
"zkp-primitives",
"zkp-primitives/pedersen-generators",
]

resolver = "2"
Expand Down Expand Up @@ -107,8 +107,8 @@ lto = "thin"

[profile.release-wasm]
inherits = "release"
lto = "fat" # Same as lto = true
opt-level = "s" # Optimize for size
lto = "fat" # Same as lto = true
opt-level = "s" # Optimize for size
strip = "debuginfo"

[profile.release-with-debug]
Expand Down Expand Up @@ -162,8 +162,8 @@ categories = ["cryptography::cryptocurrencies"]
keywords = ["nimiq", "cryptocurrency", "blockchain"]

[workspace.lints]
clippy.assigning_clones = "allow" # false positives: https://github.com/rust-lang/rust-clippy/issues/12709
clippy.empty_docs = "allow" # false positives: https://github.com/rust-lang/rust-clippy/issues/12377
clippy.assigning_clones = "allow" # false positives: https://github.com/rust-lang/rust-clippy/issues/12709
clippy.empty_docs = "allow" # false positives: https://github.com/rust-lang/rust-clippy/issues/12377
clippy.large_enum_variant = "allow"
clippy.too_many_arguments = "allow"
clippy.type_complexity = "allow"
Expand Down
11 changes: 9 additions & 2 deletions client/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use std::time::Duration;

use log::info;
use nimiq::prover::prover_main;
pub use nimiq::{
client::Client,
config::{command_line::CommandLine, config::ClientConfig, config_file::ConfigFile},
Expand All @@ -13,6 +12,7 @@ pub use nimiq::{
signal_handling::initialize_signal_handler,
},
};
use nimiq::{extras::metrics_server::install_metrics, prover::prover_main};

async fn main_inner() -> Result<(), Error> {
// Keep for potential future reactivation
Expand All @@ -39,7 +39,7 @@ async fn main_inner() -> Result<(), Error> {
// Initialize panic hook.
initialize_panic_reporting();

// Initialize signal handler
// Initialize signal handler.
initialize_signal_handler();

// Early return in case of a proving process.
Expand All @@ -63,6 +63,12 @@ async fn main_inner() -> Result<(), Error> {
let metrics_config = config.metrics_server.clone();
let metrics_enabled = metrics_config.is_some();

// Initialize database logging.
let mut metrics_collector = None;
if metrics_enabled {
metrics_collector = Some(install_metrics());
}

// Create client from config.
let mut client: Client = Client::from_config(config).await?;

Expand Down Expand Up @@ -128,6 +134,7 @@ async fn main_inner() -> Result<(), Error> {
client.consensus_proxy(),
client.network(),
&nimiq_task_metric,
metrics_collector.unwrap(),
)
}

Expand Down
12 changes: 10 additions & 2 deletions database/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
[package]
name = "nimiq-database"
version.workspace = true
authors = ["Pascal B <[email protected]>", "The Nimiq Core Development Team <[email protected]>"]
authors = [
"Pascal B <[email protected]>",
"The Nimiq Core Development Team <[email protected]>",
]
license.workspace = true
edition.workspace = true
description = "A LMDB database wrapper with support for volatile storage"
Expand All @@ -21,10 +24,15 @@ workspace = true

[dependencies]
bitflags = "2.5"
libmdbx = "0.5.0"
libmdbx = { git = "https://github.com/paberr/libmdbx-rs" }
log = { workspace = true }
tempfile = "3"
thiserror = "1.0"
metrics = "0.23"
parking_lot = "0.12"
rustc-hash = "1.1"
strum = "0.26"
strum_macros = "0.26"

nimiq-database-value = { workspace = true }

Expand Down
1 change: 1 addition & 0 deletions database/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use bitflags::bitflags;

mod error;
pub mod mdbx;
pub(crate) mod metrics;
/// Database implementation that can handle volatile and persistent storage.
pub mod proxy;
/// Abstraction for methods related to the database.
Expand Down
44 changes: 39 additions & 5 deletions database/src/mdbx/cursor.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
use std::{borrow::Cow, marker::PhantomData};
use std::{borrow::Cow, marker::PhantomData, sync::Arc};

use libmdbx::{TransactionKind, WriteFlags, RO, RW};
use nimiq_database_value::{AsDatabaseBytes, FromDatabaseValue};

use super::{DbKvPair, IntoIter};
use crate::traits::{ReadCursor, WriteCursor};
use crate::{
metrics::{DatabaseEnvMetrics, Operation},
traits::{ReadCursor, WriteCursor},
};

/// A cursor for navigating the entries within a table.
/// Wraps the libmdbx cursor so that we only expose our own methods.
pub struct MdbxCursor<'txn, K: TransactionKind> {
cursor: libmdbx::Cursor<'txn, K>,
metrics: Option<Arc<DatabaseEnvMetrics>>,
table_name: String,
}
/// Instantiation of the `MdbxCursor` for read transactions.
pub type MdbxReadCursor<'txn> = MdbxCursor<'txn, RO>;
Expand All @@ -20,8 +25,33 @@ impl<'txn, Kind> MdbxCursor<'txn, Kind>
where
Kind: TransactionKind,
{
pub(crate) fn new(cursor: libmdbx::Cursor<'txn, Kind>) -> Self {
MdbxCursor { cursor }
pub(crate) fn new(
table_name: &str,
cursor: libmdbx::Cursor<'txn, Kind>,
metrics: Option<Arc<DatabaseEnvMetrics>>,
) -> Self {
MdbxCursor {
table_name: table_name.to_string(),
cursor,
metrics,
}
}

/// If `self.metrics` is `Some(...)`, record a metric with the provided operation and value
/// size.
///
/// Otherwise, just execute the closure.
fn execute_with_operation_metric<R>(
&mut self,
operation: Operation,
value_size: Option<usize>,
f: impl FnOnce(&mut Self) -> R,
) -> R {
if let Some(metrics) = self.metrics.as_ref().cloned() {
metrics.record_operation(&self.table_name.clone(), operation, value_size, || f(self))
} else {
f(self)
}
}
}

Expand Down Expand Up @@ -246,13 +276,17 @@ where
{
fn clone(&self) -> Self {
Self {
table_name: self.table_name.clone(),
cursor: self.cursor.clone(),
metrics: self.metrics.as_ref().cloned(),
}
}
}

impl<'txn> WriteCursor<'txn> for MdbxWriteCursor<'txn> {
fn remove(&mut self) {
self.cursor.del(WriteFlags::empty()).unwrap();
self.execute_with_operation_metric(Operation::CursorDeleteCurrent, None, |cursor| {
cursor.cursor.del(WriteFlags::empty()).unwrap();
});
}
}
Loading

0 comments on commit a8a39ea

Please sign in to comment.