Squashed commit of sigp#2222

commit 13adea78354beb4ff406bd8d5748b560feb37042 Merge: 345802f a764c3b Author: Paul Hauner <[email protected]> Date: Wed Feb 24 16:21:22 2021 +1100 Merge branch 'unstable' into jemalloc-paul commit 345802f Author: Paul Hauner <[email protected]> Date: Wed Feb 24 15:58:31 2021 +1100 Jemalloc by default commit 88e54ab Author: Paul Hauner <[email protected]> Date: Wed Feb 24 15:41:35 2021 +1100 Fix config commit 6680b6d Merge: 42eb539 46920a8 Author: Paul Hauner <[email protected]> Date: Wed Feb 24 14:21:45 2021 +1100 Merge branch 'unstable' into jemalloc-sea commit 42eb539 Author: Paul Hauner <[email protected]> Date: Tue Feb 16 12:12:34 2021 +1100 Set jemalloc as default, configure arenas commit 4331ef8 Merge: d4010e3 4a357c9 Author: Paul Hauner <[email protected]> Date: Tue Feb 16 10:00:47 2021 +1100 Merge branch 'unstable' into jemalloc-sea commit d4010e3 Author: Paul Hauner <[email protected]> Date: Tue Feb 16 10:00:37 2021 +1100 Make uses jemmaloc commit 5e968df Merge: ff14ee0 e2ff9c6 Author: realbigsean <[email protected]> Date: Fri Feb 12 10:36:49 2021 -0500 Merge branch 'unstable' of https://github.com/sigp/lighthouse into jemalloc � Conflicts: � Cargo.lock � beacon_node/http_metrics/Cargo.toml � beacon_node/network/src/attestation_service/tests/mod.rs commit ff14ee0 Author: realbigsean <[email protected]> Date: Fri Feb 12 10:35:18 2021 -0500 add newlines commit ce1c4e1 Author: realbigsean <[email protected]> Date: Fri Feb 12 09:54:19 2021 -0500 move jemalloc to a feature commit a0d7119 Author: realbigsean <[email protected]> Date: Thu Feb 11 19:12:13 2021 -0500 cargo lock commit f205c18 Author: realbigsean <[email protected]> Date: Thu Feb 11 18:53:43 2021 -0500 delete unused dep commit fd17192 Author: realbigsean <[email protected]> Date: Thu Feb 11 18:51:25 2021 -0500 delete background threads commit ae8b61a Author: realbigsean <[email protected]> Date: Thu Feb 11 18:50:55 2021 -0500 fix error handling and remove background threads commit 9e7a6b7 Author: realbigsean <[email protected]> Date: Thu Feb 11 11:48:25 2021 -0500 semicolon commit 3408ee1 Author: realbigsean <[email protected]> Date: Thu Feb 11 11:46:40 2021 -0500 update to associated function commit 074ce1f Author: realbigsean <[email protected]> Date: Thu Feb 11 11:43:25 2021 -0500 enable background threads commit 8805ef9 Author: realbigsean <[email protected]> Date: Thu Feb 11 10:34:47 2021 -0500 fix arenas commit 6f17971 Author: realbigsean <[email protected]> Date: Thu Feb 11 10:25:09 2021 -0500 import arenas commit 034b677 Author: realbigsean <[email protected]> Date: Thu Feb 11 10:23:12 2021 -0500 add more jemalloc stats commit 632bff2 Author: realbigsean <[email protected]> Date: Wed Feb 10 19:11:30 2021 -0500 test add jemalloc metrics commit abf5cac Author: realbigsean <[email protected]> Date: Wed Feb 10 19:03:08 2021 -0500 test add jemalloc metrics commit 1d1fc6a Author: realbigsean <[email protected]> Date: Wed Feb 10 18:48:48 2021 -0500 add jemalloc deps commit 3c7920b Author: realbigsean <[email protected]> Date: Wed Feb 10 18:34:18 2021 -0500 test add jemalloc metrics commit db6fda6 Author: realbigsean <[email protected]> Date: Wed Feb 10 18:14:22 2021 -0500 test add jemalloc
paulhauner · Mar 30, 2021 · 954f96f · 954f96f
1 parent 3d239b8
commit 954f96f
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 0 deletions.
diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml
@@ -13,6 +13,8 @@ node_test_rig = { path = "../testing/node_test_rig" }
 
 [features]
 write_ssz_files = ["beacon_chain/write_ssz_files"]  # Writes debugging .ssz files to /tmp during block processing.
+# Use the system allocator instead of jemalloc. This replicates default Rust behaviour.
+sysalloc = ["client/sysalloc"]
 
 [dependencies]
 eth2_config = { path = "../common/eth2_config" }

diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml
@@ -44,3 +44,7 @@ http_api = { path = "../http_api" }
 http_metrics = { path = "../http_metrics" }
 slasher = { path = "../../slasher" }
 slasher_service = { path = "../../slasher/service" }
+
+[features]
+# Use the system allocator instead of jemalloc. This replicates default Rust behaviour.
+sysalloc = ["http_metrics/sysalloc"]
diff --git a/beacon_node/http_metrics/Cargo.toml b/beacon_node/http_metrics/Cargo.toml
@@ -26,3 +26,7 @@ tokio = { version = "1.1.0", features = ["sync"] }
 reqwest = { version = "0.11.0", features = ["json"] }
 environment = { path = "../../lighthouse/environment" }
 types = { path = "../../consensus/types" }
+
+[features]
+# Use the system allocator instead of jemalloc. This replicates default Rust behaviour.
+sysalloc = ["warp_utils/sysalloc"]
diff --git a/common/warp_utils/Cargo.toml b/common/warp_utils/Cargo.toml
@@ -18,3 +18,8 @@ tokio = { version = "1.1.0", features = ["sync"] }
 headers = "0.3.2"
 lighthouse_metrics = { path = "../lighthouse_metrics" }
 lazy_static = "1.4.0"
+jemalloc-ctl = { version = "0.3.3" }
+
+[features]
+# Use the system allocator instead of jemalloc. This replicates default Rust behaviour.
+sysalloc = []
diff --git a/common/warp_utils/src/metrics.rs b/common/warp_utils/src/metrics.rs
@@ -1,6 +1,9 @@
 use eth2::lighthouse::Health;
 use lighthouse_metrics::*;
 
+#[cfg(not(feature = "sysalloc"))]
+use jemalloc_ctl::{arenas, epoch, stats};
+
 lazy_static::lazy_static! {
     pub static ref PROCESS_NUM_THREADS: Result<IntGauge> = try_create_int_gauge(
         "process_num_threads",
@@ -34,6 +37,24 @@ lazy_static::lazy_static! {
         try_create_float_gauge("system_loadavg_5", "Loadavg over 5 minutes");
     pub static ref SYSTEM_LOADAVG_15: Result<Gauge> =
         try_create_float_gauge("system_loadavg_15", "Loadavg over 15 minutes");
+
+    /*
+     * jemalloc
+     */
+    pub static ref JEMALLOC_RESIDENT: Result<IntGauge> =
+        try_create_int_gauge("jemalloc_resident", "Total number of bytes in physically resident data pages mapped by the allocator.");
+    pub static ref JEMALLOC_ALLOCATED: Result<IntGauge> =
+        try_create_int_gauge("jemalloc_allocated", "Total number of bytes allocated by the application.");
+    pub static ref JEMALLOC_MAPPED: Result<IntGauge> =
+        try_create_int_gauge("jemalloc_mapped", "Total number of bytes in active extents mapped by the allocator.");
+    pub static ref JEMALLOC_METADATA: Result<IntGauge> =
+        try_create_int_gauge("jemalloc_metadata", "Total number of bytes dedicated to jemalloc metadata.");
+    pub static ref JEMALLOC_RETAINED: Result<IntGauge> =
+        try_create_int_gauge("jemalloc_retained", "Total number of bytes in virtual memory mappings that were retained rather than being returned to the operating system.");
+    pub static ref JEMALLOC_ACTIVE: Result<IntGauge> =
+        try_create_int_gauge("jemalloc_active", "Total number of bytes in active pages allocated by the application.");
+    pub static ref JEMALLOC_ARENAS: Result<IntGauge> =
+        try_create_int_gauge("jemalloc_arenas", "Current limit on the number of arenas.");
 }
 
 pub fn scrape_health_metrics() {
@@ -58,4 +79,38 @@ pub fn scrape_health_metrics() {
         set_float_gauge(&SYSTEM_LOADAVG_5, health.sys_loadavg_5);
         set_float_gauge(&SYSTEM_LOADAVG_15, health.sys_loadavg_15);
     }
+
+    scrape_jemalloc_metrics();
+}
+
+#[cfg(not(feature = "sysalloc"))]
+pub fn scrape_jemalloc_metrics() {
+    if epoch::advance().is_ok() {
+        if let Ok(allocated) = stats::allocated::read() {
+            set_gauge(&JEMALLOC_ALLOCATED, allocated as i64);
+        }
+        if let Ok(resident) = stats::resident::read() {
+            set_gauge(&JEMALLOC_RESIDENT, resident as i64);
+        }
+        if let Ok(mapped) = stats::mapped::read() {
+            set_gauge(&JEMALLOC_MAPPED, mapped as i64);
+        }
+        if let Ok(metadata) = stats::metadata::read() {
+            set_gauge(&JEMALLOC_METADATA, metadata as i64);
+        }
+        if let Ok(retained) = stats::retained::read() {
+            set_gauge(&JEMALLOC_RETAINED, retained as i64);
+        }
+        if let Ok(active) = stats::active::read() {
+            set_gauge(&JEMALLOC_ACTIVE, active as i64);
+        }
+        if let Ok(narenas) = arenas::narenas::read() {
+            set_gauge(&JEMALLOC_ARENAS, narenas as i64);
+        }
+    }
+}
+
+#[cfg(feature = "sysalloc")]
+pub fn scrape_jemalloc_metrics() {
+    // NO OP
 }
diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml
@@ -17,6 +17,8 @@ milagro = ["bls/milagro"]
 spec-minimal = []
 # Support spec v0.12 (used by Medalla testnet).
 spec-v12 = []
+# Use the system allocator instead of jemalloc. This replicates default Rust behaviour.
+sysalloc = ["beacon_node/sysalloc"]
 
 [dependencies]
 beacon_node = { "path" = "../beacon_node" }
@@ -43,6 +45,9 @@ account_utils = { path = "../common/account_utils" }
 remote_signer = { "path" = "../remote_signer" }
 lighthouse_metrics = { path = "../common/lighthouse_metrics" }
 lazy_static = "1.4.0"
+jemallocator = { version = "0.3.2" }
+jemalloc-sys = { version = "0.3.2" }
+libc = { version = "0.2.86" }
 
 [dev-dependencies]
 tempfile = "3.1.0"

diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs
@@ -12,6 +12,27 @@ use std::process::exit;
 use types::{EthSpec, EthSpecId};
 use validator_client::ProductionValidatorClient;
 
+/// Global allocator
+#[cfg(not(feature = "sysalloc"))]
+#[global_allocator]
+pub static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
+#[cfg(not(feature = "sysalloc"))]
+union U {
+    x: &'static u8,
+    y: &'static libc::c_char,
+}
+
+#[cfg(not(feature = "sysalloc"))]
+#[allow(non_upper_case_globals)]
+#[export_name = "_rjem_malloc_conf"]
+pub static malloc_conf: Option<&'static libc::c_char> = Some(unsafe {
+    U {
+        x: &b"narenas:1\0"[0],
+    }
+    .y
+});
+
 pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";
 
 fn bls_library_name() -> &'static str {