dev-libs/jemalloc: add je_get_defrag_hint() function

adjust · May 2, 2024 · 8e95ace · 8e95ace
1 parent 70f9dd4
commit 8e95ace
Show file tree

Hide file tree

Showing 6 changed files with 263 additions and 0 deletions.
diff --git a/dev-libs/jemalloc/Manifest b/dev-libs/jemalloc/Manifest
@@ -0,0 +1,6 @@
+AUX jemalloc-4.5.0-fix_html_install.patch 581 BLAKE2B 97ced7415f9d7dc9d29252ef73888e1e9685d4cfe82e08b35e3b01f1d50aefdabd463c362bc14ca29097c5ff3b24ae53f3f225305b7c5a8770562e03c92c950a SHA512 ce210e9c9394a9ba5a16b5524322f1e477a140e55c076543a1f4dcfb1c8491f7839cb49fba433088773b24d2656b821c3072e729c0c7e41cb90dc131ad6828ff
+AUX jemalloc-5.0.1-strip-optimization.patch 685 BLAKE2B f67c89bb123861cfe3c5b4616d78478693e083476f9e83648d976e613743e6367c9bce9f0b86a1e2fe514d3a3689a596ef94e4a53e26edfa95bdf0336ec0a9ef SHA512 dd4452feea0004b3f61d664a63da0751179a4be8095e76d94938034045114a68b32e0fe9872cb4810c2f3fc927963ef1b1886d210dc558b3c5a3f5eb9ef4240e
+AUX jemalloc-5.1.0-redis-6.2.14.patch 5173 BLAKE2B e74b304cf045a444c9a01bc39780be723b29742788b9c3075d463a48c7600e1d39962055196232da211bdef29e19d691c2cd6acd7cb3aab97265d8be8863850c SHA512 2d45b71d162362862462be4392b362bdf8bfa7d75edd28a7911ff204360a9c705cec2b9f0a7d2336550e7c299d78b455fab10cede0ba76f97a170deec3093a54
+DIST jemalloc-5.1.0.tar.bz2 515622 BLAKE2B 3c8b35d30fca0018e3e32452e6fa41c7ac59f9f2f7d4bc243237fde95025ab5a7562fb86b5afe2ca9b7bb072f7baf6ac7589a4862d9ebaafddae187d93e20da0 SHA512 d9abebe54d303ca931b8c31c1033f23ff5fb060f2377ec8386f4d79c352e65c78ed34f680c352dac14f7d7115d10245782d553d988bc13df2eb34a2f0942ef6f
+EBUILD jemalloc-5.1.0.ebuild 2034 BLAKE2B 746db0a5f716463c345458087fd2a4530eacd8926ed45fa3b36d598b080e3e8019c4ec685b69675d9e94cbd9b26303c63ff00749de33737dbf7e1ea36d90cbc9 SHA512 db1d34f27b449dbac45483a22c6828a73bb79cc3fdc634b71edbf02188770d969b0ddce6115d5d4d4a2b67b1880049e6d847b20eb044e31142a1464c1bce412b
+MISC metadata.xml 631 BLAKE2B ed6e4c05b52cdf7ca78dcded57fbe9252076224b16295aead82f1f18884846f7c4c7a390a5cd622b52752a0db1d4e358b2b2b38845a34a379562c7a5f052a92d SHA512 518374f3f913875b8e5e5d19fa43b754c6f53cc74bd4a98a60a2acdf8213c354aea1b7e4b9f5730da594ecc7adaff7733fab1518adea0b073aa35d523c5407fc
diff --git a/dev-libs/jemalloc/files/jemalloc-4.5.0-fix_html_install.patch b/dev-libs/jemalloc/files/jemalloc-4.5.0-fix_html_install.patch
@@ -0,0 +1,25 @@
+From c04d0b954adef2d3f1a30d91dc687f6a03b108eb Mon Sep 17 00:00:00 2001
+From: 
+Date: Mon, 29 May 2017 00:58:32 -0500
+Subject: [PATCH 2/2] Do not install html doc
+
+---
+ Makefile.in | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Makefile.in b/Makefile.in
+index e49a871..733932a 100644
+--- a/Makefile.in
++++ b/Makefile.in
+@@ -381,7 +381,7 @@ install_doc_man:
+ 	$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
+ done
+
+-install_doc: install_doc_html install_doc_man
++install_doc: install_doc_man
+
+ install: install_bin install_include install_lib install_doc
+
+-- 
+2.13.0
+
diff --git a/dev-libs/jemalloc/files/jemalloc-5.0.1-strip-optimization.patch b/dev-libs/jemalloc/files/jemalloc-5.0.1-strip-optimization.patch
@@ -0,0 +1,26 @@
+diff --git a/configure.ac b/configure.ac
+index 1551ded8..2f470672 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -976,21 +976,6 @@ if test "x$enable_debug" = "x1" ; then
+ fi
+ AC_SUBST([enable_debug])
+
+-dnl Only optimize if not debugging.
+-if test "x$enable_debug" = "x0" ; then
+-  if test "x$GCC" = "xyes" ; then
+-    JE_CFLAGS_ADD([-O3])
+-    JE_CXXFLAGS_ADD([-O3])
+-    JE_CFLAGS_ADD([-funroll-loops])
+-  elif test "x$je_cv_msvc" = "xyes" ; then
+-    JE_CFLAGS_ADD([-O2])
+-    JE_CXXFLAGS_ADD([-O2])
+-  else
+-    JE_CFLAGS_ADD([-O])
+-    JE_CXXFLAGS_ADD([-O])
+-  fi
+-fi
+-
+ dnl Enable statistics calculation by default.
+ AC_ARG_ENABLE([stats],
+   [AS_HELP_STRING([--disable-stats],
diff --git a/dev-libs/jemalloc/files/jemalloc-5.1.0-redis-6.2.14.patch b/dev-libs/jemalloc/files/jemalloc-5.1.0-redis-6.2.14.patch
@@ -0,0 +1,122 @@
+diff -urN jemalloc-5.1.0/VERSION redis-6.2.14/VERSION
+--- jemalloc-5.1.0/VERSION	2018-05-08 19:15:29.000000000 -0000
++++ redis-6.2.14/VERSION	2023-10-18 07:43:10.000000000 -0000
+@@ -1 +1 @@
+-5.1.0-0-g61efbda7098de6fe64c362d309824864308c36d4
++5.1.0-0-g0
+diff -urN jemalloc-5.1.0/configure.ac redis-6.2.14/configure.ac
+--- jemalloc-5.1.0/configure.ac	2018-05-08 19:15:01.000000000 -0000
++++ redis-6.2.14/configure.ac	2023-10-18 07:43:10.000000000 -0000
+@@ -512,7 +512,7 @@
+ LDTARGET='-o $@'
+ TEST_LD_MODE=
+ EXTRA_LDFLAGS=
+-ARFLAGS='crus'
++ARFLAGS='crs'
+ AROUT=' $@'
+ CC_MM=1
+
+diff -urN jemalloc-5.1.0/include/jemalloc/internal/jemalloc_internal_inlines_c.h redis-6.2.14/include/jemalloc/internal/jemalloc_internal_inlines_c.h
+--- jemalloc-5.1.0/include/jemalloc/internal/jemalloc_internal_inlines_c.h	2018-05-08 19:15:01.000000000 -0000
++++ redis-6.2.14/include/jemalloc/internal/jemalloc_internal_inlines_c.h	2023-10-18 07:43:10.000000000 -0000
+@@ -215,4 +215,43 @@
+ 	return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero);
+ }
+
++JEMALLOC_ALWAYS_INLINE int
++iget_defrag_hint(tsdn_t *tsdn, void* ptr) {
++	int defrag = 0;
++	rtree_ctx_t rtree_ctx_fallback;
++	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
++	szind_t szind;
++	bool is_slab;
++	rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &szind, &is_slab);
++	if (likely(is_slab)) {
++		/* Small allocation. */
++		extent_t *slab = iealloc(tsdn, ptr);
++		arena_t *arena = extent_arena_get(slab);
++		szind_t binind = extent_szind_get(slab);
++		bin_t *bin = &arena->bins[binind];
++		malloc_mutex_lock(tsdn, &bin->lock);
++		/* don't bother moving allocations from the slab currently used for new allocations */
++		if (slab != bin->slabcur) {
++			int free_in_slab = extent_nfree_get(slab);
++			if (free_in_slab) {
++				const bin_info_t *bin_info = &bin_infos[binind];
++				unsigned long curslabs = bin->stats.curslabs;
++				size_t curregs = bin->stats.curregs;
++				if (bin->slabcur) {
++					/* remove slabcur from the overall utilization */
++					curregs -= bin_info->nregs - extent_nfree_get(bin->slabcur);
++					curslabs -= 1;
++				}
++				/* Compare the utilization ratio of the slab in question to the total average,
++				 * to avoid precision lost and division, we do that by extrapolating the usage
++				 * of the slab as if all slabs have the same usage. If this slab is less used 
++				 * than the average, we'll prefer to evict the data to hopefully more used ones */
++				defrag = (bin_info->nregs - free_in_slab) * curslabs <= curregs;
++			}
++		}
++		malloc_mutex_unlock(tsdn, &bin->lock);
++	}
++	return defrag;
++}
++
+ #endif /* JEMALLOC_INTERNAL_INLINES_C_H */
+diff -urN jemalloc-5.1.0/include/jemalloc/jemalloc_macros.h.in redis-6.2.14/include/jemalloc/jemalloc_macros.h.in
+--- jemalloc-5.1.0/include/jemalloc/jemalloc_macros.h.in	2018-05-08 19:15:01.000000000 -0000
++++ redis-6.2.14/include/jemalloc/jemalloc_macros.h.in	2023-10-18 07:43:10.000000000 -0000
+@@ -120,3 +120,7 @@
+ #  define JEMALLOC_RESTRICT_RETURN
+ #  define JEMALLOC_ALLOCATOR
+ #endif
++
++/* This version of Jemalloc, modified for Redis, has the je_get_defrag_hint()
++ * function. */
++#define JEMALLOC_FRAG_HINT
+diff -urN jemalloc-5.1.0/src/background_thread.c redis-6.2.14/src/background_thread.c
+--- jemalloc-5.1.0/src/background_thread.c	2018-05-08 19:15:01.000000000 -0000
++++ redis-6.2.14/src/background_thread.c	2023-10-18 07:43:10.000000000 -0000
+@@ -787,7 +787,13 @@
+ 	nstime_init(&stats->run_interval, 0);
+ 	for (unsigned i = 0; i < max_background_threads; i++) {
+ 		background_thread_info_t *info = &background_thread_info[i];
+-		malloc_mutex_lock(tsdn, &info->mtx);
++		if (malloc_mutex_trylock(tsdn, &info->mtx)) {
++			/*
++			 * Each background thread run may take a long time;
++			 * avoid waiting on the stats if the thread is active.
++			 */
++			continue;
++		}
+ 		if (info->state != background_thread_stopped) {
+ 			num_runs += info->tot_n_runs;
+ 			nstime_add(&stats->run_interval, &info->tot_sleep_time);
+diff -urN jemalloc-5.1.0/src/jemalloc.c redis-6.2.14/src/jemalloc.c
+--- jemalloc-5.1.0/src/jemalloc.c	2018-05-08 19:15:01.000000000 -0000
++++ redis-6.2.14/src/jemalloc.c	2023-10-18 07:43:10.000000000 -0000
+@@ -3324,3 +3324,12 @@
+ }
+
+ /******************************************************************************/
++
++/* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation.
++ * returns 1 if the allocation should be moved, and 0 if the allocation be kept.
++ * If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */
++JEMALLOC_EXPORT int JEMALLOC_NOTHROW
++get_defrag_hint(void* ptr) {
++	assert(ptr != NULL);
++	return iget_defrag_hint(TSDN_NULL, ptr);
++}
+diff -urN jemalloc-5.1.0/src/malloc_io.c redis-6.2.14/src/malloc_io.c
+--- jemalloc-5.1.0/src/malloc_io.c	2018-05-08 19:15:01.000000000 -0000
++++ redis-6.2.14/src/malloc_io.c	2023-10-18 07:43:10.000000000 -0000
+@@ -362,7 +362,7 @@
+ 	}								\
+ } while (0)
+ #define GET_ARG_NUMERIC(val, len) do {					\
+-	switch (len) {							\
++	switch ((unsigned char)len) {					\
+ 	case '?':							\
+ 		val = va_arg(ap, int);					\
+ 		break;							\
diff --git a/dev-libs/jemalloc/jemalloc-5.1.0.ebuild b/dev-libs/jemalloc/jemalloc-5.1.0.ebuild
@@ -0,0 +1,67 @@
+# Copyright 1999-2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=6
+
+inherit autotools toolchain-funcs multilib-minimal
+
+DESCRIPTION="Jemalloc is a general-purpose scalable concurrent allocator"
+HOMEPAGE="http://jemalloc.net/ https://github.com/jemalloc/jemalloc"
+SRC_URI="https://github.com/jemalloc/jemalloc/releases/download/${PV}/${P}.tar.bz2"
+
+LICENSE="BSD"
+SLOT="0/2"
+KEYWORDS="~alpha ~amd64 ~arm ~arm64 ~hppa ~ia64 ~ppc ~ppc64 ~s390 ~x86 ~amd64-linux ~x86-linux ~x64-macos ~x64-solaris"
+IUSE="debug hardened lazy-lock prof static-libs stats xmalloc"
+HTML_DOCS=( doc/jemalloc.html )
+PATCHES=( "${FILESDIR}/${PN}-5.1.0-redis-6.2.14.patch"
+	"${FILESDIR}/${PN}-5.0.1-strip-optimization.patch"
+	"${FILESDIR}/${PN}-4.5.0-fix_html_install.patch"
+)
+MULTILIB_WRAPPED_HEADERS=( /usr/include/jemalloc/jemalloc.h )
+# autotools-utils.eclass auto-adds configure options when static-libs is in IUSE
+# but jemalloc doesn't implement them in its configure; need this here to
+# supress the warnings until automagic is removed from the eclass
+QA_CONFIGURE_OPTIONS="--enable-static --disable-static --enable-shared --disable-shared"
+
+# TODO: ease the restriction
+RDEPEND="!<dev-db/redis-6.2.14
+	!>dev-db/redis-6.2.14-r9999"
+
+src_prepare() {
+	default
+	eautoreconf
+}
+
+multilib_src_configure() {
+	local myconf=()
+
+	if use hardened ; then
+		myconf+=( --disable-syscall )
+	fi
+
+	ECONF_SOURCE="${S}" \
+	econf  \
+		$(use_enable debug) \
+		$(use_enable lazy-lock) \
+		$(use_enable prof) \
+		$(use_enable stats) \
+		$(use_enable xmalloc) \
+		"${myconf[@]}"
+}
+
+multilib_src_install() {
+	# Copy man file which the Makefile looks for
+	cp "${S}/doc/jemalloc.3" "${BUILD_DIR}/doc" || die
+	emake DESTDIR="${D}" install
+}
+
+multilib_src_install_all() {
+	if [[ ${CHOST} == *-darwin* ]] ; then
+		# fixup install_name, #437362
+		install_name_tool \
+			-id "${EPREFIX}"/usr/$(get_libdir)/libjemalloc.2.dylib \
+			"${ED}"/usr/$(get_libdir)/libjemalloc.2.dylib || die
+	fi
+	use static-libs || find "${ED}" -name '*.a' -delete
+}
diff --git a/dev-libs/jemalloc/metadata.xml b/dev-libs/jemalloc/metadata.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE pkgmetadata SYSTEM "https://www.gentoo.org/dtd/metadata.dtd">
+<pkgmetadata>
+	<maintainer type="person">
+			<email>[email protected]</email>
+			<name>Joshua Kinard</name>
+	</maintainer>
+	<use>
+		<flag name="lazy-lock">Enable lazy locking (only lock when multi-threaded)</flag>
+		<flag name="prof">Enable allocation profiling</flag>
+		<flag name="stats">Enable statistics calculation/reporting</flag>
+		<flag name="xmalloc">Add support for xmalloc (abort-on-out-of-memory)</flag>
+	</use>
+	<upstream>
+		<remote-id type="github">jemalloc/jemalloc</remote-id>
+	</upstream>
+</pkgmetadata>