Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sys ctl to env param #277

Open
wants to merge 2 commits into
base: vNext
Choose a base branch
from
Open

Sys ctl to env param #277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f5d01c
issue: 4209215 Read system control parameters from environment variables
BasharRadya Dec 14, 2024
5da4add
issue: 4209215 Configure TSO and LRO default to ON
BasharRadya Dec 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 3 additions & 8 deletions README
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,11 +486,8 @@ prototype TCP and IP headers of the incoming send buffer to carve out segments
of required size. Copying the prototype header and options, then calculating
the sequence number and checksum fields creates TCP segment headers.
Expected benefits: Throughput increase and CPU unload.
Default value: auto
Default value: on

auto
Depends on ethtool setting and adapter ability.
See ethtool -k <eth0> | grep tcp-segmentation-offload
on
Enabled in case adapter supports it
off
Expand All @@ -513,11 +510,9 @@ high-bandwidth network connections by reducing central processing unit (CPU)
overhead. It works by aggregating multiple incoming packets from a single stream
into a larger buffer before they are passed higher up the networking stack,
thus reducing the number of packets that must be processed.
Default value: auto
Default value: on


auto
Depends on ethtool setting and adapter ability.
See ethtool -k <eth0> | grep large-receive-offload
on
Enabled in case adapter supports it
off
Expand Down
6 changes: 2 additions & 4 deletions src/core/dev/ring_simple.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,8 +257,7 @@ void ring_simple::create_resources()

/* Detect TSO capabilities */
memset(&m_tso, 0, sizeof(m_tso));
if ((safe_mce_sys().enable_tso == option_3::ON) ||
((safe_mce_sys().enable_tso == option_3::AUTO) && (1 == validate_tso(get_if_index())))) {
if ((safe_mce_sys().enable_tso == option_3::ON)) {
if (xlio_check_dev_attr_tso(m_p_ib_ctx->get_ibv_device_attr())) {
const xlio_ibv_tso_caps *caps =
&xlio_get_tso_caps(m_p_ib_ctx->get_ibv_device_attr_ex());
Expand All @@ -280,8 +279,7 @@ void ring_simple::create_resources()

/* Detect LRO capabilities */
memset(&m_lro, 0, sizeof(m_lro));
if ((safe_mce_sys().enable_lro == option_3::ON) ||
((safe_mce_sys().enable_lro == option_3::AUTO) && (1 == validate_lro(get_if_index())))) {
if (safe_mce_sys().enable_lro == option_3::ON) {
dpcp::adapter_hca_capabilities caps;

if (m_p_ib_ctx->get_dpcp_adapter() &&
Expand Down
4 changes: 2 additions & 2 deletions src/core/dev/src_addr_selector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@
bool ipv6_source_addr_score::use_optimistic_addr() const
{
sysctl_reader_t &ctl = sysctl_reader_t::instance();
return (ctl.get_ipv6_conf_all_optimistic_dad() || m_dev->ipv6_optimistic_dad()) &&
(ctl.get_ipv6_conf_all_use_optimistic() || m_dev->ipv6_use_optimistic());
return (ctl.ipv6_conf_all_optimistic_dad || m_dev->ipv6_optimistic_dad()) &&
(ctl.ipv6_conf_all_use_optimistic || m_dev->ipv6_use_optimistic());
}

int ipv6_source_addr_score::get_score(const ipv6_source_addr_score_dst &score_dst, int rule)
Expand Down
80 changes: 80 additions & 0 deletions src/core/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,86 @@ void print_xlio_global_settings()
SYS_VAR_MULTILOCK,
(safe_mce_sys().multilock == MULTILOCK_SPIN ? "Spin " : "Mutex"));

//Sysctl parameters
VLOG_PARAM_NUMBER("TCP max syn backlog", safe_mce_sys().sysctl_reader.tcp_max_syn_backlog,
MCE_DEFAULT_TCP_MAX_SYN_BACKLOG, SYS_VAR_TCP_MAX_SYN_BACKLOG);

VLOG_PARAM_NUMBER("Listen max connections", safe_mce_sys().sysctl_reader.listen_maxconn,
MCE_DEFAULT_LISTEN_MAXCONN, SYS_VAR_LISTEN_MAXCONN);

VLOG_PARAM_NUMBER("TCP window scaling", safe_mce_sys().sysctl_reader.tcp_window_scaling,
MCE_DEFAULT_TCP_WINDOW_SCALING, SYS_VAR_TCP_WINDOW_SCALING);

VLOG_PARAM_NUMBER("TCP write memory (min)", safe_mce_sys().sysctl_reader.tcp_wmem.min_value,
MCE_DEFAULT_TCP_WMEM_MIN, SYS_VAR_TCP_WMEM_MIN);

VLOG_PARAM_NUMBER("TCP write memory (default)",
safe_mce_sys().sysctl_reader.tcp_wmem.default_value,
MCE_DEFAULT_TCP_WMEM_DEFAULT, SYS_VAR_TCP_WMEM_DEFAULT);

VLOG_PARAM_NUMBER("TCP write memory (max)", safe_mce_sys().sysctl_reader.tcp_wmem.max_value,
MCE_DEFAULT_TCP_WMEM_MAX, SYS_VAR_TCP_WMEM_MAX);

VLOG_PARAM_NUMBER("TCP read memory (min)", safe_mce_sys().sysctl_reader.tcp_rmem.min_value,
MCE_DEFAULT_TCP_RMEM_MIN, SYS_VAR_TCP_RMEM_MIN);

VLOG_PARAM_NUMBER("TCP read memory (default)",
safe_mce_sys().sysctl_reader.tcp_rmem.default_value,
MCE_DEFAULT_TCP_RMEM_DEFAULT, SYS_VAR_TCP_RMEM_DEFAULT);

VLOG_PARAM_NUMBER("TCP read memory (max)", safe_mce_sys().sysctl_reader.tcp_rmem.max_value,
MCE_DEFAULT_TCP_RMEM_MAX, SYS_VAR_TCP_RMEM_MAX);

VLOG_PARAM_NUMBER("TCP keepalive time",
safe_mce_sys().sysctl_reader.tcp_keepalive_infos.idle_secs,
MCE_DEFAULT_TCP_KEEPALIVE_TIME, SYS_VAR_TCP_KEEPALIVE_IDLE);

VLOG_PARAM_NUMBER("TCP keepalive interval",
safe_mce_sys().sysctl_reader.tcp_keepalive_infos.interval_secs,
MCE_DEFAULT_TCP_KEEPALIVE_INTERVAL, SYS_VAR_TCP_KEEPALIVE_INTERVAL);

VLOG_PARAM_NUMBER("TCP keepalive probes",
safe_mce_sys().sysctl_reader.tcp_keepalive_infos.num_probes,
MCE_DEFAULT_TCP_KEEPALIVE_PROBES, SYS_VAR_TCP_KEEPALIVE_PROBES);

VLOG_PARAM_NUMBER("Net core receive memory max", safe_mce_sys().sysctl_reader.net_core_rmem_max,
MCE_DEFAULT_NET_CORE_RMEM_MAX, SYS_VAR_NET_CORE_RMEM_MAX);

VLOG_PARAM_NUMBER("Net core write memory max", safe_mce_sys().sysctl_reader.net_core_wmem_max,
MCE_DEFAULT_NET_CORE_WMEM_MAX, SYS_VAR_NET_CORE_WMEM_MAX);

VLOG_PARAM_NUMBER("Net IPv4 TCP timestamps",
safe_mce_sys().sysctl_reader.net_ipv4_tcp_timestamps,
MCE_DEFAULT_NET_IPV4_TCP_TIMESTAMPS, SYS_VAR_NET_IPV4_TCP_TIMESTAMPS);

VLOG_PARAM_NUMBER("Net IPv4 TTL", safe_mce_sys().sysctl_reader.net_ipv4_ttl,
MCE_DEFAULT_NET_IPV4_TTL, SYS_VAR_NET_IPV4_TTL);

VLOG_PARAM_NUMBER("IGMP max membership", safe_mce_sys().sysctl_reader.igmp_max_membership,
MCE_DEFAULT_IGMP_MAX_MEMBERSHIP, SYS_VAR_IGMP_MAX_MEMBERSHIP);

VLOG_PARAM_NUMBER("IGMP max source membership",
safe_mce_sys().sysctl_reader.igmp_max_source_membership,
MCE_DEFAULT_IGMP_MAX_SOURCE_MEMBERSHIP, SYS_VAR_IGMP_MAX_SOURCE_MEMBERSHIP);

VLOG_PARAM_NUMBER("MLD max source membership",
safe_mce_sys().sysctl_reader.mld_max_source_membership,
MCE_DEFAULT_MLD_MAX_SOURCE_MEMBERSHIP, SYS_VAR_MLD_MAX_SOURCE_MEMBERSHIP);

VLOG_PARAM_NUMBER("Net IPv6 hop limit", safe_mce_sys().sysctl_reader.net_ipv6_hop_limit,
MCE_DEFAULT_NET_IPV6_HOP_LIMIT, SYS_VAR_NET_IPV6_HOP_LIMIT);

VLOG_PARAM_NUMBER("IPv6 bindv6only", safe_mce_sys().sysctl_reader.ipv6_bindv6only,
MCE_DEFAULT_IPV6_BINDV6ONLY, SYS_VAR_IPV6_BINDV6ONLY);

VLOG_PARAM_NUMBER(
"IPv6 conf all optimistic dad", safe_mce_sys().sysctl_reader.ipv6_conf_all_optimistic_dad,
MCE_DEFAULT_IPV6_CONF_ALL_OPTIMISTIC_DAD, SYS_VAR_IPV6_CONF_ALL_OPTIMISTIC_DAD);

VLOG_PARAM_NUMBER(
"IPv6 conf all use optimistic", safe_mce_sys().sysctl_reader.ipv6_conf_all_use_optimistic,
MCE_DEFAULT_IPV6_CONF_ALL_USE_OPTIMISTIC, SYS_VAR_IPV6_CONF_ALL_USE_OPTIMISTIC);

vlog_printf(VLOG_INFO,
"---------------------------------------------------------------------------\n");
}
Expand Down
8 changes: 4 additions & 4 deletions src/core/proto/xlio_lwip.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ u32_t xlio_lwip::sys_now(void)
u8_t xlio_lwip::read_tcp_timestamp_option(void)
{
u8_t res = (safe_mce_sys().tcp_ts_opt == TCP_TS_OPTION_FOLLOW_OS)
? safe_mce_sys().sysctl_reader.get_net_ipv4_tcp_timestamps()
? safe_mce_sys().sysctl_reader.net_ipv4_tcp_timestamps
: (safe_mce_sys().tcp_ts_opt == TCP_TS_OPTION_ENABLE ? 1 : 0);
if (res) {
#if LWIP_TCP_TIMESTAMPS
Expand Down Expand Up @@ -108,10 +108,10 @@ xlio_lwip::xlio_lwip()

enable_push_flag = !!safe_mce_sys().tcp_push_flag;
enable_ts_option = read_tcp_timestamp_option();
int is_window_scaling_enabled = safe_mce_sys().sysctl_reader.get_tcp_window_scaling();
int is_window_scaling_enabled = safe_mce_sys().sysctl_reader.tcp_window_scaling;
if (is_window_scaling_enabled) {
int rmem_max_value = safe_mce_sys().sysctl_reader.get_tcp_rmem()->max_value;
int core_rmem_max = safe_mce_sys().sysctl_reader.get_net_core_rmem_max();
int rmem_max_value = safe_mce_sys().sysctl_reader.tcp_rmem.max_value;
int core_rmem_max = safe_mce_sys().sysctl_reader.net_core_rmem_max;
enable_wnd_scale = 1;
rcv_wnd_scale = get_window_scaling_factor(rmem_max_value, core_rmem_max);
} else {
Expand Down
12 changes: 5 additions & 7 deletions src/core/sock/sockinfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,16 +104,15 @@ sockinfo::sockinfo(int fd, int domain, bool use_ring_locks)
, m_family(domain)
, m_fd(fd)
, m_rx_num_buffs_reuse(safe_mce_sys().rx_bufs_batch)
, m_is_ipv6only(safe_mce_sys().sysctl_reader.get_ipv6_bindv6only())
, m_is_ipv6only(safe_mce_sys().sysctl_reader.ipv6_bindv6only)
, m_lock_rcv(MULTILOCK_RECURSIVE, MODULE_NAME "::m_lock_rcv")
, m_lock_snd(MODULE_NAME "::m_lock_snd")
, m_so_bindtodevice_ip(ip_address::any_addr(), domain)
, m_rx_ring_map_lock(MODULE_NAME "::m_rx_ring_map_lock")
, m_ring_alloc_log_rx(safe_mce_sys().ring_allocation_logic_rx, use_ring_locks)
, m_ring_alloc_log_tx(safe_mce_sys().ring_allocation_logic_tx, use_ring_locks)
, m_n_uc_ttl_hop_lim(m_family == AF_INET
? safe_mce_sys().sysctl_reader.get_net_ipv4_ttl()
: safe_mce_sys().sysctl_reader.get_net_ipv6_hop_limit())
, m_n_uc_ttl_hop_lim(m_family == AF_INET ? safe_mce_sys().sysctl_reader.net_ipv4_ttl
: safe_mce_sys().sysctl_reader.net_ipv6_hop_limit)
{
m_rx_epfd = SYSCALL(epoll_create, 128);
if (unlikely(m_rx_epfd == -1)) {
Expand Down Expand Up @@ -601,9 +600,8 @@ int sockinfo::setsockopt(int __level, int __optname, const void *__optval, sockl
ret = SOCKOPT_NO_XLIO_SUPPORT;
errno = EINVAL;
} else {
m_n_uc_ttl_hop_lim = (val == -1)
? safe_mce_sys().sysctl_reader.get_net_ipv4_ttl()
: (uint8_t)val;
m_n_uc_ttl_hop_lim =
(val == -1) ? safe_mce_sys().sysctl_reader.net_ipv4_ttl : (uint8_t)val;
header_ttl_hop_limit_updater du(m_n_uc_ttl_hop_lim, false);
update_header_field(&du);
si_logdbg("IPPROTO_IP, optname=IP_TTL (%d)", m_n_uc_ttl_hop_lim);
Expand Down
18 changes: 9 additions & 9 deletions src/core/sock/sockinfo_tcp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ sockinfo_tcp::sockinfo_tcp(int fd, int domain)

tcp_pcb_init(&m_pcb, TCP_PRIO_NORMAL, this);

const tcp_keepalive_info keepalive_info = safe_mce_sys().sysctl_reader.get_tcp_keepalive_info();
const tcp_keepalive_info keepalive_info = safe_mce_sys().sysctl_reader.tcp_keepalive_infos;
tcp_set_keepalive(&m_pcb, static_cast<u32_t>(1000U * keepalive_info.idle_secs),
static_cast<u32_t>(1000U * keepalive_info.interval_secs),
static_cast<u32_t>(keepalive_info.num_probes));
Expand All @@ -350,7 +350,7 @@ sockinfo_tcp::sockinfo_tcp(int fd, int domain)
m_iomux_ready_fd_array = nullptr;

/* RCVBUF accounting */
m_rcvbuff_max = safe_mce_sys().sysctl_reader.get_tcp_rmem()->default_value;
m_rcvbuff_max = safe_mce_sys().sysctl_reader.tcp_rmem.default_value;

m_rcvbuff_current = 0;
m_rcvbuff_non_tcp_recved = 0;
Expand Down Expand Up @@ -1716,7 +1716,7 @@ void sockinfo_tcp::process_my_ctl_packets()
mem_buf_desc_t *desc = temp_list.get_and_pop_front();

static const unsigned int MAX_SYN_RCVD = tcp_ctl_thread_on(safe_mce_sys().tcp_ctl_thread)
? safe_mce_sys().sysctl_reader.get_tcp_max_syn_backlog()
? safe_mce_sys().sysctl_reader.tcp_max_syn_backlog
: 0;
// NOTE: currently, in case tcp_ctl_thread is disabled, only established backlog is
// supported (no syn-rcvd backlog)
Expand Down Expand Up @@ -2536,7 +2536,7 @@ bool sockinfo_tcp::rx_input_cb(mem_buf_desc_t *p_rx_pkt_mem_buf_desc_info, void
/// distinguish between backlog of established sockets vs. backlog of syn-rcvd
static const unsigned int MAX_SYN_RCVD =
tcp_ctl_thread_on(safe_mce_sys().tcp_ctl_thread)
? safe_mce_sys().sysctl_reader.get_tcp_max_syn_backlog()
? safe_mce_sys().sysctl_reader.tcp_max_syn_backlog
: 0;
// NOTE: currently, in case tcp_ctl_thread is disabled, only established backlog is
// supported (no syn-rcvd backlog)
Expand Down Expand Up @@ -2974,7 +2974,7 @@ int sockinfo_tcp::listen(int backlog)
* Note: backlog behavior depends on safe_mce_sys().tcp_ctl_thread status.
*/
if (backlog < 0) {
backlog = safe_mce_sys().sysctl_reader.get_listen_maxconn();
backlog = safe_mce_sys().sysctl_reader.listen_maxconn;
si_tcp_logdbg("changing listen backlog=%d to the maximum=%d", orig_backlog, backlog);
} else if (backlog == 0) {
backlog = 1;
Expand All @@ -2986,8 +2986,8 @@ int sockinfo_tcp::listen(int backlog)
/* If an application calls listen() with a backlog value larger than net.core.somaxconn,
* then the backlog for that listener will be silently truncated to the somaxconn value.
*/
if (backlog > safe_mce_sys().sysctl_reader.get_listen_maxconn()) {
backlog = safe_mce_sys().sysctl_reader.get_listen_maxconn();
if (backlog > safe_mce_sys().sysctl_reader.listen_maxconn) {
backlog = safe_mce_sys().sysctl_reader.listen_maxconn;
si_tcp_logdbg("truncating listen backlog=%d to the maximun=%d", orig_backlog, backlog);
}
}
Expand Down Expand Up @@ -4478,7 +4478,7 @@ int sockinfo_tcp::tcp_setsockopt(int __level, int __optname, __const void *__opt
si_tcp_logdbg("(SO_KEEPALIVE) val: %d", val);
break;
case SO_RCVBUF:
val = std::min(*(int *)__optval, safe_mce_sys().sysctl_reader.get_net_core_rmem_max());
val = std::min(*(int *)__optval, safe_mce_sys().sysctl_reader.net_core_rmem_max);
lock_tcp_con();
// OS allocates double the size of memory requested by the application - not sure we
// need it.
Expand All @@ -4489,7 +4489,7 @@ int sockinfo_tcp::tcp_setsockopt(int __level, int __optname, __const void *__opt
si_tcp_logdbg("setsockopt SO_RCVBUF: %d", m_rcvbuff_max);
break;
case SO_SNDBUF:
val = std::min(*(int *)__optval, safe_mce_sys().sysctl_reader.get_net_core_wmem_max());
val = std::min(*(int *)__optval, safe_mce_sys().sysctl_reader.net_core_wmem_max);
lock_tcp_con();
// OS allocates double the size of memory requested by the application - not sure we
// need it.
Expand Down
12 changes: 6 additions & 6 deletions src/core/sock/sockinfo_udp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2675,7 +2675,7 @@ int sockinfo_udp::mc_change_membership_start_helper_ip4(const ip_address &mc_grp
case IP_ADD_MEMBERSHIP:
if (m_mc_memberships_map.find(mc_grp) == m_mc_memberships_map.end() &&
m_mc_memberships_map.size() >=
(size_t)safe_mce_sys().sysctl_reader.get_igmp_max_membership()) {
(size_t)safe_mce_sys().sysctl_reader.igmp_max_membership) {
errno = ENOBUFS;
return -1;
}
Expand All @@ -2684,13 +2684,13 @@ int sockinfo_udp::mc_change_membership_start_helper_ip4(const ip_address &mc_grp
if (m_mc_memberships_map.find(mc_grp) != m_mc_memberships_map.end()) { // This group is
// exist
if (m_mc_memberships_map[mc_grp].size() >=
(size_t)safe_mce_sys().sysctl_reader.get_igmp_max_source_membership()) {
(size_t)safe_mce_sys().sysctl_reader.igmp_max_source_membership) {
errno = ENOBUFS;
return -1;
}
} else { // This group is not exist
if (m_mc_memberships_map.size() >=
(size_t)safe_mce_sys().sysctl_reader.get_igmp_max_membership()) {
(size_t)safe_mce_sys().sysctl_reader.igmp_max_membership) {
errno = ENOBUFS;
return -1;
}
Expand Down Expand Up @@ -2917,7 +2917,7 @@ int sockinfo_udp::mc_change_membership_start_helper_ip6(const mc_pending_pram *p
case MCAST_JOIN_SOURCE_GROUP: {
if (group_exists) {
if (m_mc_memberships_map[mc_grp].size() >=
(size_t)safe_mce_sys().sysctl_reader.get_mld_max_source_membership()) {
(size_t)safe_mce_sys().sysctl_reader.mld_max_source_membership) {
errno = ENOBUFS;
return -1;
}
Expand All @@ -2936,8 +2936,8 @@ int sockinfo_udp::mc_change_membership_start_helper_ip6(const mc_pending_pram *p
open MCAST_BLOCK_SOURCE and MCAST_UNBLOCK_SOURCE
requires a fix in drop packet logic - see bug #3202713 above
size_t max_cap = is_ipv6 ?
(size_t)safe_mce_sys().sysctl_reader.get_mld_max_source_membership() :
(size_t)safe_mce_sys().sysctl_reader.get_igmp_max_source_membership(); if (group_exists
(size_t)safe_mce_sys().sysctl_reader.mld_max_source_membership :
(size_t)safe_mce_sys().sysctl_reader.igmp_max_source_membership; if (group_exists
&& !src_exists) {
if (m_mc_memberships_map[mc_grp].size() >= max_cap) {
errno = ENOBUFS;
Expand Down
Loading