Skip to content

Commit

Permalink
Issue #435: Add the possibility to use the same connector with differ…
Browse files Browse the repository at this point in the history
…ent connector variables

* Added the possibility to define additional connectors, which are connectors with variables where the user can customize variables values.
* It is possible to define many instances of the same connector, and this by customizing values for each instance.
* Updated unit tests to make sure everything is working.
* Tested the changes on MetricsHub.
* Updated the MetricsHub documentation to explain how to define and customize connector variables.
* Updated MetricsHub maven plugin to generate additionalConnectors configuration.
  • Loading branch information
CherfaElyes committed Oct 15, 2024
1 parent 2affb96 commit 1b686c0
Show file tree
Hide file tree
Showing 15 changed files with 569 additions and 358 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package org.sentrysoftware.metricshub.agent.config;

/*-
* ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
* MetricsHub Engine
* ჻჻჻჻჻჻
* Copyright 2023 - 2024 Sentry Software
* ჻჻჻჻჻჻
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
*/

import static com.fasterxml.jackson.annotation.Nulls.SKIP;

import com.fasterxml.jackson.annotation.JsonSetter;
import java.util.HashMap;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Builder.Default;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
* Configures additional connectors with variables, the connector ID to use, and a force flag.
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class AdditionalConnector {

/**
* The connector Id of the additional connector instance.
*/
private String uses;

/**
* A map representing the variables for the additional connector.
* The keys are the names of the variables, and the values are the values assigned to those variables.
*/
@Default
@JsonSetter(nulls = SKIP)
private Map<String, String> variables = new HashMap<>();

/**
* A flag indicating whether this connector is forced.
*/
@Default
@JsonSetter(nulls = SKIP)
private boolean force = true;

/**
* Setter that removes variables with null values.
*
* @param variables Map of variables to set.
*/
@JsonSetter
public void setVariables(Map<String, String> variables) {
this.variables = variables;
if (variables != null) {
variables.entrySet().removeIf(entry -> entry.getValue() == null);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonSetter;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
Expand All @@ -38,7 +40,6 @@
import lombok.Data;
import lombok.NoArgsConstructor;
import org.sentrysoftware.metricshub.agent.deserialization.AttributesDeserializer;
import org.sentrysoftware.metricshub.agent.deserialization.ConnectorVariablesDeserializer;
import org.sentrysoftware.metricshub.agent.deserialization.ExtensionProtocolsDeserializer;
import org.sentrysoftware.metricshub.agent.deserialization.MonitorJobsDeserializer;
import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
Expand Down Expand Up @@ -89,11 +90,6 @@ public class ResourceConfig {
@JsonDeserialize(using = ExtensionProtocolsDeserializer.class)
private Map<String, IConfiguration> protocols = new HashMap<>();

@Default
@JsonSetter(nulls = SKIP)
@JsonDeserialize(using = ConnectorVariablesDeserializer.class)
private Map<String, ConnectorVariables> variables = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);

@Default
@JsonSetter(nulls = SKIP)
private Set<String> connectors = new HashSet<>();
Expand All @@ -103,6 +99,10 @@ public class ResourceConfig {
@Default
private Map<String, MonitorJob> monitors = new HashMap<>();

@Default
@JsonSetter(nulls = SKIP)
private Map<String, AdditionalConnector> additionalConnectors = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);

@JsonIgnore
private Connector connector;

Expand Down Expand Up @@ -138,10 +138,32 @@ public ResourceConfig copy() {
.protocols(
protocols.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().copy()))
)
.variables(variables)
.additionalConnectors(additionalConnectors)
.connectors(connectors)
.connector(connector)
.stateSetCompression(stateSetCompression)
.build();
}

/**
* Retrieves the set of connector variables where map keys are connectorIds and values are connectorVariables.
* @return the map of connectorIds and their variables values.
*/
public Map<String, ConnectorVariables> getConnectorVariables() {
return Optional
.ofNullable(additionalConnectors)
.map(map ->
map
.entrySet()
.stream()
.filter(entry -> entry.getValue() != null && entry.getValue().getVariables() != null) // Filter out null values
.collect(
Collectors.toMap(
Map.Entry::getKey,
entry -> new ConnectorVariables(new HashMap<>(entry.getValue().getVariables())) // Create new object with copied map
)
)
)
.orElseGet(Collections::emptyMap); // Return an empty map if additionalConnectors is null
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package org.sentrysoftware.metricshub.agent.deserialization;

/*-
* ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
* MetricsHub Engine
* ჻჻჻჻჻჻
* Copyright 2023 - 2024 Sentry Software
* ჻჻჻჻჻჻
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
*/

import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.sentrysoftware.metricshub.engine.connector.model.Connector;

/**
* Represents the result of parsing additional connectors, containing a map of connectors
* and a set of forced connector IDs.
*/
@Builder
@Data
@NoArgsConstructor
public class AdditionalConnectorsParsingResult {

/**
* The map containing the parsed additional custom connectors.
*/
final Map<String, Connector> customConnectorsMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);

/**
* A set of connector IDs to add into the host connectors set.
*/
final Set<String> hostConnectors = new HashSet<>();
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,13 @@
import org.sentrysoftware.metricshub.agent.config.ResourceConfig;
import org.sentrysoftware.metricshub.agent.config.ResourceGroupConfig;
import org.sentrysoftware.metricshub.agent.context.MetricDefinitions;
import org.sentrysoftware.metricshub.agent.deserialization.AdditionalConnectorsParsingResult;
import org.sentrysoftware.metricshub.agent.security.PasswordEncrypt;
import org.sentrysoftware.metricshub.engine.common.exception.InvalidConfigurationException;
import org.sentrysoftware.metricshub.engine.common.helpers.JsonHelper;
import org.sentrysoftware.metricshub.engine.common.helpers.LocalOsHandler;
import org.sentrysoftware.metricshub.engine.common.helpers.MetricsHubConstants;
import org.sentrysoftware.metricshub.engine.common.helpers.ResourceHelper;
import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
import org.sentrysoftware.metricshub.engine.configuration.HostConfiguration;
import org.sentrysoftware.metricshub.engine.configuration.IConfiguration;
import org.sentrysoftware.metricshub.engine.connector.model.Connector;
Expand Down Expand Up @@ -827,14 +827,14 @@ private static void updateResourceGroupTelemetryManagers(
addConfiguredConnector(resourceConnectorStore, resourceConfig.getConnector());

// Read connectors with configuration variables safely
final Map<String, Connector> connectorsWithConfigVariables = readConnectorsWithConfigurationVariablesSafe(
resourceGroupKey,
resourceKey,
resourceConfig
);
final AdditionalConnectorsParsingResult additionalConnectorsParsingResult =
readConnectorsWithConfigurationVariablesSafe(resourceGroupKey, resourceKey, resourceConfig);

// Overwrite resourceConnectorStore
updateConnectorStore(resourceConnectorStore, connectorsWithConfigVariables);
updateConnectorStore(resourceConnectorStore, additionalConnectorsParsingResult.getCustomConnectorsMap());

// Add custom connectors to the host configuration.
hostConfiguration.getConnectors().addAll(additionalConnectorsParsingResult.getHostConnectors());

resourceGroupTelemetryManagers.putIfAbsent(
resourceKey,
Expand All @@ -858,23 +858,20 @@ private static void updateResourceGroupTelemetryManagers(
* @param resourceGroupKey The resource group key under which the resource is configured for logging purposes.
* @param resourceKey The resource key for logging purposes.
* @param resourceConfig The resource configuration.
* @return Map of connectors with configuration variables
* @return an AdditionalConnectorsParserResult which contains a map of connectors to force and a map of custom connectors.
*/
private static Map<String, Connector> readConnectorsWithConfigurationVariablesSafe(
private static AdditionalConnectorsParsingResult readConnectorsWithConfigurationVariablesSafe(
final String resourceGroupKey,
final String resourceKey,
final ResourceConfig resourceConfig
) {
// Retrieve connectors variables map from the resource configuration
final Map<String, ConnectorVariables> connectorVariablesMap = resourceConfig.getVariables();

// Call ConnectorTemplateLibraryParser and parse the custom connectors
final ConnectorTemplateLibraryParser connectorTemplateLibraryParser = new ConnectorTemplateLibraryParser();
// Call ConnectorVariablesLibraryParser and parse the additional connectors
final ConnectorVariablesLibraryParser connectorVariablesLibraryParser = new ConnectorVariablesLibraryParser();

try {
return connectorTemplateLibraryParser.parse(
return connectorVariablesLibraryParser.parse(
ConfigHelper.getSubDirectory("connectors", false),
connectorVariablesMap
resourceConfig.getAdditionalConnectors()
);
} catch (Exception e) {
log.warn(
Expand All @@ -885,7 +882,7 @@ private static Map<String, Connector> readConnectorsWithConfigurationVariablesSa
resourceKey,
e.getMessage()
);
return new HashMap<>();
return new AdditionalConnectorsParsingResult();
}
}

Expand Down Expand Up @@ -1046,7 +1043,7 @@ static HostConfiguration buildHostConfiguration(
.hostType(hostType)
.sequential(Boolean.TRUE.equals(resourceConfig.getSequential()))
.configuredConnectorId(configuredConnectorId)
.connectorVariables(resourceConfig.getVariables())
.connectorVariables(resourceConfig.getConnectorVariables())
.resolveHostnameToFqdn(resourceConfig.getResolveHostnameToFqdn())
.build();
}
Expand Down
Loading

0 comments on commit 1b686c0

Please sign in to comment.