Issue #435: Add the possibility to use the same connector with different connector variables

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java

@@ -0,0 +1,76 @@
+package org.sentrysoftware.metricshub.agent.config;
+
+/*-
+ * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
+ * MetricsHub Engine
+ * ჻჻჻჻჻჻
+ * Copyright 2023 - 2024 Sentry Software
+ * ჻჻჻჻჻჻
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
+ */
+
+import static com.fasterxml.jackson.annotation.Nulls.SKIP;
+
+import com.fasterxml.jackson.annotation.JsonSetter;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Builder.Default;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * Configures additional connectors with variables, the connector ID to use, and a force flag.
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class AdditionalConnector {
+
+	/**
+	 * The connector Id of the additional connector instance.
+	 */
+	private String uses;
+
+	/**
+	 * A map representing the variables for the additional connector.
+	 * The keys are the names of the variables, and the values are the values assigned to those variables.
+	 */
+	@Default
+	@JsonSetter(nulls = SKIP)
+	private Map<String, String> variables = new HashMap<>();
+
+	/**
+	 * A flag indicating whether this connector is forced.
+	 */
+	@Default
+	@JsonSetter(nulls = SKIP)
+	private boolean force = true;
+
+	/**
+	 * Setter that removes variables with null values.
+	 *
+	 * @param variables Map of variables to set.
+	 */
+	@JsonSetter
+	public void setVariables(Map<String, String> variables) {
+		this.variables = variables;
+		if (variables != null) {
+			variables.entrySet().removeIf(entry -> entry.getValue() == null);
+		}
+	}
+}

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/ResourceConfig.java

@@ -26,9 +26,11 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonSetter;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
@@ -38,7 +40,6 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import org.sentrysoftware.metricshub.agent.deserialization.AttributesDeserializer;
-import org.sentrysoftware.metricshub.agent.deserialization.ConnectorVariablesDeserializer;
 import org.sentrysoftware.metricshub.agent.deserialization.ExtensionProtocolsDeserializer;
 import org.sentrysoftware.metricshub.agent.deserialization.MonitorJobsDeserializer;
 import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
@@ -92,11 +93,6 @@ public class ResourceConfig {
 	@JsonDeserialize(using = ExtensionProtocolsDeserializer.class)
 	private Map<String, IConfiguration> protocols = new HashMap<>();
 
-	@Default
-	@JsonSetter(nulls = SKIP)
-	@JsonDeserialize(using = ConnectorVariablesDeserializer.class)
-	private Map<String, ConnectorVariables> variables = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
 	@Default
 	@JsonSetter(nulls = SKIP)
 	private Set<String> connectors = new HashSet<>();
@@ -106,6 +102,10 @@ public class ResourceConfig {
 	@Default
 	private Map<String, MonitorJob> monitors = new HashMap<>();
 
+	@Default
+	@JsonSetter(nulls = SKIP)
+	private Map<String, AdditionalConnector> additionalConnectors = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+
 	@JsonIgnore
 	private Connector connector;
 
@@ -142,10 +142,32 @@ public ResourceConfig copy() {
 			.protocols(
 				protocols.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().copy()))
 			)
-			.variables(variables)
+			.additionalConnectors(additionalConnectors)
 			.connectors(connectors)
 			.connector(connector)
 			.stateSetCompression(stateSetCompression)
 			.build();
 	}
+
+	/**
+	 * Retrieves the set of connector variables where map keys are connectorIds and values are connectorVariables.
+	 * @return the map of connectorIds and their variables values.
+	 */
+	public Map<String, ConnectorVariables> getConnectorVariables() {
+		return Optional
+			.ofNullable(additionalConnectors)
+			.map(map ->
+				map
+					.entrySet()
+					.stream()
+					.filter(entry -> entry.getValue() != null && entry.getValue().getVariables() != null) // Filter out null values
+					.collect(
+						Collectors.toMap(
+							Map.Entry::getKey,
+							entry -> new ConnectorVariables(new HashMap<>(entry.getValue().getVariables())) // Create new object with copied map
+						)
+					)
+			)
+			.orElseGet(Collections::emptyMap); // Return an empty map if additionalConnectors is null
+	}
 }

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/AdditionalConnectorsParsingResult.java

@@ -0,0 +1,51 @@
+package org.sentrysoftware.metricshub.agent.deserialization;
+
+/*-
+ * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
+ * MetricsHub Engine
+ * ჻჻჻჻჻჻
+ * Copyright 2023 - 2024 Sentry Software
+ * ჻჻჻჻჻჻
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
+ */
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.sentrysoftware.metricshub.engine.connector.model.Connector;
+
+/**
+ * Represents the result of parsing additional connectors, containing a map of connectors
+ * and a set of forced connector IDs.
+ */
+@Builder
+@Data
+@NoArgsConstructor
+public class AdditionalConnectorsParsingResult {
+
+	/**
+	 * The map containing the parsed additional custom connectors.
+	 */
+	final Map<String, Connector> customConnectorsMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+
+	/**
+	 * A set of connector IDs to add into the host connectors set.
+	 */
+	final Set<String> hostConnectors = new HashSet<>();
+}

metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java

@@ -67,13 +67,13 @@
 import org.sentrysoftware.metricshub.agent.config.ResourceConfig;
 import org.sentrysoftware.metricshub.agent.config.ResourceGroupConfig;
 import org.sentrysoftware.metricshub.agent.context.MetricDefinitions;
+import org.sentrysoftware.metricshub.agent.deserialization.AdditionalConnectorsParsingResult;
 import org.sentrysoftware.metricshub.agent.security.PasswordEncrypt;
 import org.sentrysoftware.metricshub.engine.common.exception.InvalidConfigurationException;
 import org.sentrysoftware.metricshub.engine.common.helpers.JsonHelper;
 import org.sentrysoftware.metricshub.engine.common.helpers.LocalOsHandler;
 import org.sentrysoftware.metricshub.engine.common.helpers.MetricsHubConstants;
 import org.sentrysoftware.metricshub.engine.common.helpers.ResourceHelper;
-import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables;
 import org.sentrysoftware.metricshub.engine.configuration.HostConfiguration;
 import org.sentrysoftware.metricshub.engine.configuration.IConfiguration;
 import org.sentrysoftware.metricshub.engine.connector.model.Connector;
@@ -843,14 +843,14 @@ private static void updateResourceGroupTelemetryManagers(
 			addConfiguredConnector(resourceConnectorStore, resourceConfig.getConnector());
 
 			// Read connectors with configuration variables safely
-			final Map<String, Connector> connectorsWithConfigVariables = readConnectorsWithConfigurationVariablesSafe(
-				resourceGroupKey,
-				resourceKey,
-				resourceConfig
-			);
+			final AdditionalConnectorsParsingResult additionalConnectorsParsingResult =
+				readConnectorsWithConfigurationVariablesSafe(resourceGroupKey, resourceKey, resourceConfig);
 
 			// Overwrite resourceConnectorStore
-			updateConnectorStore(resourceConnectorStore, connectorsWithConfigVariables);
+			updateConnectorStore(resourceConnectorStore, additionalConnectorsParsingResult.getCustomConnectorsMap());
+
+			// Add custom connectors to the host configuration.
+			hostConfiguration.getConnectors().addAll(additionalConnectorsParsingResult.getHostConnectors());
 
 			resourceGroupTelemetryManagers.putIfAbsent(
 				resourceKey,
@@ -874,23 +874,20 @@ private static void updateResourceGroupTelemetryManagers(
 	 * @param resourceGroupKey The resource group key under which the resource is configured for logging purposes.
 	 * @param resourceKey      The resource key for logging purposes.
 	 * @param resourceConfig   The resource configuration.
-	 * @return Map of connectors with configuration variables
+	 * @return an AdditionalConnectorsParserResult which contains a map of connectors to force and a map of custom connectors.
 	 */
-	private static Map<String, Connector> readConnectorsWithConfigurationVariablesSafe(
+	private static AdditionalConnectorsParsingResult readConnectorsWithConfigurationVariablesSafe(
 		final String resourceGroupKey,
 		final String resourceKey,
 		final ResourceConfig resourceConfig
 	) {
-		// Retrieve connectors variables map from the resource configuration
-		final Map<String, ConnectorVariables> connectorVariablesMap = resourceConfig.getVariables();
-
-		// Call ConnectorTemplateLibraryParser and parse the custom connectors
-		final ConnectorTemplateLibraryParser connectorTemplateLibraryParser = new ConnectorTemplateLibraryParser();
+		// Call ConnectorVariablesLibraryParser and parse the additional connectors
+		final ConnectorVariablesLibraryParser connectorVariablesLibraryParser = new ConnectorVariablesLibraryParser();
 
 		try {
-			return connectorTemplateLibraryParser.parse(
+			return connectorVariablesLibraryParser.parse(
 				ConfigHelper.getSubDirectory("connectors", false),
-				connectorVariablesMap
+				resourceConfig.getAdditionalConnectors()
 			);
 		} catch (Exception e) {
 			log.warn(
@@ -901,7 +898,7 @@ private static Map<String, Connector> readConnectorsWithConfigurationVariablesSa
 				resourceKey,
 				e.getMessage()
 			);
-			return new HashMap<>();
+			return new AdditionalConnectorsParsingResult();
 		}
 	}
 
@@ -1086,7 +1083,7 @@ static HostConfiguration buildHostConfiguration(
 			.includedMonitors(includedMonitors)
 			.excludedMonitors(excludedMonitors)
 			.configuredConnectorId(configuredConnectorId)
-			.connectorVariables(resourceConfig.getVariables())
+			.connectorVariables(resourceConfig.getConnectorVariables())
 			.resolveHostnameToFqdn(resourceConfig.getResolveHostnameToFqdn())
 			.build();
 	}