From 1b686c03f346afd729af41211b1485bfb3cc25d8 Mon Sep 17 00:00:00 2001 From: Elyes Cherfa Date: Tue, 15 Oct 2024 16:00:02 +0200 Subject: [PATCH 1/8] Issue #435: Add the possibility to use the same connector with different connector variables * Added the possibility to define additional connectors, which are connectors with variables where the user can customize variables values. * It is possible to define many instances of the same connector, and this by customizing values for each instance. * Updated unit tests to make sure everything is working. * Tested the changes on MetricsHub. * Updated the MetricsHub documentation to explain how to define and customize connector variables. * Updated MetricsHub maven plugin to generate additionalConnectors configuration. --- .../agent/config/AdditionalConnector.java | 76 +++++ .../agent/config/ResourceConfig.java | 36 ++- .../AdditionalConnectorsParsingResult.java | 51 ++++ .../ConnectorVariablesDeserializer.java | 88 ------ .../metricshub/agent/helper/ConfigHelper.java | 33 +- .../ConnectorTemplateLibraryParser.java | 216 ------------- .../ConnectorVariablesLibraryParser.java | 288 ++++++++++++++++++ .../agent/context/AgentContextTest.java | 42 ++- ... ConnectorVariablesLibraryParserTest.java} | 35 ++- .../config/metricshub-connectorVariables.yaml | 30 +- .../connectorVariable.yaml} | 0 .../noConnectorVariable.yaml} | 0 .../configuration/configure-monitoring.md | 18 +- .../metricshub/config/metricshub-example.yaml | 7 + .../MetricsHub/config/metricshub-example.yaml | 7 + 15 files changed, 569 insertions(+), 358 deletions(-) create mode 100644 metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java create mode 100644 metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/AdditionalConnectorsParsingResult.java delete mode 100644 metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/ConnectorVariablesDeserializer.java delete mode 100644 metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParser.java create mode 100644 metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParser.java rename metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/{ConnectorTemplateLibraryParserTest.java => ConnectorVariablesLibraryParserTest.java} (60%) rename metricshub-agent/src/test/resources/{connectorTemplateLibraryParser/templateVariable.yaml => connectorVariablesLibraryParser/connectorVariable.yaml} (100%) rename metricshub-agent/src/test/resources/{connectorTemplateLibraryParser/noTemplateVariable.yaml => connectorVariablesLibraryParser/noConnectorVariable.yaml} (100%) diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java new file mode 100644 index 000000000..f6a52ef7f --- /dev/null +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java @@ -0,0 +1,76 @@ +package org.sentrysoftware.metricshub.agent.config; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * MetricsHub Engine + * ჻჻჻჻჻჻ + * Copyright 2023 - 2024 Sentry Software + * ჻჻჻჻჻჻ + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import static com.fasterxml.jackson.annotation.Nulls.SKIP; + +import com.fasterxml.jackson.annotation.JsonSetter; +import java.util.HashMap; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Builder.Default; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Configures additional connectors with variables, the connector ID to use, and a force flag. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class AdditionalConnector { + + /** + * The connector Id of the additional connector instance. + */ + private String uses; + + /** + * A map representing the variables for the additional connector. + * The keys are the names of the variables, and the values are the values assigned to those variables. + */ + @Default + @JsonSetter(nulls = SKIP) + private Map variables = new HashMap<>(); + + /** + * A flag indicating whether this connector is forced. + */ + @Default + @JsonSetter(nulls = SKIP) + private boolean force = true; + + /** + * Setter that removes variables with null values. + * + * @param variables Map of variables to set. + */ + @JsonSetter + public void setVariables(Map variables) { + this.variables = variables; + if (variables != null) { + variables.entrySet().removeIf(entry -> entry.getValue() == null); + } + } +} diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/ResourceConfig.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/ResourceConfig.java index 5992dc322..5a9bb0962 100644 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/ResourceConfig.java +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/ResourceConfig.java @@ -26,9 +26,11 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -38,7 +40,6 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.sentrysoftware.metricshub.agent.deserialization.AttributesDeserializer; -import org.sentrysoftware.metricshub.agent.deserialization.ConnectorVariablesDeserializer; import org.sentrysoftware.metricshub.agent.deserialization.ExtensionProtocolsDeserializer; import org.sentrysoftware.metricshub.agent.deserialization.MonitorJobsDeserializer; import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables; @@ -89,11 +90,6 @@ public class ResourceConfig { @JsonDeserialize(using = ExtensionProtocolsDeserializer.class) private Map protocols = new HashMap<>(); - @Default - @JsonSetter(nulls = SKIP) - @JsonDeserialize(using = ConnectorVariablesDeserializer.class) - private Map variables = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - @Default @JsonSetter(nulls = SKIP) private Set connectors = new HashSet<>(); @@ -103,6 +99,10 @@ public class ResourceConfig { @Default private Map monitors = new HashMap<>(); + @Default + @JsonSetter(nulls = SKIP) + private Map additionalConnectors = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + @JsonIgnore private Connector connector; @@ -138,10 +138,32 @@ public ResourceConfig copy() { .protocols( protocols.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().copy())) ) - .variables(variables) + .additionalConnectors(additionalConnectors) .connectors(connectors) .connector(connector) .stateSetCompression(stateSetCompression) .build(); } + + /** + * Retrieves the set of connector variables where map keys are connectorIds and values are connectorVariables. + * @return the map of connectorIds and their variables values. + */ + public Map getConnectorVariables() { + return Optional + .ofNullable(additionalConnectors) + .map(map -> + map + .entrySet() + .stream() + .filter(entry -> entry.getValue() != null && entry.getValue().getVariables() != null) // Filter out null values + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> new ConnectorVariables(new HashMap<>(entry.getValue().getVariables())) // Create new object with copied map + ) + ) + ) + .orElseGet(Collections::emptyMap); // Return an empty map if additionalConnectors is null + } } diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/AdditionalConnectorsParsingResult.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/AdditionalConnectorsParsingResult.java new file mode 100644 index 000000000..94b2372f9 --- /dev/null +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/AdditionalConnectorsParsingResult.java @@ -0,0 +1,51 @@ +package org.sentrysoftware.metricshub.agent.deserialization; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * MetricsHub Engine + * ჻჻჻჻჻჻ + * Copyright 2023 - 2024 Sentry Software + * ჻჻჻჻჻჻ + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.sentrysoftware.metricshub.engine.connector.model.Connector; + +/** + * Represents the result of parsing additional connectors, containing a map of connectors + * and a set of forced connector IDs. + */ +@Builder +@Data +@NoArgsConstructor +public class AdditionalConnectorsParsingResult { + + /** + * The map containing the parsed additional custom connectors. + */ + final Map customConnectorsMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + /** + * A set of connector IDs to add into the host connectors set. + */ + final Set hostConnectors = new HashSet<>(); +} diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/ConnectorVariablesDeserializer.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/ConnectorVariablesDeserializer.java deleted file mode 100644 index 4f7f4f156..000000000 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/ConnectorVariablesDeserializer.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.sentrysoftware.metricshub.agent.deserialization; - -/*- - * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ - * MetricsHub Agent - * ჻჻჻჻჻჻ - * Copyright 2023 - 2024 Sentry Software - * ჻჻჻჻჻჻ - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ - */ - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; -import java.util.Map; -import java.util.TreeMap; -import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables; - -/** - * Custom JSON deserializer for deserializing a JSON object into a TreeMap with - * ConnectorVariables instances and case-insensitive comparator for keys. Each - * ConnectorVariables instance has a Map for variableValues. - */ -public class ConnectorVariablesDeserializer extends JsonDeserializer> { - - /** - * Deserializes a JSON object into a TreeMap with ConnectorVariables - * instances and case-insensitive comparator for keys.
Each - * ConnectorVariables instance has a Map for variableValues. - * - * @param jsonParser The JsonParser object for reading JSON content. - * @param context The DeserializationContext object. - * @return The deserialized Map with ConnectorVariables instances. - * @throws IOException If an I/O error occurs during deserialization. - */ - @Override - public Map deserialize(final JsonParser jsonParser, final DeserializationContext context) - throws IOException { - final JsonNode node = jsonParser.getCodec().readTree(jsonParser); - - // TreeMap with case-insensitive comparator for storing key-ConnectorVariables pairs - final Map treeMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - - // Iterate through the fields of the JSON object - if (node != null) { - node - .fields() - .forEachRemaining(entry -> { - final String name = entry.getKey(); - final JsonNode value = entry.getValue(); - - if (value != null && !value.isNull()) { - final ConnectorVariables connectorVariables = new ConnectorVariables(); - - // Iterate through the variableValues of the ConnectorVariables instance - value - .fields() - .forEachRemaining(variableValuesEntry -> { - final String variableName = variableValuesEntry.getKey(); - final JsonNode variableValue = variableValuesEntry.getValue(); - - if (variableValue != null && !variableValue.isNull()) { - connectorVariables.addVariableValue(variableName, variableValue.asText()); - } - }); - - treeMap.put(name, connectorVariables); - } - }); - } - - return treeMap; - } -} diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java index 8195c337f..5812c051c 100644 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java @@ -66,13 +66,13 @@ import org.sentrysoftware.metricshub.agent.config.ResourceConfig; import org.sentrysoftware.metricshub.agent.config.ResourceGroupConfig; import org.sentrysoftware.metricshub.agent.context.MetricDefinitions; +import org.sentrysoftware.metricshub.agent.deserialization.AdditionalConnectorsParsingResult; import org.sentrysoftware.metricshub.agent.security.PasswordEncrypt; import org.sentrysoftware.metricshub.engine.common.exception.InvalidConfigurationException; import org.sentrysoftware.metricshub.engine.common.helpers.JsonHelper; import org.sentrysoftware.metricshub.engine.common.helpers.LocalOsHandler; import org.sentrysoftware.metricshub.engine.common.helpers.MetricsHubConstants; import org.sentrysoftware.metricshub.engine.common.helpers.ResourceHelper; -import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables; import org.sentrysoftware.metricshub.engine.configuration.HostConfiguration; import org.sentrysoftware.metricshub.engine.configuration.IConfiguration; import org.sentrysoftware.metricshub.engine.connector.model.Connector; @@ -827,14 +827,14 @@ private static void updateResourceGroupTelemetryManagers( addConfiguredConnector(resourceConnectorStore, resourceConfig.getConnector()); // Read connectors with configuration variables safely - final Map connectorsWithConfigVariables = readConnectorsWithConfigurationVariablesSafe( - resourceGroupKey, - resourceKey, - resourceConfig - ); + final AdditionalConnectorsParsingResult additionalConnectorsParsingResult = + readConnectorsWithConfigurationVariablesSafe(resourceGroupKey, resourceKey, resourceConfig); // Overwrite resourceConnectorStore - updateConnectorStore(resourceConnectorStore, connectorsWithConfigVariables); + updateConnectorStore(resourceConnectorStore, additionalConnectorsParsingResult.getCustomConnectorsMap()); + + // Add custom connectors to the host configuration. + hostConfiguration.getConnectors().addAll(additionalConnectorsParsingResult.getHostConnectors()); resourceGroupTelemetryManagers.putIfAbsent( resourceKey, @@ -858,23 +858,20 @@ private static void updateResourceGroupTelemetryManagers( * @param resourceGroupKey The resource group key under which the resource is configured for logging purposes. * @param resourceKey The resource key for logging purposes. * @param resourceConfig The resource configuration. - * @return Map of connectors with configuration variables + * @return an AdditionalConnectorsParserResult which contains a map of connectors to force and a map of custom connectors. */ - private static Map readConnectorsWithConfigurationVariablesSafe( + private static AdditionalConnectorsParsingResult readConnectorsWithConfigurationVariablesSafe( final String resourceGroupKey, final String resourceKey, final ResourceConfig resourceConfig ) { - // Retrieve connectors variables map from the resource configuration - final Map connectorVariablesMap = resourceConfig.getVariables(); - - // Call ConnectorTemplateLibraryParser and parse the custom connectors - final ConnectorTemplateLibraryParser connectorTemplateLibraryParser = new ConnectorTemplateLibraryParser(); + // Call ConnectorVariablesLibraryParser and parse the additional connectors + final ConnectorVariablesLibraryParser connectorVariablesLibraryParser = new ConnectorVariablesLibraryParser(); try { - return connectorTemplateLibraryParser.parse( + return connectorVariablesLibraryParser.parse( ConfigHelper.getSubDirectory("connectors", false), - connectorVariablesMap + resourceConfig.getAdditionalConnectors() ); } catch (Exception e) { log.warn( @@ -885,7 +882,7 @@ private static Map readConnectorsWithConfigurationVariablesSa resourceKey, e.getMessage() ); - return new HashMap<>(); + return new AdditionalConnectorsParsingResult(); } } @@ -1046,7 +1043,7 @@ static HostConfiguration buildHostConfiguration( .hostType(hostType) .sequential(Boolean.TRUE.equals(resourceConfig.getSequential())) .configuredConnectorId(configuredConnectorId) - .connectorVariables(resourceConfig.getVariables()) + .connectorVariables(resourceConfig.getConnectorVariables()) .resolveHostnameToFqdn(resourceConfig.getResolveHostnameToFqdn()) .build(); } diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParser.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParser.java deleted file mode 100644 index 231b4b837..000000000 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParser.java +++ /dev/null @@ -1,216 +0,0 @@ -package org.sentrysoftware.metricshub.agent.helper; - -/*- - * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ - * MetricsHub Agent - * ჻჻჻჻჻჻ - * Copyright 2023 - 2024 Sentry Software - * ჻჻჻჻჻჻ - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ - */ - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables; -import org.sentrysoftware.metricshub.engine.connector.model.Connector; -import org.sentrysoftware.metricshub.engine.connector.model.identity.ConnectorDefaultVariable; -import org.sentrysoftware.metricshub.engine.connector.parser.ConnectorParser; - -/** - * Utility class for parsing connector template YAML files and creating a map of custom connectors. - * This class provides functionality to visit YAML files in a specified directory, read connector data, - * and create Connector objects based on the parsed data. The resulting connectors are stored in a map - * with the connector ID as the key and the corresponding Connector object as the value. - *

- * The parsing process involves checking for YAML files, validating whether the YAML structure defines a - * final Connector (with a displayName section), and using ConnectorParser to parse the YAML file and create - * Connector objects. - *

- */ -@Slf4j -public class ConnectorTemplateLibraryParser { - - /** - * This inner class allows to visit the files contained within the Yaml directory - */ - private static class ConnectorFileVisitor extends SimpleFileVisitor { - - private final Map connectorVariablesMap; - private final Map customConnectorsMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - - public Map getCustomConnectorsMap() { - return customConnectorsMap; - } - - ConnectorFileVisitor(final Map connectorVariablesMap) { - this.connectorVariablesMap = connectorVariablesMap; - } - - @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { - // Skip this path if it is a directory or not a YAML file - if (Files.isDirectory(path) || !isYamlFile(path.toFile().getName())) { - return FileVisitResult.CONTINUE; - } - final ObjectMapper yamlMapper = new YAMLMapper(); - final JsonNode connectorNode = yamlMapper.readTree(path.toFile()); - if (!isConnector(connectorNode)) { - return FileVisitResult.CONTINUE; - } - - // Get the connector's file name - final String filename = path.getFileName().toString(); - final String connectorId = filename.substring(0, filename.lastIndexOf('.')); - - if (!connectorNode.toString().contains("${var::")) { - return FileVisitResult.CONTINUE; - } - - // User connector variables - final ConnectorVariables connectorUserVariables = connectorVariablesMap.computeIfAbsent( - connectorId, - id -> new ConnectorVariables() - ); - - // Retrieve the default connector variables that have been specified in this connector. - final Map connectorDefaultVariables = getConnectorVariables(connectorNode); - - // User didn't configure variables for this connector, and no connector default variables are configured - if (connectorUserVariables.getVariableValues().isEmpty() && connectorDefaultVariables.isEmpty()) { - return FileVisitResult.CONTINUE; - } - - // For each configured default connector variable, if the user didn't specify a value to that variable, user default value. - for (final Entry entry : connectorDefaultVariables.entrySet()) { - connectorUserVariables.getVariableValues().putIfAbsent(entry.getKey(), entry.getValue().getDefaultValue()); - } - - final ConnectorParser connectorParser = ConnectorParser.withNodeProcessorAndUpdateChain( - path.getParent(), - connectorVariablesMap.get(connectorId).getVariableValues() - ); - - // Put in the custom connectorsMap - try { - customConnectorsMap.put(connectorId, connectorParser.parse(path.toFile())); - } catch (Exception e) { - log.error("Error while parsing connector with template variables {}: {}", filename, e.getMessage()); - log.debug("Exception: ", e); - } - - return FileVisitResult.CONTINUE; - } - - /** - * Whether the JsonNode is a final Connector. It means that this JsonNode defines the displayName section. - * - * @param connector JsonNode that contains connector's data - * @return true if the {@link JsonNode} is a final connector, otherwise false. - */ - private boolean isConnector(final JsonNode connector) { - final JsonNode connectorNode = connector.get("connector"); - if (connectorNode != null && !connectorNode.isNull()) { - final JsonNode displayName = connectorNode.get("displayName"); - return displayName != null && !displayName.isNull(); - } - - return false; - } - - /** - * Whether the connector is a YAML file or not - * - * @param name given fileName - * @return boolean value - */ - private boolean isYamlFile(final String name) { - return name.toLowerCase().endsWith(".yaml"); - } - - /** - * Converts the "variables" section of a {@link JsonNode} into a {@link Map} where each entry consists of - * a variable name as the key and a {@link ConnectorDefaultVariable} as the value. - * Each {@link ConnectorDefaultVariable} contains a description and a default value extracted from the JSON node. - * - * @param connectorNode the {@link JsonNode} representing the connector, which includes a "variables" section. - * @return a map where the key is the variable name, and the value is a {@link ConnectorDefaultVariable} object - * containing the description and defaultValue for that variable. If the "variables" section is not present, - * an empty map is returned. - */ - private static Map getConnectorVariables(final JsonNode connectorNode) { - final JsonNode variablesNode = connectorNode.get("connector").get("variables"); - if (variablesNode == null) { - return new HashMap<>(); - } - - final Map connectorVariablesMap = new HashMap<>(); - - // Iterate over the variables and extract description and defaultValue - variablesNode - .fields() - .forEachRemaining(entry -> { - final String variableName = entry.getKey(); - final JsonNode variableValue = entry.getValue(); - - final String description = variableValue.get("description").asText(); - final String defaultValue = variableValue.get("defaultValue").asText(); - - // Create a ConnectorDefaultVariable object and put it into the map - final ConnectorDefaultVariable connectorDefaultVariable = new ConnectorDefaultVariable( - description, - defaultValue - ); - connectorVariablesMap.put(variableName, connectorDefaultVariable); - }); - - return connectorVariablesMap; - } - } - - /** - * Parses connector template YAML files in the specified directory and creates a map of custom connectors. - * - * @param yamlParentDirectory The directory containing connector YAML files - * @param connectorVariablesMap A map of ConnectorVariables for variable substitution - * @return Map<String, Connector> (connectors map: key=yamlFileName, value=Connector) - * @throws IOException if the file does not exist or an I/O error occurs during processing - */ - public Map parse( - @NonNull final Path yamlParentDirectory, - @NonNull final Map connectorVariablesMap - ) throws IOException { - final long startTime = System.currentTimeMillis(); - final ConnectorFileVisitor connectorFileVisitor = new ConnectorFileVisitor(connectorVariablesMap); - Files.walkFileTree(yamlParentDirectory, connectorFileVisitor); - log.info( - "Connectors with template variables parsing duration: {} seconds", - (System.currentTimeMillis() - startTime) / 1000 - ); - return connectorFileVisitor.getCustomConnectorsMap(); - } -} diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParser.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParser.java new file mode 100644 index 000000000..953f3a23c --- /dev/null +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParser.java @@ -0,0 +1,288 @@ +package org.sentrysoftware.metricshub.agent.helper; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * MetricsHub Agent + * ჻჻჻჻჻჻ + * Copyright 2023 - 2024 Sentry Software + * ჻჻჻჻჻჻ + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.sentrysoftware.metricshub.agent.config.AdditionalConnector; +import org.sentrysoftware.metricshub.agent.deserialization.AdditionalConnectorsParsingResult; +import org.sentrysoftware.metricshub.engine.connector.model.Connector; +import org.sentrysoftware.metricshub.engine.connector.model.identity.ConnectorDefaultVariable; +import org.sentrysoftware.metricshub.engine.connector.parser.ConnectorParser; + +/** + * Utility class for parsing connectors with variables and creating a map of custom connectors. + * This class provides functionality to visit YAML files in a specified directory, read connector data, + * and create Connector objects based on the parsed data. The resulting connectors are stored in a map + * with the connector ID as the key and the corresponding Connector object as the value. + *

+ * The parsing process involves checking for YAML files, validating whether the YAML structure defines a + * final Connector (with a displayName section), and using ConnectorParser to parse the YAML file and create + * Connector objects. + *

+ */ +@Slf4j +public class ConnectorVariablesLibraryParser { + + /** + * This inner class allows to visit the files contained within the Yaml directory + */ + private static class ConnectorFileVisitor extends SimpleFileVisitor { + + private final Map additionalConnectorConfig; + private final AdditionalConnectorsParsingResult connectorsParsingResult = new AdditionalConnectorsParsingResult(); + + public AdditionalConnectorsParsingResult getConnectorsParsingResult() { + return connectorsParsingResult; + } + + ConnectorFileVisitor(final Map additionalConnectorConfig) { + this.additionalConnectorConfig = additionalConnectorConfig; + } + + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + // Skip this path if it is a directory or not a YAML file + if (Files.isDirectory(path) || !isYamlFile(path.toFile().getName())) { + return FileVisitResult.CONTINUE; + } + final ObjectMapper yamlMapper = new YAMLMapper(); + final JsonNode connectorNode = yamlMapper.readTree(path.toFile()); + if (!isConnector(connectorNode)) { + return FileVisitResult.CONTINUE; + } + + // Get the connector's file name + final String filename = path.getFileName().toString(); + final String connectorId = filename.substring(0, filename.lastIndexOf('.')); + + if (!connectorNode.toString().contains("${var::")) { + return FileVisitResult.CONTINUE; + } + + // Normalize additionalConnectors + normalizeAdditionalConnectors(); + + // Removing all the configurations that are not using this connector. + final Map filteredConnectors = additionalConnectorConfig + .entrySet() + .stream() + .filter(entry -> connectorId.equalsIgnoreCase(entry.getValue().getUses())) + .collect( + Collectors.toMap( + Map.Entry::getKey, // Keep the original key + Map.Entry::getValue // Keep the original AdditionalConnectorConfig as the value + ) + ); + + // Construct a variables map from the default connector variables. + final Map defaultVariables = new HashMap<>(getDefaultConnectorVariables(connectorNode)); + + // Parse the connector even if it is not configured as an additional connector. + // This ensures that the connector will function with the default variables if the user forces it. + if (filteredConnectors.isEmpty()) { + parseConnectorWithModifier(path, defaultVariables, connectorId, filename, connector -> {}); + } + + // For each configuration, we create a new custom connector and a new variables map to be used in the connector update. + for (final Entry connectorConfigurationEntry : filteredConnectors.entrySet()) { + final String additionalConnectorId = connectorConfigurationEntry.getKey(); + final AdditionalConnector additionalConnectorValue = connectorConfigurationEntry.getValue(); + + // Add the connector to the host connectors set. + connectorsParsingResult + .getHostConnectors() + .add(additionalConnectorValue.isForce() ? "+" + additionalConnectorId : additionalConnectorId); + + // Retrieve and use default connector variables on this connector for this configuration. + final Map connectorVariables = new HashMap<>(defaultVariables); + + // Override the default connector variables by the connector variables that the user configured. + final Map configuredVariables = additionalConnectorValue.getVariables(); + if (configuredVariables != null) { + connectorVariables.putAll(configuredVariables); + } + // There are at least two additional connectors that use the current connector. + // This means that the compiled filename of these connectors needs to be modified. + if (filteredConnectors.size() > 1) { + parseConnectorWithModifier( + path, + connectorVariables, + additionalConnectorId, + filename, + connector -> connector.getConnectorIdentity().setCompiledFilename(additionalConnectorId) + ); + continue; + } + parseConnectorWithModifier(path, connectorVariables, connectorId, filename, connector -> {}); + } + return FileVisitResult.CONTINUE; + } + + /** + * Whether the JsonNode is a final Connector. It means that this JsonNode defines the displayName section. + * + * @param connector JsonNode that contains connector's data + * @return true if the {@link JsonNode} is a final connector, otherwise false. + */ + private boolean isConnector(final JsonNode connector) { + final JsonNode connectorNode = connector.get("connector"); + if (connectorNode != null && !connectorNode.isNull()) { + final JsonNode displayName = connectorNode.get("displayName"); + return displayName != null && !displayName.isNull(); + } + + return false; + } + + /** + * Whether the connector is a YAML file or not + * + * @param name given fileName + * @return boolean value + */ + private boolean isYamlFile(final String name) { + return name.toLowerCase().endsWith(".yaml"); + } + + /** + * Converts the "variables" section of a {@link JsonNode} into a {@link Map} where each entry consists of + * a variable name as the key and a {@link ConnectorDefaultVariable} as the value. + * Each {@link ConnectorDefaultVariable} contains a description and a default value extracted from the JSON node. + * + * @param connectorNode the {@link JsonNode} representing the connector, which includes a "variables" section. + * @return a map where the key is the variable name, and the value is a {@link ConnectorDefaultVariable} object + * containing the description and defaultValue for that variable. If the "variables" section is not present, + * an empty map is returned. + */ + private static Map getDefaultConnectorVariables(final JsonNode connectorNode) { + final JsonNode variablesNode = connectorNode.get("connector").get("variables"); + if (variablesNode == null) { + return new HashMap<>(); + } + + final Map connectorVariablesMap = new HashMap<>(); + + // Iterate over the variables and extract description and defaultValue + variablesNode + .fields() + .forEachRemaining(entry -> { + final String variableName = entry.getKey(); + final JsonNode variableValue = entry.getValue(); + + connectorVariablesMap.put(variableName, variableValue.get("defaultValue").asText()); + }); + + return connectorVariablesMap; + } + + /** + * Parses a connector file located at the specified path, using the provided variables and connector Id. + * The parsed connector is then added to the custom connectors map, with the option to perform additional + * operations on the connector before adding it to the map. + * + * @param path The path to the connector file that needs to be parsed. + * @param variables A map of variables to be used for processing the connector, where the keys are variable names and the values are the corresponding values. + * @param connectorId The unique identifier for the connector being parsed, used as the key in the custom connectors map. + * @param filename The name of the connector file, used for logging in case of an error. + * @param connectorModifier A function to apply additional changes to the parsed connector before adding it to the map. + */ + private void parseConnectorWithModifier( + final Path path, + final Map variables, + final String connectorId, + final String filename, + final Consumer connectorModifier + ) { + final ConnectorParser connectorParser = ConnectorParser.withNodeProcessorAndUpdateChain( + path.getParent(), + variables + ); + try { + final Connector connector = connectorParser.parse(path.toFile()); + connectorModifier.accept(connector); + connectorsParsingResult.getCustomConnectorsMap().put(connectorId, connector); + } catch (Exception e) { + log.error("Error while parsing connector with variables {}: {}", filename, e.getMessage()); + log.debug("Exception: ", e); + } + } + + /** + * Updates the additional connectors configuration by ensuring that each entry has a valid + * {@code AdditionalConnector} object. If the connector or its {@code uses} field is null, it is set + * to the connector Id. + */ + private void normalizeAdditionalConnectors() { + additionalConnectorConfig + .entrySet() + .forEach(entry -> { + final String connectorId = entry.getKey(); + final AdditionalConnector additionalConnector = entry.getValue(); + + // If additionalConnector is null, create a new object and update the entry + if (additionalConnector == null) { + entry.setValue(AdditionalConnector.builder().force(true).uses(connectorId).variables(null).build()); + return; + } + + // If uses() is null, set it to the connectorId + if (additionalConnector.getUses() == null) { + additionalConnector.setUses(connectorId); + } + }); + } + } + + /** + * Parses connectors with variables YAML files in the specified directory and creates a map of custom connectors. + * + * @param yamlParentDirectory The directory containing connector YAML files + * @param additionalConnectorConfig A map of additional connector configurations + * @return {@link AdditionalConnectorsParsingResult} containing parsed connectors and any forced connectors + * @throws IOException if the file does not exist or an I/O error occurs during processing + */ + public AdditionalConnectorsParsingResult parse( + @NonNull final Path yamlParentDirectory, + @NonNull final Map additionalConnectorConfig + ) throws IOException { + final long startTime = System.currentTimeMillis(); + final ConnectorFileVisitor connectorFileVisitor = new ConnectorFileVisitor(additionalConnectorConfig); + Files.walkFileTree(yamlParentDirectory, connectorFileVisitor); + log.info("Connectors with variables parsing duration: {} seconds", (System.currentTimeMillis() - startTime) / 1000); + return connectorFileVisitor.getConnectorsParsingResult(); + } +} diff --git a/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/context/AgentContextTest.java b/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/context/AgentContextTest.java index 8c1edb446..4a4fdbebc 100644 --- a/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/context/AgentContextTest.java +++ b/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/context/AgentContextTest.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import org.junit.jupiter.api.Test; +import org.sentrysoftware.metricshub.agent.config.AdditionalConnector; import org.sentrysoftware.metricshub.agent.config.AgentConfig; import org.sentrysoftware.metricshub.agent.config.ResourceConfig; import org.sentrysoftware.metricshub.agent.config.ResourceGroupConfig; @@ -205,14 +206,43 @@ void testInitializeWithConnectorVariables() throws IOException { .get(SENTRY_PARIS_RESOURCE_GROUP_KEY) .getResources() .get(SERVER_1_RESOURCE_GROUP_KEY); - final Map variables = resourceConfig.getVariables(); - final ConnectorVariables expectedConnectorVariables = ConnectorVariables + + final Map additionalConnectors = resourceConfig.getAdditionalConnectors(); + // Check the number of additional connectors + assertEquals(5, additionalConnectors.size()); + + final Map variables = resourceConfig.getConnectorVariables(); + // Check the number of configured ConnectorVariables + assertEquals(3, variables.size()); + + AdditionalConnector pureStorageREST = AdditionalConnector + .builder() + .uses("PureStorageREST") + .variables(Map.of("restQueryPath", "/pure/api/v2")) + .force(false) + .build(); + assertEquals(pureStorageREST, additionalConnectors.get("PureStorageREST")); + AdditionalConnector windows = AdditionalConnector .builder() - .variableValues(Map.of("restQueryPath", "/pure/api/v2")) + .variables(Map.of("osType", "windows")) + .force(true) .build(); - assertEquals(Map.of("PureStorageREST", expectedConnectorVariables), variables); - // Case insensitive check - assertEquals(expectedConnectorVariables, variables.get("purestoragerest")); + AdditionalConnector linux = AdditionalConnector.builder().uses("Linux").variables(null).force(true).build(); + AdditionalConnector ipmiTool = AdditionalConnector + .builder() + .uses("IpmiTool") + .variables(Map.of()) + .force(true) + .build(); + + final Map expectedAdditionalConnectors = new LinkedHashMap<>(); + + expectedAdditionalConnectors.put("PureStorageREST", pureStorageREST); + expectedAdditionalConnectors.put("Windows", windows); + expectedAdditionalConnectors.put("Linux", linux); + expectedAdditionalConnectors.put("IpmiTool", ipmiTool); + expectedAdditionalConnectors.put("LinuxProcess", null); + assertEquals(expectedAdditionalConnectors, additionalConnectors); } @Test diff --git a/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParserTest.java b/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParserTest.java similarity index 60% rename from metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParserTest.java rename to metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParserTest.java index 78edde3e1..51895ca98 100644 --- a/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorTemplateLibraryParserTest.java +++ b/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParserTest.java @@ -11,33 +11,42 @@ import java.util.Map; import java.util.Set; import org.junit.jupiter.api.Test; -import org.sentrysoftware.metricshub.engine.configuration.ConnectorVariables; +import org.sentrysoftware.metricshub.agent.config.AdditionalConnector; +import org.sentrysoftware.metricshub.agent.deserialization.AdditionalConnectorsParsingResult; import org.sentrysoftware.metricshub.engine.connector.model.Connector; import org.sentrysoftware.metricshub.engine.connector.model.identity.Detection; import org.sentrysoftware.metricshub.engine.connector.model.identity.criterion.SnmpGetNextCriterion; -class ConnectorTemplateLibraryParserTest { +class ConnectorVariablesLibraryParserTest { - private static final String CONNECTOR_ID = "templateVariable"; + private static final String CONNECTOR_ID = "connectorVariable"; @Test void testParse() throws IOException { // Define the yaml test files path - final Path yamlTestPath = Paths.get("src", "test", "resources", "connectorTemplateLibraryParser"); + final Path yamlTestPath = Paths.get("src", "test", "resources", "connectorVariablesLibraryParser"); - // Call ConnectorTemplateLibraryParser to parse the custom connectors files using the connectorVariables map and the connector id - final ConnectorTemplateLibraryParser connectorTemplateLibraryParser = new ConnectorTemplateLibraryParser(); + // Call ConnectorVariablesLibraryParser to parse the custom connectors files using the connectorVariables map and the connector id + final ConnectorVariablesLibraryParser connectorVariablesLibraryParser = new ConnectorVariablesLibraryParser(); - final ConnectorVariables connectorVariables = new ConnectorVariables(new HashMap<>()); - connectorVariables.addVariableValue("snmp-get-next", "snmpGetNext"); - connectorVariables.addVariableValue("local-variable", "local"); - final Map connectorVariablesMap = new HashMap<>(); - connectorVariablesMap.put(CONNECTOR_ID, connectorVariables); - final Map customConnectorsMap = connectorTemplateLibraryParser.parse( + final Map connectorVariables = new HashMap<>(); + connectorVariables.put("snmp-get-next", "snmpGetNext"); + connectorVariables.put("local-variable", "local"); + final Map additionalConnectorConfigMap = new HashMap<>(); + final AdditionalConnector additionalConnectorConfig = AdditionalConnector + .builder() + .force(true) + .uses(CONNECTOR_ID) + .variables(connectorVariables) + .build(); + additionalConnectorConfigMap.put(CONNECTOR_ID, additionalConnectorConfig); + final AdditionalConnectorsParsingResult parsingResult = connectorVariablesLibraryParser.parse( yamlTestPath, - connectorVariablesMap + additionalConnectorConfigMap ); + final Map customConnectorsMap = parsingResult.getCustomConnectorsMap(); + // Check that only the connector containing variables is returned in the map assertEquals(1, customConnectorsMap.size()); diff --git a/metricshub-agent/src/test/resources/config/metricshub-connectorVariables.yaml b/metricshub-agent/src/test/resources/config/metricshub-connectorVariables.yaml index 60c0098a0..119c26fac 100644 --- a/metricshub-agent/src/test/resources/config/metricshub-connectorVariables.yaml +++ b/metricshub-agent/src/test/resources/config/metricshub-connectorVariables.yaml @@ -56,10 +56,28 @@ resourceGroups: port: 443 username: username password: password - variables: - PureStorageREST: - restQueryPath: /pure/api/v2 - nullValue: - OtherNull: connectors: - - +PureStorageREST \ No newline at end of file + - +PureStorageREST + additionalConnectors: + # With an empty variable value and false force + PureStorageREST: + uses: PureStorageREST + force: false + variables: + restQueryPath: /pure/api/v2 + nullValue: + # Without uses and force + Windows: + variables: + osType: windows + # With an empty variables map + Linux: + uses: Linux + force: true + variables: + # Without variables + IpmiTool: + uses: IpmiTool + force: true + # Without uses, force, & variables + LinuxProcess: \ No newline at end of file diff --git a/metricshub-agent/src/test/resources/connectorTemplateLibraryParser/templateVariable.yaml b/metricshub-agent/src/test/resources/connectorVariablesLibraryParser/connectorVariable.yaml similarity index 100% rename from metricshub-agent/src/test/resources/connectorTemplateLibraryParser/templateVariable.yaml rename to metricshub-agent/src/test/resources/connectorVariablesLibraryParser/connectorVariable.yaml diff --git a/metricshub-agent/src/test/resources/connectorTemplateLibraryParser/noTemplateVariable.yaml b/metricshub-agent/src/test/resources/connectorVariablesLibraryParser/noConnectorVariable.yaml similarity index 100% rename from metricshub-agent/src/test/resources/connectorTemplateLibraryParser/noTemplateVariable.yaml rename to metricshub-agent/src/test/resources/connectorVariablesLibraryParser/noConnectorVariable.yaml diff --git a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md index f4e7080bc..99ce61713 100644 --- a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md +++ b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md @@ -828,7 +828,9 @@ loggerLevel: ... #### Configure Connector Variables -In **MetricsHub**, connector variables are essential for customizing the behavior of data collection. The connector variables are configured in the `metricshub.yaml` file under the `variables` section of your configured resource. These variables are specified under the name of the connector to which they belong and contain key-value pairs. The key of each variable corresponds to a variable already configured in the connector. +In **MetricsHub**, connector variables are essential for customizing the behavior of data collection. These variables are configured in the `additionalConnectors` section of your `metricshub.yaml` file. When configuring variables for connectors with variables, the connector becomes an additional connector, which is forced by default and will use the variables configured by the user. + +Each additional connector is identified by its ID, and the variables are specified under this ID in the additionalConnectors section. The variables consist of key-value pairs, where the key corresponds to a variable already defined in the connector. * Example : @@ -843,11 +845,19 @@ resources: protocols: wmi: timeout: 120 - variables: - windowsProcess: # Connector ID - processName: "('msedge.exe', 'metricshub.exe')" + additionalConnectors: + metricshubWindowsProcess: # Additional Connector ID + uses: WindowsProcess # ID of the connector to customize. + force: true # true by default + variables: + processName: 'metricshub.exe' + ``` +If a connector with variables is forced or configured under `additionalConnectors` section but without variables, this latter will still be used with default connector variables that are defined in the connector under `defaultVariables` section. + +If `uses` is not specified, MetricsHub will deduce that the additional connectorId is the connectorId. + #### Discovery cycle **MetricsHub** periodically performs discoveries to detect new components in your monitored environment. By default, **MetricsHub** runs a discovery after 30 collects. To change this default discovery cycle: diff --git a/metricshub-linux/src/main/resources/jpackage/metricshub/config/metricshub-example.yaml b/metricshub-linux/src/main/resources/jpackage/metricshub/config/metricshub-example.yaml index 4ff3ba3da..e24c506bd 100644 --- a/metricshub-linux/src/main/resources/jpackage/metricshub/config/metricshub-example.yaml +++ b/metricshub-linux/src/main/resources/jpackage/metricshub/config/metricshub-example.yaml @@ -199,6 +199,13 @@ resourceGroups: # username: myusername # password: mypwd # privateKey: /tmp/ssh-key.txt + # additionalConnectors: + # ConnectorId: + # uses: LinuxProcess + # force: true + # variables: + # var1: value1 + # var2: value2 #═══════════════════════════════════════════════════ # OSCommand configuration diff --git a/metricshub-windows/src/main/resources/jpackage/MetricsHub/config/metricshub-example.yaml b/metricshub-windows/src/main/resources/jpackage/MetricsHub/config/metricshub-example.yaml index 65a39fb9e..e81086467 100644 --- a/metricshub-windows/src/main/resources/jpackage/MetricsHub/config/metricshub-example.yaml +++ b/metricshub-windows/src/main/resources/jpackage/MetricsHub/config/metricshub-example.yaml @@ -116,6 +116,13 @@ resourceGroups: # username: server-3\username # password: password # timeout: 120 + # additionalConnectors: + # ConnectorId: + # uses: WindowsService + # force: true + # variables: + # var1: value1 + # var2: value2 #═══════════════════════════════════════════════════ # SNMP v1 protocol configuration From 295894cb8032ce93c4793c30c7573ea0e10dd2e4 Mon Sep 17 00:00:00 2001 From: Elyes Cherfa Date: Thu, 24 Oct 2024 14:33:48 +0200 Subject: [PATCH 2/8] Issue #435: Add the possibility to use the same connector with different connector variables * Refactored ConnectorVariablesLibraryParser for better clarity. * Corrected some Javadoc errors. * Moved some classes to a new package. --- .../agent/config/AdditionalConnector.java | 2 +- .../AdditionalConnectorsParsingResult.java | 4 +- .../ConnectorVariablesLibraryParser.java | 95 +++++++++++-------- .../metricshub/agent/helper/ConfigHelper.java | 3 +- .../ConnectorVariablesLibraryParserTest.java | 3 +- 5 files changed, 65 insertions(+), 42 deletions(-) rename metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/{deserialization => connector}/AdditionalConnectorsParsingResult.java (93%) rename metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/{helper => connector}/ConnectorVariablesLibraryParser.java (86%) diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java index f6a52ef7f..c55f5e71a 100644 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/config/AdditionalConnector.java @@ -2,7 +2,7 @@ /*- * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ - * MetricsHub Engine + * MetricsHub Agent * ჻჻჻჻჻჻ * Copyright 2023 - 2024 Sentry Software * ჻჻჻჻჻჻ diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/AdditionalConnectorsParsingResult.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/AdditionalConnectorsParsingResult.java similarity index 93% rename from metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/AdditionalConnectorsParsingResult.java rename to metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/AdditionalConnectorsParsingResult.java index 94b2372f9..eb8bb047e 100644 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/deserialization/AdditionalConnectorsParsingResult.java +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/AdditionalConnectorsParsingResult.java @@ -1,8 +1,8 @@ -package org.sentrysoftware.metricshub.agent.deserialization; +package org.sentrysoftware.metricshub.agent.connector; /*- * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ - * MetricsHub Engine + * MetricsHub Agent * ჻჻჻჻჻჻ * Copyright 2023 - 2024 Sentry Software * ჻჻჻჻჻჻ diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParser.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java similarity index 86% rename from metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParser.java rename to metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java index 953f3a23c..5b6053f1e 100644 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParser.java +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java @@ -1,4 +1,4 @@ -package org.sentrysoftware.metricshub.agent.helper; +package org.sentrysoftware.metricshub.agent.connector; /*- * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ @@ -38,7 +38,6 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.sentrysoftware.metricshub.agent.config.AdditionalConnector; -import org.sentrysoftware.metricshub.agent.deserialization.AdditionalConnectorsParsingResult; import org.sentrysoftware.metricshub.engine.connector.model.Connector; import org.sentrysoftware.metricshub.engine.connector.model.identity.ConnectorDefaultVariable; import org.sentrysoftware.metricshub.engine.connector.parser.ConnectorParser; @@ -111,44 +110,15 @@ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IO // Construct a variables map from the default connector variables. final Map defaultVariables = new HashMap<>(getDefaultConnectorVariables(connectorNode)); - // Parse the connector even if it is not configured as an additional connector. - // This ensures that the connector will function with the default variables if the user forces it. if (filteredConnectors.isEmpty()) { + // Parse the connector even if it is not configured as an additional connector. + // This ensures that the connector will function with the default variables if the user forces it. parseConnectorWithModifier(path, defaultVariables, connectorId, filename, connector -> {}); + } else { + // Parse and generate connectors that have been added and configured. + generateNewConnectors(path, filename, filteredConnectors, defaultVariables); } - // For each configuration, we create a new custom connector and a new variables map to be used in the connector update. - for (final Entry connectorConfigurationEntry : filteredConnectors.entrySet()) { - final String additionalConnectorId = connectorConfigurationEntry.getKey(); - final AdditionalConnector additionalConnectorValue = connectorConfigurationEntry.getValue(); - - // Add the connector to the host connectors set. - connectorsParsingResult - .getHostConnectors() - .add(additionalConnectorValue.isForce() ? "+" + additionalConnectorId : additionalConnectorId); - - // Retrieve and use default connector variables on this connector for this configuration. - final Map connectorVariables = new HashMap<>(defaultVariables); - - // Override the default connector variables by the connector variables that the user configured. - final Map configuredVariables = additionalConnectorValue.getVariables(); - if (configuredVariables != null) { - connectorVariables.putAll(configuredVariables); - } - // There are at least two additional connectors that use the current connector. - // This means that the compiled filename of these connectors needs to be modified. - if (filteredConnectors.size() > 1) { - parseConnectorWithModifier( - path, - connectorVariables, - additionalConnectorId, - filename, - connector -> connector.getConnectorIdentity().setCompiledFilename(additionalConnectorId) - ); - continue; - } - parseConnectorWithModifier(path, connectorVariables, connectorId, filename, connector -> {}); - } return FileVisitResult.CONTINUE; } @@ -203,12 +173,63 @@ private static Map getDefaultConnectorVariables(final JsonNode c final String variableName = entry.getKey(); final JsonNode variableValue = entry.getValue(); - connectorVariablesMap.put(variableName, variableValue.get("defaultValue").asText()); + final JsonNode defaultValue = variableValue.get("defaultValue"); + if (defaultValue != null && !defaultValue.isNull()) { + connectorVariablesMap.put(variableName, variableValue.get("defaultValue").asText()); + } }); return connectorVariablesMap; } + /** + * Generates custom connectors for each additional connector configuration. + * + *

This method iterates over the provided map of filtered connectors, generating a new {@link Connector} + * object for each. It handles the default and user-configured variables, ensuring that any forced connectors + * are correctly identified. The method can modify the compiled filename of connectors when needed.

+ * + * @param path The path to the directory containing the connector files. + * @param filename The filename of the connector file. + * @param filteredConnectors A map of filtered additional connectors, with their configurations. + * @param defaultVariables A map of default variables to be used during connector generation. + */ + private void generateNewConnectors( + final Path path, + final String filename, + final Map filteredConnectors, + final Map defaultVariables + ) { + // For each configuration, we create a new custom connector and a new variables map to be used in the connector update. + for (final Entry connectorConfigurationEntry : filteredConnectors.entrySet()) { + final String additionalConnectorId = connectorConfigurationEntry.getKey(); + final AdditionalConnector additionalConnectorValue = connectorConfigurationEntry.getValue(); + + // Add the connector to the host connectors set. + connectorsParsingResult + .getHostConnectors() + .add(additionalConnectorValue.isForce() ? "+" + additionalConnectorId : additionalConnectorId); + + // Retrieve and use default connector variables on this connector for this configuration. + final Map connectorVariables = new HashMap<>(defaultVariables); + + // Override the default connector variables by the connector variables that the user configured. + final Map configuredVariables = additionalConnectorValue.getVariables(); + if (configuredVariables != null) { + connectorVariables.putAll(configuredVariables); + } + // There are at least two additional connectors that use the current connector. + // This means that the compiled filename of these connectors needs to be modified. + parseConnectorWithModifier( + path, + connectorVariables, + additionalConnectorId, + filename, + connector -> connector.getConnectorIdentity().setCompiledFilename(additionalConnectorId) + ); + } + } + /** * Parses a connector file located at the specified path, using the provided variables and connector Id. * The parsed connector is then added to the custom connectors map, with the option to perform additional diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java index 5f32917a6..0e597c3b5 100644 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/helper/ConfigHelper.java @@ -66,8 +66,9 @@ import org.sentrysoftware.metricshub.agent.config.AlertingSystemConfig; import org.sentrysoftware.metricshub.agent.config.ResourceConfig; import org.sentrysoftware.metricshub.agent.config.ResourceGroupConfig; +import org.sentrysoftware.metricshub.agent.connector.AdditionalConnectorsParsingResult; +import org.sentrysoftware.metricshub.agent.connector.ConnectorVariablesLibraryParser; import org.sentrysoftware.metricshub.agent.context.MetricDefinitions; -import org.sentrysoftware.metricshub.agent.deserialization.AdditionalConnectorsParsingResult; import org.sentrysoftware.metricshub.agent.security.PasswordEncrypt; import org.sentrysoftware.metricshub.engine.common.exception.InvalidConfigurationException; import org.sentrysoftware.metricshub.engine.common.helpers.JsonHelper; diff --git a/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParserTest.java b/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParserTest.java index 51895ca98..c7a2cd6f5 100644 --- a/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParserTest.java +++ b/metricshub-agent/src/test/java/org/sentrysoftware/metricshub/agent/helper/ConnectorVariablesLibraryParserTest.java @@ -12,7 +12,8 @@ import java.util.Set; import org.junit.jupiter.api.Test; import org.sentrysoftware.metricshub.agent.config.AdditionalConnector; -import org.sentrysoftware.metricshub.agent.deserialization.AdditionalConnectorsParsingResult; +import org.sentrysoftware.metricshub.agent.connector.AdditionalConnectorsParsingResult; +import org.sentrysoftware.metricshub.agent.connector.ConnectorVariablesLibraryParser; import org.sentrysoftware.metricshub.engine.connector.model.Connector; import org.sentrysoftware.metricshub.engine.connector.model.identity.Detection; import org.sentrysoftware.metricshub.engine.connector.model.identity.criterion.SnmpGetNextCriterion; From f00bff694ccd551cb0753e944afb6d4be625777a Mon Sep 17 00:00:00 2001 From: Elyes Cherfa Date: Fri, 25 Oct 2024 18:37:27 +0200 Subject: [PATCH 3/8] Issue #435: Add the possibility to use the same connector with different connector variables * Updated Documentation. --- .../ConnectorVariablesLibraryParser.java | 2 +- .../configuration/configure-monitoring.md | 77 +++++++++++++++++-- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java index 5b6053f1e..bbb0b807a 100644 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java @@ -95,7 +95,7 @@ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IO // Normalize additionalConnectors normalizeAdditionalConnectors(); - // Removing all the configurations that are not using this connector. + // Filtering all the configurations that are not using this connector. final Map filteredConnectors = additionalConnectorConfig .entrySet() .stream() diff --git a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md index 8c6b02a3e..ce7ab2f70 100644 --- a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md +++ b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md @@ -826,15 +826,53 @@ patchDirectory: /opt/patch/connectors # Replace with the path to your patch conn loggerLevel: ... ``` -#### Configure Connector Variables +#### Configure connector variables -In **MetricsHub**, connector variables are essential for customizing the behavior of data collection. These variables are configured in the `additionalConnectors` section of your `metricshub.yaml` file. When configuring variables for connectors with variables, the connector becomes an additional connector, which is by default forced, and uses the variables configured by the user. +In **MetricsHub**, connector variables are essential for customizing the behavior of data collection. These variables allow you to adapt connectors to different scenarios by providing specific configurations tailored to their needs. -Each additional connector is identified by its ID, and the variables are specified under this ID in the additionalConnectors section. The variables consist of key-value pairs, where the key corresponds to a variable already defined in the connector. +##### Definition -* Example : +A variable in a connector represents a configurable parameter that controls specific aspects of the connector’s functionality. By defining these variables, you can customize the behavior of connectors. - Below is a configuration using the `WindowsProcess` connector. The `processName` variable, defined in the variables section, specifies a list of process names (msedge.exe and metricshub.exe) to monitor: +##### Use cases + +Configuring variables enables you to tailor connectors to their specific requirements. For instance: +- **Example 1**: You want to monitor specific processes on a Windows server, so you configure the `matchName` variable to specify which processes to include. +- **Example 2**: You want to monitor specific services on a Linux machine, so you configure the `serviceNames` variable to specify which services to include. + +##### Configuration Process + +To configure variables, you need to specify them under the `variables` section for each connector in the `additionalConnectors` section of the configured resource. Each variable should be defined as a key-value pair, where the key is the variable name specifying a desired value. + +```yaml +resources: + : + attributes: + host.name: + host.type: + additionalConnectors: + : # Unique ID. Use 'uses' if different from the original connector ID + uses: # Optional - Original ID if not in key + force: true # Optional (default: true); false for auto-detection only + variables: + : +``` + +| Property | Description | +|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| `uses` | Optional. Specifies the original connector ID to refer to. If not specified, the ID of the key is used. | +| `force` | Optional. Defaults to `true`. When set to `true`, the connector is always activated. If `false`, the connector is only activated when detected. | +| `variables` | Key-value pairs defining the variables for customizing the connector behavior. | + +To obtain the connectors that work with variables: + +1. Refer to the [`MetricsHub parameterized connectors page`](../connectors/tags/parameterized.html) +2. Click on the connector you want to use (e.g.: [WindowsProcess](../connectors/windowsprocess.html)) +3. Scroll-down to the **variables** section under `Prerequesites` and note down the relevant variables (e.g.: [WindowsProcess Prerequesites](../connectors/linuxprocess.html#prerequisites)). + +##### Example 1: + +Below is a configuration using the `WindowsProcess` connector. The `matchCommand` variable, defined in the `variables` section, specifies a list of process command lines `metricshub` to monitor: ```yaml resources: @@ -846,15 +884,40 @@ resources: wmi: timeout: 120 additionalConnectors: - metricshubWindowsProcess: # Unique ID. Use 'uses' if different from the original connector ID + metricshubWindowsProcess: # Unique ID. Use 'uses' if different from the original connector ID uses: WindowsProcess # Optional - Original ID if not in key force: true # Optional (default: true); false for auto-detection only variables: - processName: "('msedge.exe', 'metricshub.exe')" + matchCommand: metricshub +``` + +##### Example 2: + +Below is a configuration using the `LinuxService` connector. The `serviceNames` variable, defined in the `variables` section, specifies a list of Linux services named `metricshub` to monitor: + +```yaml +resources: + localhost: + attributes: + host.name: localhost + host.type: linux + protocols: + ssh: + timeout: 120 + additionalConnectors: + metricshubLinuxService: # Unique ID. Use 'uses' if different from the original connector ID + uses: LinuxService # Optional - Original ID if not in key + force: true # Optional (default: true); false for auto-detection only + variables: + serviceNames: metricshub ``` If a connector with variables is forced or configured under the `additionalConnectors` section, but some or all of its variables are not specified, the missing variables will be automatically replaced with the default values defined in the `defaultVariables` section of the connector. +##### Locating Variables for Configuration + +The variables that can be configured are defined in the connector's documentation or the `defaultVariables` section within the connector's definition. Users should refer to these sections to identify the available variables and their default values. + #### Filter monitors A monitor is any entity tracked by **MetricsHub** within the main resource, such as processes, services, storage volumes, or physical devices like disks. From 47fed536daa7d05078554688baaaf1bb08487b29 Mon Sep 17 00:00:00 2001 From: Elyes Cherfa Date: Fri, 25 Oct 2024 18:39:34 +0200 Subject: [PATCH 4/8] IssueIssue #435: Add the possibility to use the same connector with different connector variables * Reformatted the code. --- .../connector/ConnectorVariablesLibraryParser.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java index bbb0b807a..2e370e921 100644 --- a/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java +++ b/metricshub-agent/src/main/java/org/sentrysoftware/metricshub/agent/connector/ConnectorVariablesLibraryParser.java @@ -195,11 +195,11 @@ private static Map getDefaultConnectorVariables(final JsonNode c * @param defaultVariables A map of default variables to be used during connector generation. */ private void generateNewConnectors( - final Path path, - final String filename, - final Map filteredConnectors, - final Map defaultVariables - ) { + final Path path, + final String filename, + final Map filteredConnectors, + final Map defaultVariables + ) { // For each configuration, we create a new custom connector and a new variables map to be used in the connector update. for (final Entry connectorConfigurationEntry : filteredConnectors.entrySet()) { final String additionalConnectorId = connectorConfigurationEntry.getKey(); From e4367b884b7a2c43a5e3023140df56c25aaa79ad Mon Sep 17 00:00:00 2001 From: Elyes Cherfa Date: Fri, 25 Oct 2024 18:43:54 +0200 Subject: [PATCH 5/8] Issue #435: Add the possibility to use the same connector with different connector variables * Updated configure-monitoring.md --- .../src/site/markdown/configuration/configure-monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md index e22d4ed4f..37b2e8463 100644 --- a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md +++ b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md @@ -874,7 +874,7 @@ To obtain the connectors that work with variables: 1. Refer to the [`MetricsHub parameterized connectors page`](../connectors/tags/parameterized.html) 2. Click on the connector you want to use (e.g.: [WindowsProcess](../connectors/windowsprocess.html)) -3. Scroll-down to the **variables** section under `Prerequesites` and note down the relevant variables (e.g.: [WindowsProcess Prerequesites](../connectors/linuxprocess.html#prerequisites)). +3. Scroll-down to the **variables** section under `Prerequesites` and note down the relevant variables (e.g.: [WindowsProcess Prerequesites](../connectors/windowsprocess.html#prerequisites)). ##### Example 1: From 9272ce9467b00923ed02c0f42008e9329ea5a72a Mon Sep 17 00:00:00 2001 From: isabelle-guitton Date: Mon, 28 Oct 2024 11:47:24 +0100 Subject: [PATCH 6/8] Issue #435 Add the possibility to use the same connector with different connector variables values * Reviewed and updated the user documentation --- .../configuration/configure-monitoring.md | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md index 37b2e8463..650552ad3 100644 --- a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md +++ b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md @@ -832,23 +832,23 @@ patchDirectory: /opt/patch/connectors # Replace with the path to your patch conn loggerLevel: ... ``` -#### Configure connector variables +#### Customize data collection -In **MetricsHub**, connector variables are essential for customizing the behavior of data collection. These variables allow you to adapt connectors to different scenarios by providing specific configurations tailored to their needs. +**MetricsHub** allows you to customize data collection on your Windows or Linux servers, specifying exactly which processes or services to monitor. This customization is achieved by configuring the following connector variables: -##### Definition +| Connector Variable | Available for | Usage | +|---|---|---| +| `matchCommand` | [Linux - Processes (ps)](https://sentrysoftware.org/metricshub-community-connectors/connectors/linuxprocess.html#linux---processes-28ps-29)
[Windows - Processes (WMI)](https://sentrysoftware.org/metricshub-community-connectors/connectors/windowsprocess.html)| Used to specify the command lines to monitor on a Linux or Windows server. | +| `matchName` | [Linux - Processes (ps)](https://sentrysoftware.org/metricshub-community-connectors/connectors/linuxprocess.html#linux---processes-28ps-29)
[Windows - Processes (WMI)](https://sentrysoftware.org/metricshub-community-connectors/connectors/windowsprocess.html) | Used to specify the processes to monitor on a Linux or Windows server. | +| `matchUser` | [Linux - Processes (ps)](https://sentrysoftware.org/metricshub-community-connectors/connectors/linuxprocess.html#linux---processes-28ps-29) | Used to specify the users to include. | +|`serviceNames` | [Linux - Service (systemctl)](https://sentrysoftware.org/metricshub-community-connectors/connectors/linuxservice.html)
[Windows - Services (WMI)](https://sentrysoftware.org/metricshub-community-connectors/connectors/windowsservice.html#!#windows---services-28wmi-29) | Used to specify the services to monitor on a Linux or Windows server. | -A variable in a connector represents a configurable parameter that controls specific aspects of the connector’s functionality. By defining these variables, you can customize the behavior of connectors. +Refer to the [Connectors directory](https://sentrysoftware.org/metricshub-community-connectors/metricshub-connectors-directory.html#) and more especially to the `Variables` section of the connector to know the supported variables and their accepted values. -##### Use cases -Configuring variables enables you to tailor connectors to their specific requirements. For instance: -- **Example 1**: You want to monitor specific processes on a Windows server, so you configure the `matchName` variable to specify which processes to include. -- **Example 2**: You want to monitor specific services on a Linux machine, so you configure the `serviceNames` variable to specify which services to include. +##### Procedure -##### Configuration Process - -To configure variables, you need to specify them under the `variables` section for each connector in the `additionalConnectors` section of the configured resource. Each variable should be defined as a key-value pair, where the key is the variable name specifying a desired value. +In the `config/metricshub.yaml` file, locate the resource for which you wish to customize data collection and specify the `variables` attribute available under the `additionalConnectors` section: ```yaml resources: @@ -866,19 +866,16 @@ resources: | Property | Description | |------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| -| `uses` | Optional. Specifies the original connector ID to refer to. If not specified, the ID of the key is used. | -| `force` | Optional. Defaults to `true`. When set to `true`, the connector is always activated. If `false`, the connector is only activated when detected. | -| `variables` | Key-value pairs defining the variables for customizing the connector behavior. | - -To obtain the connectors that work with variables: +| ` ` | Custom ID for this additional connector. | +| `uses` | *(Optional)* Provide an ID for this additional connector. If not specified, the key ID will be used. | +| `force` | *(Optional)* Set to `false` if you want the connector to only be activated when detected (Default: `true` - always activated).| +| `variables` | Specify the connector variable to be used and its value (Format: `: `).| -1. Refer to the [`MetricsHub parameterized connectors page`](../connectors/tags/parameterized.html) -2. Click on the connector you want to use (e.g.: [WindowsProcess](../connectors/windowsprocess.html)) -3. Scroll-down to the **variables** section under `Prerequesites` and note down the relevant variables (e.g.: [WindowsProcess Prerequesites](../connectors/windowsprocess.html#prerequisites)). +> Note: If a connector is added under the `additionalConnectors` section with missing or unspecified variables, those variables will automatically be populated with default values defined in the connector’s `defaultVariables` section. -##### Example 1: +##### Example 1: Collecting data for the metricshub process command line on a Windows server -Below is a configuration using the `WindowsProcess` connector. The `matchCommand` variable, defined in the `variables` section, specifies a list of process command lines `metricshub` to monitor: +In this example, we created a `metricshubWindowsProcess` additional connector using the `WindowsProcess` connector. This connector will always be actived and monitor the `metricshub` process command lines: ```yaml resources: @@ -897,9 +894,9 @@ resources: matchCommand: metricshub ``` -##### Example 2: +##### Example 2: Collecting data for the metricshub service running on a Linux server -Below is a configuration using the `LinuxService` connector. The `serviceNames` variable, defined in the `variables` section, specifies a list of Linux services named `metricshub` to monitor: +In this example, we created a `metricshubLinuxService` additional connector using the `LinuxService` connector. This connector will always be actived and monitor the `metricshub` service running on our Linux server: ```yaml resources: @@ -918,12 +915,6 @@ resources: serviceNames: metricshub ``` -If a connector with variables is forced or configured under the `additionalConnectors` section, but some or all of its variables are not specified, the missing variables will be automatically replaced with the default values defined in the `defaultVariables` section of the connector. - -##### Locating Variables for Configuration - -The variables that can be configured are defined in the connector's documentation or the `defaultVariables` section within the connector's definition. Users should refer to these sections to identify the available variables and their default values. - #### Filter monitors A monitor is any entity tracked by **MetricsHub** within the main resource, such as processes, services, storage volumes, or physical devices like disks. From 41a4be58c5af3b2cef6c24664a030d997f4090c8 Mon Sep 17 00:00:00 2001 From: isabelle-guitton Date: Wed, 30 Oct 2024 11:45:44 +0100 Subject: [PATCH 7/8] Issue #435 Fixed a typo --- .../src/site/markdown/configuration/configure-monitoring.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md index 650552ad3..c6aadc497 100644 --- a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md +++ b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md @@ -875,7 +875,7 @@ resources: ##### Example 1: Collecting data for the metricshub process command line on a Windows server -In this example, we created a `metricshubWindowsProcess` additional connector using the `WindowsProcess` connector. This connector will always be actived and monitor the `metricshub` process command lines: +In this example, we created a `metricshubWindowsProcess` additional connector using the `WindowsProcess` connector. This connector will always be activated and monitor the `metricshub` process command lines: ```yaml resources: @@ -896,7 +896,7 @@ resources: ##### Example 2: Collecting data for the metricshub service running on a Linux server -In this example, we created a `metricshubLinuxService` additional connector using the `LinuxService` connector. This connector will always be actived and monitor the `metricshub` service running on our Linux server: +In this example, we created a `metricshubLinuxService` additional connector using the `LinuxService` connector. This connector will always be activated and monitor the `metricshub` service running on our Linux server: ```yaml resources: From a332ef8c708da3de3ee6940aed37b72a1ee8711f Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Mon, 4 Nov 2024 15:54:39 +0100 Subject: [PATCH 8/8] Issue #435 Add the possibility to use the same connector with different connector variables * Reformatted tables. * Updated connector links to use relative paths. --- .../configuration/configure-monitoring.md | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md index c6aadc497..6b3fa5eeb 100644 --- a/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md +++ b/metricshub-doc/src/site/markdown/configuration/configure-monitoring.md @@ -836,15 +836,14 @@ loggerLevel: ... **MetricsHub** allows you to customize data collection on your Windows or Linux servers, specifying exactly which processes or services to monitor. This customization is achieved by configuring the following connector variables: -| Connector Variable | Available for | Usage | -|---|---|---| -| `matchCommand` | [Linux - Processes (ps)](https://sentrysoftware.org/metricshub-community-connectors/connectors/linuxprocess.html#linux---processes-28ps-29)
[Windows - Processes (WMI)](https://sentrysoftware.org/metricshub-community-connectors/connectors/windowsprocess.html)| Used to specify the command lines to monitor on a Linux or Windows server. | -| `matchName` | [Linux - Processes (ps)](https://sentrysoftware.org/metricshub-community-connectors/connectors/linuxprocess.html#linux---processes-28ps-29)
[Windows - Processes (WMI)](https://sentrysoftware.org/metricshub-community-connectors/connectors/windowsprocess.html) | Used to specify the processes to monitor on a Linux or Windows server. | -| `matchUser` | [Linux - Processes (ps)](https://sentrysoftware.org/metricshub-community-connectors/connectors/linuxprocess.html#linux---processes-28ps-29) | Used to specify the users to include. | -|`serviceNames` | [Linux - Service (systemctl)](https://sentrysoftware.org/metricshub-community-connectors/connectors/linuxservice.html)
[Windows - Services (WMI)](https://sentrysoftware.org/metricshub-community-connectors/connectors/windowsservice.html#!#windows---services-28wmi-29) | Used to specify the services to monitor on a Linux or Windows server. | - -Refer to the [Connectors directory](https://sentrysoftware.org/metricshub-community-connectors/metricshub-connectors-directory.html#) and more especially to the `Variables` section of the connector to know the supported variables and their accepted values. +| Connector Variable | Available for | Usage | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| `matchCommand` | [Linux - Processes (ps)](../connectors/linuxprocess.html#linux---processes-28ps-29)
[Windows - Processes (WMI)](../connectors/windowsprocess.html) | Used to specify the command lines to monitor on a Linux or Windows server. | +| `matchName` | [Linux - Processes (ps)](../connectors/linuxprocess.html#linux---processes-28ps-29)
[Windows - Processes (WMI)](../connectors/windowsprocess.html) | Used to specify the processes to monitor on a Linux or Windows server. | +| `matchUser` | [Linux - Processes (ps)](../connectors/linuxprocess.html#linux---processes-28ps-29) | Used to specify the users to include. | +| `serviceNames` | [Linux - Service (systemctl)](../connectors/linuxservice.html)
[Windows - Services (WMI)](../connectors/windowsservice.html#!#windows---services-28wmi-29) | Used to specify the services to monitor on a Linux or Windows server. | +Refer to the [Connectors directory](../metricshub-connectors-directory.html#) and more especially to the `Variables` section of the connector to know the supported variables and their accepted values. ##### Procedure @@ -864,14 +863,14 @@ resources: : ``` -| Property | Description | -|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| -| ` ` | Custom ID for this additional connector. | -| `uses` | *(Optional)* Provide an ID for this additional connector. If not specified, the key ID will be used. | -| `force` | *(Optional)* Set to `false` if you want the connector to only be activated when detected (Default: `true` - always activated).| -| `variables` | Specify the connector variable to be used and its value (Format: `: `).| +| Property | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------ | +| ` ` | Custom ID for this additional connector. | +| `uses` | _(Optional)_ Provide an ID for this additional connector. If not specified, the key ID will be used. | +| `force` | _(Optional)_ Set to `false` if you want the connector to only be activated when detected (Default: `true` - always activated). | +| `variables` | Specify the connector variable to be used and its value (Format: `: `). | -> Note: If a connector is added under the `additionalConnectors` section with missing or unspecified variables, those variables will automatically be populated with default values defined in the connector’s `defaultVariables` section. +> Note: If a connector is added under the `additionalConnectors` section with missing or unspecified variables, those variables will automatically be populated with default values defined by the connector itself. ##### Example 1: Collecting data for the metricshub process command line on a Windows server @@ -929,11 +928,11 @@ You can apply monitor inclusion or exclusion in data collection for the followin This is done by adding the `monitorFilters` parameter in the relevant section of the `config/metricshub.yaml` file as described below: -| Filter monitors | Add monitorFilters | -|----------------------------------------------------|---| -| For all resources |In the global section (top of the file) | +| Filter monitors | Add monitorFilters | +| -------------------------------------------------- | ------------------------------------------------------- | +| For all resources | In the global section (top of the file) | | For all the resources of a specific resource group | Under the corresponding `` section | -| For a specific resource |Under the corresponding `` section| +| For a specific resource | Under the corresponding `` section | The `monitorFilters` parameter accepts the following values: