feat: Provide capability to use multiple metric dimensions

changelog/content/experimental/unreleased.md

@@ -8,8 +8,10 @@ version:
 
 - {{% tag added %}} Provide support for all label scenarios in StatsD & OpenTelemetry metric sink. This includes
 dimensions, customer & default labels.
+- {{% tag added %}} Provide support for scraping multiple metrics dimensions.
 - {{% tag changed %}} Switch to Mariner distroless base images
 - {{% tag security %}} Patch for [CVE-2023-29331](https://github.com/advisories/GHSA-555c-2p6r-68mm) (High)
+- {{% tag deprecated %}} Support for scraping single metric dimension by using `metrics[x].azureMetricConfiguration.dimension`.
 
 #### Resource Discovery
 

config/promitor/scraper/metrics.yaml

@@ -173,7 +173,7 @@ metrics:
     resources:
       - accountName: promitor-testing-resource-eu-automation-1
   - name: promitor_demo_frontdoor_backend_health_per_backend_pool
-    description: "Health percentage for a backed in Azure Front Door"
+    description: "Health percentage for a backend in Azure Front Door"
     resourceType: FrontDoor
     labels:
       app: promitor
@@ -243,7 +243,7 @@ metrics:
         type: Total
     resourceDiscoveryGroups:
       - name: data-factory-landscape
-  - name: promitor_demo_application_insights_availability
+  - name: application_insights_availability
     description: "Availability (%) of promitor.io measured in Azure Application Insights"
     resourceType: ApplicationInsights
     azureMetricConfiguration:
@@ -257,6 +257,39 @@ metrics:
       # Application Insights with data in Log Analytics
     - name: promitor-testing-resource-eu-telemetry
       resourceGroupName: promitor-testing-infrastructure-europe
+  - name: application_insights_availability_per_name
+    description: "Availability (%) of promitor.io measured in Azure Application Insights per name"
+    resourceType: ApplicationInsights
+    azureMetricConfiguration:
+      metricName: availabilityResults/availabilityPercentage
+      aggregation:
+        type: Average
+      dimension:
+        name: availabilityResult/name
+    resources:
+      # Application Insights with data in the service itself (classic, deprecated)
+      - name: promitor-testing-resource-eu-telemetry-classic
+        resourceGroupName: promitor-testing-infrastructure-europe
+        # Application Insights with data in Log Analytics
+      - name: promitor-testing-resource-eu-telemetry
+        resourceGroupName: promitor-testing-infrastructure-europe
+  - name: application_insights_availability_per_name_and_location
+    description: "Availability (%) of promitor.io measured in Azure Application Insights per name and location"
+    resourceType: ApplicationInsights
+    azureMetricConfiguration:
+      metricName: availabilityResults/availabilityPercentage
+      aggregation:
+        type: Average
+      dimensions:
+        - name: availabilityResult/name
+        - name: availabilityResult/location
+    resources:
+      # Application Insights with data in the service itself (classic, deprecated)
+    - name: promitor-testing-resource-eu-telemetry-classic
+      resourceGroupName: promitor-testing-infrastructure-europe
+      # Application Insights with data in Log Analytics
+    - name: promitor-testing-resource-eu-telemetry
+      resourceGroupName: promitor-testing-infrastructure-europe
   - name: promitor_demo_cdn_requests_discovered
     description: "Amount of requests sent to Azure CDN"
     resourceType: Cdn

src/Promitor.Agents.Scraper/Promitor.Agents.Scraper.csproj

@@ -12,11 +12,11 @@
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
-    <NoWarn>1701;1702;1591</NoWarn>
+    <NoWarn>1701;1702;1591;618</NoWarn>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
-    <NoWarn>1701;1702;1591</NoWarn>
+    <NoWarn>1701;1702;1591;618</NoWarn>
   </PropertyGroup>
 
   <ItemGroup>

src/Promitor.Agents.Scraper/Validation/MetricDefinitions/AzureMetricConfigurationValidator.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using Kusto.Language;
+using System.Linq;
 using Promitor.Core.Scraping.Configuration.Model;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
 using ResourceType = Promitor.Core.Contracts.ResourceType;
@@ -54,6 +55,11 @@ private IEnumerable<string> ValidateAzureMetricConfiguration(AzureMetricConfigur
                 }
             }
 
+            if (azureMetricConfiguration.Dimension != null && azureMetricConfiguration.Dimensions.Any())
+            {
+                errorMessages.Add("Only one of 'dimensions' and 'dimension' is allowed. Please use 'dimensions'.");
+            }
+
             var metricAggregationValidator = new MetricAggregationValidator(_metricDefaults);
             var metricsAggregationErrorMessages = metricAggregationValidator.Validate(azureMetricConfiguration.Aggregation);
             errorMessages.AddRange(metricsAggregationErrorMessages);

src/Promitor.Agents.Scraper/Validation/MetricDefinitions/ResourceTypes/EventHubsMetricValidator.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using GuardNet;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
@@ -18,8 +17,12 @@ public IEnumerable<string> Validate(MetricDefinition metricDefinition)
 
             var errorMessages = new List<string>();
 
-            var configuredDimension = metricDefinition.AzureMetricConfiguration?.Dimension?.Name;
-            var isEntityNameDimensionConfigured = string.IsNullOrWhiteSpace(configuredDimension) == false && configuredDimension.Equals(EntityNameDimension, StringComparison.InvariantCultureIgnoreCase);
+            if (metricDefinition.AzureMetricConfiguration?.Dimensions?.Count > 1)
+            {
+                errorMessages.Add("At least one Dimension other than EntityName is defined.");
+            }
+
+            var isEntityNameDimensionConfigured = metricDefinition.AzureMetricConfiguration?.HasDimension(EntityNameDimension) ?? false;
 
             foreach (var resourceDefinition in metricDefinition.Resources.Cast<EventHubResourceDefinition>())
             {

src/Promitor.Agents.Scraper/Validation/MetricDefinitions/ResourceTypes/ServiceBusNamespaceMetricValidator.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using GuardNet;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
@@ -12,38 +11,42 @@
     {
         private const string EntityNameDimension = "EntityName";
 
         public IEnumerable<string> Validate(MetricDefinition metricDefinition)
         {
             Guard.NotNull(metricDefinition, nameof(metricDefinition));
 
             var errorMessages = new List<string>();
 
-            var configuredDimension = metricDefinition.AzureMetricConfiguration?.Dimension?.Name;
-            var isEntityNameDimensionConfigured = string.IsNullOrWhiteSpace(configuredDimension) == false && configuredDimension.Equals(EntityNameDimension, StringComparison.InvariantCultureIgnoreCase);
+            if (metricDefinition.AzureMetricConfiguration?.Dimensions?.Count > 1)
+            {
+                errorMessages.Add("At least one Dimension other than EntityName is defined.");
+            }
+
+            var isEntityNameDimensionConfigured = metricDefinition.AzureMetricConfiguration?.HasDimension(EntityNameDimension) ?? false;
 
             foreach (var resourceDefinition in metricDefinition.Resources.Cast<ServiceBusNamespaceResourceDefinition>())
             {
                 if (string.IsNullOrWhiteSpace(resourceDefinition.Namespace))
                 {
                     errorMessages.Add("No Service Bus Namespace is configured");
                 }
 
                 if (isEntityNameDimensionConfigured && string.IsNullOrWhiteSpace(resourceDefinition.QueueName) == false)
                 {
                     errorMessages.Add($"Queue name is configured while '{EntityNameDimension}' dimension is configured as well. We only support one or the other.");
                 }
 
                 if (isEntityNameDimensionConfigured && string.IsNullOrWhiteSpace(resourceDefinition.TopicName) == false)
                 {
                     errorMessages.Add($"Topic name is configured while '{EntityNameDimension}' dimension is configured as well. We only support one or the other.");
                 }
 
                 if (string.IsNullOrWhiteSpace(resourceDefinition.QueueName) == false && string.IsNullOrWhiteSpace(resourceDefinition.TopicName) == false)
                 {
                     errorMessages.Add("Queue & topic name are both configured while we only support one or the other.");
                 }
             }
 
             return errorMessages;
         }
     }

src/Promitor.Core.Scraping/AzureMonitorScraper.cs

@@ -1,14 +1,15 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using GuardNet;
 using Microsoft.Azure.Management.Monitor.Fluent.Models;
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Contracts;
 using Promitor.Core.Metrics;
+using Promitor.Core.Scraping.Configuration.Model;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
 using Promitor.Integrations.AzureMonitor.Exceptions;
-using MetricDimension = Promitor.Core.Scraping.Configuration.Model.MetricDimension;
 
 namespace Promitor.Core.Scraping
 {
@@ -45,27 +46,27 @@ protected override async Task<ScrapeResult> ScrapeResourceAsync(string subscript
             var metricLimit = DetermineMetricLimit(scrapeDefinition);
 
             // Determine the metric dimension to use, if any
-            var dimensionName = DetermineMetricDimension(metricName, resourceDefinition, scrapeDefinition.AzureMetricConfiguration?.Dimension);
+            var dimensionNames = DetermineMetricDimensions(metricName, resourceDefinition, scrapeDefinition.AzureMetricConfiguration);
 
-            List<MeasuredMetric> measuredMetrics = new List<MeasuredMetric>();
+            var measuredMetrics = new List<MeasuredMetric>();
             try
             {
                 // Query Azure Monitor for metrics
-                measuredMetrics = await AzureMonitorClient.QueryMetricAsync(metricName, dimensionName, aggregationType, aggregationInterval, resourceUri, metricFilter, metricLimit);
+                measuredMetrics = await AzureMonitorClient.QueryMetricAsync(metricName, dimensionNames, aggregationType, aggregationInterval, resourceUri, metricFilter, metricLimit);
             }
             catch (MetricInformationNotFoundException metricsNotFoundException)
             {
-                Logger.LogWarning("No metric information found for metric {MetricName} with dimension {MetricDimension}. Details: {Details}", metricsNotFoundException.Name, metricsNotFoundException.Dimension, metricsNotFoundException.Details);
+                Logger.LogWarning("No metric information found for metric {MetricName} with dimensions {MetricDimensions}. Details: {Details}", metricsNotFoundException.Name, metricsNotFoundException.Dimensions, metricsNotFoundException.Details);
 
-                var measuredMetric = string.IsNullOrWhiteSpace(dimensionName) ? MeasuredMetric.CreateWithoutDimension(null) : MeasuredMetric.CreateForDimension(null, dimensionName, "unknown");
+                var measuredMetric = dimensionNames.Any() ? MeasuredMetric.CreateForDimensions(dimensionNames) : MeasuredMetric.CreateWithoutDimensions(null);
                 measuredMetrics.Add(measuredMetric);
             }
 
             // Provide more metric labels, if we need to
             var metricLabels = DetermineMetricLabels(resourceDefinition);
 
             // Enrich measured metrics, in case we need to
-            var finalMetricValues = EnrichMeasuredMetrics(resourceDefinition, dimensionName, measuredMetrics);
+            var finalMetricValues = EnrichMeasuredMetrics(resourceDefinition, dimensionNames, measuredMetrics);
 
             // We're done!
             return new ScrapeResult(subscriptionId, scrapeDefinition.ResourceGroupName, resourceDefinition.ResourceName, resourceUri, finalMetricValues, metricLabels);
@@ -84,10 +85,10 @@ protected override async Task<ScrapeResult> ScrapeResourceAsync(string subscript
         ///     metrics to align with others
         /// </remarks>
         /// <param name="resourceDefinition">Contains the resource cast to the specific resource type.</param>
-        /// <param name="dimensionName">Name of the specified dimension provided by the scraper</param>
+        /// <param name="dimensionNames">List of names of the specified dimensions provided by the scraper.</param>
         /// <param name="metricValues">Measured metric values that were found</param>
         /// <returns></returns>
-        protected virtual List<MeasuredMetric> EnrichMeasuredMetrics(TResourceDefinition resourceDefinition, string dimensionName, List<MeasuredMetric> metricValues)
+        protected virtual List<MeasuredMetric> EnrichMeasuredMetrics(TResourceDefinition resourceDefinition, List<string> dimensionNames, List<MeasuredMetric> metricValues)
         {
             return metricValues;
         }
@@ -107,10 +108,16 @@ protected virtual string DetermineMetricFilter(string metricName, TResourceDefin
         /// </summary>
         /// <param name="metricName">Name of the metric being queried</param>
         /// <param name="resourceDefinition">Contains the resource cast to the specific resource type.</param>
-        /// <param name="dimension">Provides information concerning the configured metric dimension.</param>
-        protected virtual string DetermineMetricDimension(string metricName, TResourceDefinition resourceDefinition, MetricDimension dimension)
+        /// <param name="configuration"></param>
+        protected virtual List<string> DetermineMetricDimensions(string metricName, TResourceDefinition resourceDefinition, AzureMetricConfiguration configuration)
         {
-            return dimension?.Name;
+            if (configuration.Dimension != null)
+            {
+                Logger.LogWarning("AzureMetricConfiguration property 'dimension' has been deprecated and will be removed in a future update. Please use 'dimensions' instead.");
+                return string.IsNullOrWhiteSpace(configuration.Dimension.Name) ? new List<string>() : new List<string>{ configuration.Dimension.Name };
+            }
+
+            return configuration.Dimensions?.Select(dimension => dimension.Name).Where(dimensionName => !string.IsNullOrWhiteSpace(dimensionName)).Distinct().ToList();
         }
 
         /// <summary>

src/Promitor.Core.Scraping/Configuration/Model/AzureMetricConfiguration.cs

@@ -1,4 +1,8 @@
-namespace Promitor.Core.Scraping.Configuration.Model
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Promitor.Core.Scraping.Configuration.Model
 {
     public class AzureMetricConfiguration
     {
@@ -12,14 +16,34 @@ public class AzureMetricConfiguration
         /// </summary>
         public int? Limit { get; set; }
 
+        /// <summary>
+        ///     Information about the dimensions of an Azure Monitor metric
+        /// </summary>
+        public IReadOnlyCollection<MetricDimension> Dimensions { get; set; }
+
         /// <summary>
         ///     Information about the dimension of an Azure Monitor metric
         /// </summary>
+        [Obsolete("Dimension is deprecated, please use Dimensions instead.")]
         public MetricDimension Dimension { get; set; }
 
         /// <summary>
         ///     Configuration on how to aggregate the metric
         /// </summary>
         public MetricAggregation Aggregation { get; set; }
+
+        /// <summary>
+        ///    Checks whether the configuration contains a dimension with the given name.
+        /// </summary>
+        /// <param name="dimensionName">Dimension name to be checked for.</param>
+        /// <returns>true if the dimension name was found, false otherwise</returns>
+        public bool? HasDimension(string dimensionName)
+        {
+            if (Dimension != null)
+            {
+                return Dimension?.Name?.Equals(dimensionName, StringComparison.InvariantCultureIgnoreCase);
+            }
+            return Dimensions?.Any(dimension => dimension.Name.Equals(dimensionName, StringComparison.InvariantCultureIgnoreCase));
+        }
     }
 }

src/Promitor.Core.Scraping/Configuration/Serialization/Deserializer.cs

@@ -116,7 +116,12 @@ private static object GetFieldValue(
 
             if (fieldDeserializationInfo.Deserializer != null)
             {
-                return fieldDeserializationInfo.Deserializer.DeserializeObject((YamlMappingNode)fieldNodePair.Value, errorReporter);
+                return fieldNodePair.Value switch
+                {
+                    YamlMappingNode node => fieldDeserializationInfo.Deserializer.Deserialize(node, errorReporter),
+                    YamlSequenceNode node => fieldDeserializationInfo.Deserializer.Deserialize(node, errorReporter),
+                    _ => null
+                };
             }
 
             var propertyType = Nullable.GetUnderlyingType(fieldDeserializationInfo.PropertyInfo.PropertyType) ?? fieldDeserializationInfo.PropertyInfo.PropertyType;

src/Promitor.Core.Scraping/Configuration/Serialization/FieldDeserializationInfo.cs

@@ -27,7 +27,7 @@ public FieldDeserializationInfo(
             bool isRequired,
             object defaultValue,
             Func<string, KeyValuePair<YamlNode, YamlNode>, IErrorReporter, object> customMapperFunc,
-            IDeserializer deserializer,
+            IDeserializer<object> deserializer,
             IReadOnlyCollection<IFieldValidator> validators)
         {
             YamlFieldName = GetName(propertyInfo);
@@ -67,7 +67,7 @@ public FieldDeserializationInfo(
         /// <summary>
         /// Gets a deserializer to use when deserializing the field.
         /// </summary>
-        public IDeserializer Deserializer { get; }
+        public IDeserializer<object> Deserializer { get; }
 
         /// <summary>
         /// Gets the custom validators for the field.

src/Promitor.Core.Scraping/Configuration/Serialization/FieldDeserializationInfoBuilder.cs

@@ -17,7 +17,7 @@ public class FieldDeserializationInfoBuilder<TObject, TReturn> : IFieldDeseriali
         private bool _isRequired;
         private object _defaultValue;
         private Func<string, KeyValuePair<YamlNode, YamlNode>, IErrorReporter, object> _customMapperFunc;
-        private IDeserializer _deserializer;
+        private IDeserializer<object> _deserializer;
 
         /// <summary>
         /// Sets the expression that defines the property to map.
@@ -71,12 +71,13 @@ public FieldDeserializationInfoBuilder<TObject, TReturn> MapUsing(Func<string, K
             return this;
         }
 
+        // ReSharper disable once InvalidXmlDocComment
         /// <summary>
-        /// Specifies an <see cref="IDeserializer" /> to use to map the field.
+        /// Specifies an <see cref="IDeserializer"/> to use to map the field.
         /// </summary>
         /// <param name="deserializer">The deserializer.</param>
         /// <returns>The builder.</returns>
-        public FieldDeserializationInfoBuilder<TObject, TReturn> MapUsingDeserializer(IDeserializer deserializer)
+        public FieldDeserializationInfoBuilder<TObject, TReturn> MapUsingDeserializer(IDeserializer<object> deserializer)
         {
             _deserializer = deserializer;
 

src/Promitor.Core.Scraping/Configuration/Serialization/IDeserializer.cs

@@ -3,25 +3,11 @@
 
 namespace Promitor.Core.Scraping.Configuration.Serialization
 {
-    /// <summary>
-    /// An object that can deserialize a yaml node into an object.
-    /// </summary>
-    public interface IDeserializer
-    {
-        /// <summary>
-        /// Deserializes the specified node.
-        /// </summary>
-        /// <param name="node">The node to deserialize.</param>
-        /// <param name="errorReporter">Used to report deserialization errors.</param>
-        /// <returns>The deserialized object.</returns>
-        object DeserializeObject(YamlMappingNode node, IErrorReporter errorReporter);
-    }
-
     /// <summary>
     /// An object that can deserialize a yaml node into an object.
     /// </summary>
     /// <typeparam name="TObject">The type of object that can be deserialized.</typeparam>
-    public interface IDeserializer<out TObject> : IDeserializer where TObject: new()
+    public interface IDeserializer<out TObject>
     {
         /// <summary>
         /// Deserializes the specified node.

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetricConfigurationDeserializer.cs

@@ -14,11 +14,13 @@ public AzureMetricConfigurationDeserializer(IDeserializer<MetricDimensionV1> dim
                 .IsRequired();
             Map(config => config.Limit)
                 .MapUsing(DetermineLimit);
-            Map(config => config.Dimension)
-                .MapUsingDeserializer(dimensionDeserializer);
             Map(config => config.Aggregation)
                 .IsRequired()
                 .MapUsingDeserializer(aggregationDeserializer);
+            Map(config => config.Dimensions)
+                .MapUsingDeserializer(dimensionDeserializer);
+            Map(config => config.Dimension)
+                .MapUsingDeserializer(dimensionDeserializer);
         }
 
         private object DetermineLimit(string rawLimit, KeyValuePair<YamlNode, YamlNode> nodePair, IErrorReporter errorReporter)