-
Notifications
You must be signed in to change notification settings - Fork 209
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Return total memory when limit is max value. (#2214)
* Update metrics providers - Implement changes to unlimited memory metric. - Remove inactive_file.bytes per elastic/apm#339 - Refactor code - Optimise code to avoid allocations. * Update and add tests * Update benchmarks and add profiling project * Update editorconfig to avoid range errors * Cleanup white space in build.fs * Linux bug fixes * Fix formatting * Remove commented code * Add code comment * Fixes after rebase
- Loading branch information
1 parent
dbbcd45
commit edf2bc1
Showing
42 changed files
with
1,326 additions
and
608 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
130 changes: 130 additions & 0 deletions
130
benchmarks/Elastic.Apm.Benchmarks/CgroupMetricsProviderBenchmarks.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| // Licensed to Elasticsearch B.V under one or more agreements. | ||
| // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. | ||
| // See the LICENSE file in the project root for more information | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using BenchmarkDotNet.Attributes; | ||
| using BenchmarkDotNet.Engines; | ||
| using Elastic.Apm.Helpers; | ||
| using Elastic.Apm.Metrics.MetricsProvider; | ||
| using Elastic.Apm.Tests.Utilities; | ||
| using static Elastic.Apm.Metrics.MetricsProvider.CgroupMetricsProvider; | ||
| using static Elastic.Apm.Tests.TestHelpers.CgroupFileHelper; | ||
|
|
||
| namespace Elastic.Apm.Benchmarks; | ||
|
|
||
| [MemoryDiagnoser] | ||
| public class CgroupMetricsProviderBenchmarks | ||
| { | ||
| private CgroupPaths _cgroupPaths; | ||
| private CgroupMetricsProvider _cgroupMetricsProvider; | ||
| private readonly Consumer _consumer = new(); | ||
|
|
||
| [GlobalSetup] | ||
| public void Setup() | ||
| { | ||
| _cgroupPaths = CreateDefaultCgroupFiles(CgroupVersion.CgroupV2); | ||
| _cgroupMetricsProvider = TestableCgroupMetricsProvider(new NoopLogger(), new List<WildcardMatcher>(), _cgroupPaths.RootPath, true); | ||
| } | ||
|
|
||
| [GlobalSetup(Target = nameof(GetSamplesUnlimited))] | ||
| public void SetupUnlimited() | ||
| { | ||
| _cgroupPaths = CreateDefaultCgroupFiles(CgroupVersion.CgroupV2); | ||
| UnlimitedMaxMemoryFiles(_cgroupPaths); | ||
| _cgroupMetricsProvider = TestableCgroupMetricsProvider(new NoopLogger(), new List<WildcardMatcher>(), _cgroupPaths.RootPath, true); | ||
| } | ||
|
|
||
| private static void UnlimitedMaxMemoryFiles(CgroupPaths paths) | ||
| { | ||
| if (paths.CgroupVersion == CgroupVersion.CgroupV1) | ||
| { | ||
| using var sr = new StreamWriter(File.Create(Path.Combine(paths.CgroupV1MemoryControllerPath, "memory.limit_in_bytes"))); | ||
| sr.WriteAsync($"9223372036854771712\n"); | ||
| } | ||
|
|
||
| if (paths.CgroupVersion == CgroupVersion.CgroupV2) | ||
| { | ||
| using var sr = new StreamWriter(File.Create(Path.Combine(paths.CgroupV2SlicePath, "memory.max"))); | ||
| sr.WriteAsync($"max\n"); | ||
| } | ||
| } | ||
|
|
||
| [GlobalCleanup] | ||
| public void Cleanup() | ||
| => Directory.Delete(_cgroupPaths.RootPath, true); | ||
|
|
||
| [Benchmark] | ||
| public void GetSamples() | ||
| { | ||
| foreach (var metricSet in _cgroupMetricsProvider.GetSamples()) | ||
| { | ||
| metricSet.Samples.Consume(_consumer); | ||
| } | ||
| } | ||
|
|
||
| // WINDOWS: | ||
| //| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio | | ||
| //|-------------------- |---------:|--------:|---------:|---------:|--------:|-------:|----------:|------------:| | ||
| //| GetSamplesOriginal | 433.0 us | 8.39 us | 11.20 us | baseline | | 1.9531 | 29.4 KB | | | ||
| //| GetSamplesOptimised | 409.9 us | 5.11 us | 4.53 us | -5% | 3.0% | - | 1.47 KB | -95% | | ||
|
|
||
| // WINDOWS: After yield return | ||
| //| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio | | ||
| //|-------------------- |---------:|--------:|---------:|---------:|--------:|-------:|----------:|------------:| | ||
| //| GetSamplesOriginal | 429.1 us | 8.08 us | 13.93 us | baseline | | 1.9531 | 29.44 KB | | | ||
| //| GetSamplesOptimised | 419.7 us | 6.83 us | 6.39 us | -5% | 3.3% | - | 1.46 KB | -95% | | ||
|
|
||
| // WINDOWS: Remove metric | ||
| //| Method | Mean | Error | StdDev | Allocated | Alloc Ratio | | ||
| //|-------------------- |---------:|--------:|--------:|----------:|------------:| | ||
| //| GetSamples | 296.9 us | 3.46 us | 3.23 us | 1.05 KB | -96% | | ||
|
|
||
| // LINUX: After yield return | ||
| //| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | | ||
| //|-------------------- |---------:|----------:|----------:|---------:|--------:|-------:|-------:|----------:|------------:| | ||
| //| GetSamplesOriginal | 9.643 us | 0.1600 us | 0.1418 us | baseline | | 2.3346 | 0.0610 | 29328 B | | | ||
| //| GetSamplesOptimised | 5.525 us | 0.0580 us | 0.0569 us | -43% | 1.6% | 0.0534 | - | 680 B | -98% | | ||
|
|
||
| // LINUX: Remove metric | ||
| //| Method | Mean | Error | Ratio | StdDev | Gen0 | Allocated | Alloc Ratio | | ||
| //|-------------------- |---------:|----------:|---------:|----------:|-------:|----------:| | | ||
| //| GetSamples | 4.363 us | 0.0458 us | -55% | 0.0382 us | 0.0381 | 496 B | -98% | | ||
|
|
||
| [Benchmark] | ||
| public void GetSamplesUnlimited() | ||
| { | ||
| foreach (var metricSet in _cgroupMetricsProvider.GetSamples()) | ||
| { | ||
| metricSet.Samples.Consume(_consumer); | ||
| } | ||
| } | ||
|
|
||
| // WINDOWS: | ||
| //| Method | Mean | Error | StdDev | Gen0 | Allocated | | ||
| //|-------------------- |---------:|--------:|--------:|-------:|----------:| | ||
| //| GetSamplesUnlimited | 467.1 us | 9.28 us | 9.11 us | 0.4883 | 9.5 KB | | ||
|
|
||
| // WINDOWS: Optimised | ||
| //| Method | Mean | Error | StdDev | Allocated | | ||
| //|-------------------- |---------:|--------:|--------:|----------:| | ||
| //| GetSamplesUnlimited | 444.9 us | 8.46 us | 8.69 us | 1.88 KB | | ||
|
|
||
| // WINDOWS: Remove metric | ||
| //| Method | Mean | Error | StdDev | Allocated | Alloc Ratio | | ||
| //|-------------------- |---------:|--------:|--------:|----------:| | | ||
| //| GetSamplesUnlimited | 305.4 us | 5.04 us | 4.71 us | 1.43 KB | -85% | | ||
|
|
||
| // LINUX: Optimised | ||
| //| Method | Mean | Error | StdDev | Gen0 | Allocated | | ||
| //|-------------------- |---------:|----------:|----------:|-------:|----------:| | ||
| //| GetSamplesUnlimited | 7.217 us | 0.1375 us | 0.1350 us | 0.0687 | 864 B | | ||
| //** NOTE: This includes some overhead (176 bytes) for building the test path which is not incurred | ||
| //in production. This is there equivient to the limited benchmark when this is taken into account. | ||
|
|
||
| // LINUX: Remove metric | ||
| //| Method | Mean | Error | StdDev | Gen0 | Allocated | | ||
| //|-------------------- |---------:|----------:|----------:|-------:|----------:| | ||
| //| GetSamplesUnlimited | 4.908 us | 0.0736 us | 0.0688 us | 0.0534 | 680 B | | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
55 changes: 55 additions & 0 deletions
55
benchmarks/Elastic.Apm.Benchmarks/FreeAndTotalMemoryProviderBenchmarks.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| // Licensed to Elasticsearch B.V under one or more agreements. | ||
| // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. | ||
| // See the LICENSE file in the project root for more information | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using BenchmarkDotNet.Attributes; | ||
| using BenchmarkDotNet.Engines; | ||
| using Elastic.Apm.Helpers; | ||
| using Elastic.Apm.Metrics.MetricsProvider; | ||
| using Elastic.Apm.Tests.Utilities; | ||
| using static Elastic.Apm.Tests.TestHelpers.CgroupFileHelper; | ||
|
|
||
| namespace Elastic.Apm.Benchmarks; | ||
|
|
||
| [MemoryDiagnoser] | ||
| public class FreeAndTotalMemoryProviderBenchmarks | ||
| { | ||
| private CgroupPaths _cgroupPaths; | ||
| private FreeAndTotalMemoryProvider _freeAndTotalMemoryProvider; | ||
| private readonly Consumer _consumer = new(); | ||
|
|
||
| [GlobalSetup] | ||
| public void Setup() | ||
| { | ||
| _cgroupPaths = CreateDefaultCgroupFiles(CgroupVersion.CgroupV2); | ||
| _freeAndTotalMemoryProvider = FreeAndTotalMemoryProvider | ||
| .TestableFreeAndTotalMemoryProvider(new NoopLogger(), new List<WildcardMatcher>(), _cgroupPaths.RootPath, true); | ||
| } | ||
|
|
||
| [GlobalCleanup] | ||
| public void Cleanup() | ||
| => Directory.Delete(_cgroupPaths.RootPath, true); | ||
|
|
||
| [Benchmark] | ||
| public void GetSamples() | ||
| { | ||
| foreach (var metricSet in _freeAndTotalMemoryProvider.GetSamples()) | ||
| { | ||
| metricSet.Samples.Consume(_consumer); | ||
| } | ||
| } | ||
|
|
||
| // WINDOWS | ||
| //| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio | | ||
| //|------------------- |---------:|---------:|---------:|---------:|--------:|-------:|----------:|------------:| | ||
| //| GetSamplesOriginal | 600.1 ns | 12.00 ns | 17.22 ns | baseline | | 0.0277 | 352 B | | | ||
| //| GetSamples | 553.8 ns | 10.57 ns | 12.98 ns | -8% | 3.2% | 0.0162 | 208 B | -41% | | ||
|
|
||
| // LINUX | ||
| //| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | | ||
| //|------------------- |-----------:|---------:|----------:|---------:|--------:|-------:|-------:|----------:|------------:| | ||
| //| GetSamplesOriginal | 3,693.2 ns | 73.86 ns | 108.27 ns | baseline | | 0.7324 | 0.0191 | 9216 B | | | ||
| //| GetSamples | 438.7 ns | 8.57 ns | 8.41 ns | -88% | 3.1% | 0.0215 | - | 272 B | -97% | | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
21 changes: 21 additions & 0 deletions
21
benchmarks/Elastic.Apm.Profiling/Elastic.Apm.Profiling.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFramework>net7.0</TargetFramework> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| <IsTestProject>false</IsTestProject> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="JetBrains.Profiler.Api" Version="1.4.0" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\src\Elastic.Apm\Elastic.Apm.csproj" /> | ||
| <ProjectReference Include="..\..\test\Elastic.Apm.Tests\Elastic.Apm.Tests.csproj" /> | ||
| <ProjectReference Include="..\Elastic.Apm.Benchmarks\Elastic.Apm.Benchmarks.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| // Licensed to Elasticsearch B.V under one or more agreements. | ||
| // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. | ||
| // See the LICENSE file in the project root for more information | ||
|
|
||
| using Elastic.Apm.Helpers; | ||
| using Elastic.Apm.Logging; | ||
| using Elastic.Apm.Metrics.MetricsProvider; | ||
| using JetBrains.Profiler.Api; | ||
| using static Elastic.Apm.Metrics.MetricsProvider.CgroupMetricsProvider; | ||
| using static Elastic.Apm.Tests.TestHelpers.CgroupFileHelper; | ||
|
|
||
| var paths = CreateDefaultCgroupFiles(CgroupVersion.CgroupV1); | ||
| UnlimitedMaxMemoryFiles(paths); | ||
|
|
||
| // WARMUP | ||
|
|
||
| var sut = TestableCgroupMetricsProvider(new NoopLogger(), new List<WildcardMatcher>(), paths.RootPath, true); | ||
| //var sut = new FreeAndTotalMemoryProvider(new NoopLogger(), new List<WildcardMatcher>()); | ||
| foreach (var metricSet in sut.GetSamples()) | ||
| foreach (var _ in metricSet.Samples) | ||
| { | ||
| } | ||
|
|
||
| // PROFILING | ||
|
|
||
| MemoryProfiler.CollectAllocations(true); | ||
|
|
||
| MemoryProfiler.GetSnapshot("Before create"); | ||
|
|
||
| sut = TestableCgroupMetricsProvider(new NoopLogger(), new List<WildcardMatcher>(), paths.RootPath, true); | ||
| //sut = new FreeAndTotalMemoryProvider(new NoopLogger(), new List<WildcardMatcher>()); | ||
|
|
||
| MemoryProfiler.GetSnapshot("After create"); | ||
|
|
||
| foreach (var metricSet in sut.GetSamples()) | ||
| foreach (var _ in metricSet.Samples) | ||
| { | ||
| } | ||
|
|
||
| MemoryProfiler.GetSnapshot("After get samples"); | ||
|
|
||
| MemoryProfiler.CollectAllocations(false); | ||
|
|
||
| static void UnlimitedMaxMemoryFiles(CgroupPaths paths) | ||
| { | ||
| if (paths.CgroupVersion == CgroupVersion.CgroupV1) | ||
| { | ||
| using var sr = new StreamWriter(File.Create(Path.Combine(paths.CgroupV1MemoryControllerPath, "memory.limit_in_bytes"))); | ||
| sr.WriteAsync($"9223372036854771712\n"); | ||
| } | ||
|
|
||
| if (paths.CgroupVersion == CgroupVersion.CgroupV2) | ||
| { | ||
| using var sr = new StreamWriter(File.Create(Path.Combine(paths.CgroupV2SlicePath, "memory.max"))); | ||
| sr.WriteAsync($"max\n"); | ||
| } | ||
| } | ||
|
|
||
| internal sealed class NoopLogger : IApmLogger | ||
| { | ||
| public bool IsEnabled(LogLevel level) => false; | ||
| public void Log<TState>(LogLevel level, TState state, Exception e, Func<TState, Exception, string> formatter) { } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.