Readymetrics (#81)

* Added new ProcessKey extension to resolve ETWProcessIndex from ProcessKey

* Fix .NET 4.8 app.config binding to System.Runtime.CompilerServices.Unsafe

* Suppress windows platfrom warnings.

* - Extract average process priority from CPU sampling data.

* Fix GetProcessIndexByPidAtTime which would not return the index when process start or end time was given.

* - Dump Process Priority to CSV and print besides exe
- Added -NoPriority flag to get old output

* -Extend Percentile to decimal and long

* - Collect for Ready Times idle and non idle times
- Print idle and non-idle ready times
- Fixed column algnment

* For .NET 8.0 use     <GarbageCollectionAdaptationMode>1</GarbageCollectionAdaptationMode> to use server GC with a dynamic number of GC Heaps instead of the currently hard coded 4.
This shows significant memory savings in high memory pressure scenarios.

* - Fix BuildProcessIndexCache during extraction. Process name can be null for processes which are still existing as ID but no name can be retrieved.

* - Fix Priority alignment when -NoPriority is used in CPU Total mode
- Reordered CSV columns and added NonIdle Ready Times
    The idle and non idle cswitch counts can be off by the total context switch count because for each method
    we only calculate percentiles if at least 50 CSwitch events were found.
- Fix output regressions due to new column in tests

* Add extended CPU Ready Metric integrationt tests

* - Fix bug slice.PreviousActivityOnProcessor.Process.Id does not return the idle process (it is always 0) instead slice?.Wait?.ReadyThreadEvent?.ReadyingProcess?.Id is the correct one to get the extracted data to match with WPA.
- Adapted test data to match with values displayed in WPA

* Fix unit tests

* ETWAnalyzer 3.0.0.1
Renamed properties to reflect that we measure C1 states from idle and own process which we call DeeSleep Read Times. All others are non deep sleep states.

* Renamed detailed output and made output more compact.
Added sum of all values which are >99% of all events to make it easy to check if some mount everest outliers are responsible for the majority of delays

* - Added to Ready data >99% sum to get total sum and sum of mount everest outliers to be able to judge the overal distribution shape.
- Fixed formatting when only stacktags are visible
- Added to CSV data Sum and 99% sum columns

* - Extract P/E core consumption when CPU has E-Cores present regardless if we have collected Frequency data.

* - Dump CPU print frequency data only when we did collect frqeuency data.
- Renamed ReadyTimes to stay consistent with file naming

* -Added to -Dump CPU -SortOrder Priority which only sorts in CPU total mode.

* - Extract Power
   Parse events not exposed by TraceProcessing on our own:
   Autonomous Mode
   BaseProfile
   HeteroPolicy
   HeteroThread
   HeteroThreadShort
- Dump Power print newly exposed properties
- Dump Power fixed issue where too many files were supressed in diff mode

* -Dump Power fix file name width formatting.

* -dump Power optimize single file name case output.

* Added to -dump Power -extract Power currently active power profile to extract and displayed data.

* -Dump Power add to CSV newly added columns.

* Revert change to use slice?.Wait?.ReadyThreadEvent?.ReadyingProcess?.Id == WindowsConstants.IdleProcessId and use slice.PreviousActivityOnProcessor.Process.Id == WindowsConstants.IdleProcessId && slice?.SwitchIn?.ContextSwitch?.PreviousCState == 1 again which is the correct way. When PreviousCState is 1 it must have been switched by the dile thread.

* -Dump Stats print CPU Topology for P and E Cores in NumberOfProcessors and SpeedMHz

* Fix: Spurious Frequency data even when not explicitely recorded.

* Error messge spelling

* Simplify Code

* Fix issue when CPU topology returns wrong core count

* -Dump Power fix issue if multiple files contain the same power setting but in the end 0 files are printed.
  Now we print the skipped files and keep of each group the first file so we print one file of an identical power settings group.

* -Dump Power fix skip message.

* Fix: Do not crash when CPU topology data is not present.

* -Extract
  - Added -NoCSwitch and -NoSampling to ignore CPU sampling or context switch data when extracing CPU or stacktag data.
  - Added extraction of process pirorities via Context Switch events in addition to CPU sampling data.

ETWAnalyzer/Analyzers/ProcessExtensions.cs

@@ -168,18 +168,30 @@ public static ETWProcess FindSingleProcessByName(TestDataFile file, string exeNa
         }
 
         /// <summary>
-        /// Find a process by its process key from a TestDataFile
+        /// Find a process by its process key from a TestDataFile. You should cache the results since it involves an array lookup!
         /// </summary>
         /// <param name="file"></param>
         /// <param name="key">ProcessKey</param>
         /// <returns>Found process or null if it was not found.</returns>
         public static ETWProcess FindProcessByKey(this TestDataFile file, ProcessKey key)
         {
-            if( key == null )
+            return key.FindProcessByKey(file.Extract);
+        }
+
+        /// <summary>
+        /// Find process by key in extracted data. You should cache the results since it involves an array lookup!
+        /// </summary>
+        /// <param name="extract"></param>
+        /// <param name="key"></param>
+        /// <returns>Process or null if process was not found.</returns>
+        public static ETWProcess FindProcessByKey(this ProcessKey key, IETWExtract extract)
+        {
+            if (key == null)
             {
                 return null;
             }
-            return file.Extract.Processes.Where(x => x.ProcessName != null && (x.ProcessID == key.Pid && key.Name == x.ProcessName)  && ( x.StartTime == key.StartTime ) ).FirstOrDefault();
+
+            return extract.Processes.Where(x => x.ProcessName != null && (x.ProcessID == key.Pid && key.Name == x.ProcessName) && (x.StartTime == key.StartTime)).FirstOrDefault();
         }
 
         /// <summary>

ETWAnalyzer/App.config

@@ -12,8 +12,12 @@
     <gcServer enabled="true"/>
     <GCNoAffinitize enabled="true"/>
     <GCHeapCount enabled="4"/>
+    <dependentAssembly>
+      <assemblyIdentity name="System.Runtime.CompilerServices.Unsafe" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
+      <bindingRedirect oldVersion="6.0.0.0" newVersion="5.0.0.0" />
+    </dependentAssembly>
   </runtime>
-    <applicationSettings>
+  <applicationSettings>
         <ETWAnalyzer.Settings>
             <setting name="SymbolDownloadFolder" serializeAs="String">
                 <value>C:\Symbols</value>

ETWAnalyzer/Commands/DumpCommand.cs

@@ -168,18 +168,20 @@ class DumpCommand : ArgParser
         "                         -topNMethods dd nn         Include dd most expensive methods/stacktags which consume most CPU in trace. Optional nn skips the first nn lines." + Environment.NewLine +
         "                         -ThreadCount               Show # of unique threads that did execute that method." + Environment.NewLine +
         "                         -ProcessFmt timefmt        Add besides process name start/stop time and duration. See -TimeFmt for available options." + Environment.NewLine +
-        "                         -SortBy [CPU/Wait/CPUWait/CPUWaitReady/ReadyAvg/CSwitchCount/StackDepth/First/Last/TestTime/StartTime] Default method sort order is CPU consumption. Wait sorts by wait time, First/Last sorts by first/last occurrence of method/stacktags." + Environment.NewLine +
+        "                         -SortBy [CPU/Wait/CPUWait/CPUWaitReady/ReadyAvg/CSwitchCount/StackDepth/First/Last/Priority/TestTime/StartTime] Default method sort order is CPU consumption. Wait sorts by wait time, First/Last sorts by first/last occurrence of method/stacktags." + Environment.NewLine +
         "                                                    StackDepth shows hottest methods which consume most CPU but are deepest in the call stack." + Environment.NewLine +
         "                                                    StartTime sorts by process start time to correlate things in the order the processes were started." + Environment.NewLine +   
         "                                                    TestTime can be used to force sort order of files by test time when -ShowTotal is used. When totals are enabled the files are sorted by highest totals." + Environment.NewLine +
+        "                                                    Sorting by process Priority is only applicable when you sort CPU totals without methods. " + Environment.NewLine + 
         "                         -MinMaxCSwitchCount xx-yy or xx  Filter by context switch count." + Environment.NewLine +
         "                         -MinMaxReadyAvgus xx-yy    Filter by Ready Average time in us." + Environment.NewLine +
         "                         -MinMaxReadyMs xx-yy or xx Only include methods (stacktags have no recorded ready times) with a minimum ready time of [xx, yy] ms." + Environment.NewLine +
         "                         -MinMaxCpuMs xx-yy or xx   Only include methods/stacktags with a minimum CPU consumption of [xx,yy] ms." + Environment.NewLine +
         "                         -MinMaxWaitMs xx-yy or xx  Only include methods/stacktags with a minimum wait time of [xx,yy] ms." + Environment.NewLine +
         "                         -Details                   Show additionally Session Id, Ready Average time, Context Switch Count, average CPU frequency per CPU efficiency class and ready percentiles." + Environment.NewLine +
         "                           -Normalize               Normalize CPU time to 100% of CPU frequency. Enables comparison of CPU time independant of the used power profile." + Environment.NewLine +
-        "                           -NoFrequency             When -Details is present do not print average CPU frequency and CPU usage per processor efficiency class (e.g. P/E Cores)." + Environment.NewLine +
+        "                           -NoFrequency             When -Details is present do not print P/E core CPU usage and average frequency." + Environment.NewLine +
+        "                         -NoPriority                Omit process Priority in total cpu mode and when methods are printed in -Details mode." + Environment.NewLine +   
         "                         -NoReady                   Do not print Ready time, average or percentiles (when -Details is used) per method." + Environment.NewLine +
         "                         -Session dd;yy             Filter processes by Windows session id. Multiple filters are separated by ;" + Environment.NewLine +
         "                                                    E.g. dd;dd2 will filter for all dd instances and dd2. The wildcards * and ? are supported for all filter strings." + Environment.NewLine +
@@ -653,6 +655,7 @@ internal enum SortOrders
             StartTime,
             ReadyAvg,
             CSwitchCount,
+            Priority,
 
             // Process sort order
             StopTime,
@@ -831,6 +834,8 @@ internal enum ZeroTimeModes
         public MinMaxRange<int> MinMaxCSwitch { get; private set; } = new();
         public bool NoReadyDetails { get; private set; }
         public bool NoFrequencyDetails { get; private set; }
+        public bool NoPriorityDetails { get; private set; }
+
         public bool Normalize { get; private set; }
 
 
@@ -1450,6 +1455,9 @@ public override void Parse()
                     case "-nofrequency":
                         NoFrequencyDetails = true;
                         break;
+                    case "-nopriority":
+                        NoPriorityDetails = true;
+                        break;
                     case "-normalize":
                         Normalize = true;
                         break;
@@ -1811,6 +1819,7 @@ public override void Run()
                             MinMaxReadyMs = MinMaxReadyMs,
                             NoReadyDetails = NoReadyDetails,
                             NoFrequencyDetails = NoFrequencyDetails,
+                            NoPriorityDetails = NoPriorityDetails,
                             Normalize = Normalize,
                             MinMaxReadyAverageUs = MinMaxReadyAverageUs,
                             MinMaxCSwitch = MinMaxCSwitch,

ETWAnalyzer/Commands/ExtractCommand.cs

@@ -90,6 +90,8 @@ class ExtractCommand : ArgParser
          "                      must have been successfully loaded during extraction. Otherwise a warning is printed due to symbol loading errors." + Environment.NewLine +
          " -keepTemp            If you want to analyze the ETL files more than once from compressed files you can use this option to keep the uncompressed ETL files in the output folder. " + Environment.NewLine +
          " -allCPU              By default only methods with CPU or Wait > 10 ms are extracted. Used together with -extract CPU." + Environment.NewLine +
+         " -NoSampling          Do not process CPU sampling data. Sometimes VMWare VMs produce invalid CPU sampling data. In that case you can ignore it and process only the Context Switch data. " + Environment.NewLine + 
+         " -NoCSwitch           Do not process Context Switch data. Useful to reduce memory consumption if you do not need CPU Wait/Ready timings." + Environment.NewLine +
          " -NoReady             By default when Context switch data is present an extra Json file with extended CPU data is created." + Environment.NewLine +
          "                      You will miss with -Dump CPU -Details Ready time percentiles." + Environment.NewLine +
          " -allExceptions       By default exceptions are filtered away by the rules configured in Configuration\\ExceptionFilters.xml. To get all specify this flag." + Environment.NewLine +
@@ -143,6 +145,8 @@ class ExtractCommand : ArgParser
         internal const string AllCPUArg = "-allcpu";
         internal const string ConcurrencyArg = "-concurrency";
         internal const string DryRunArg = "-dryrun";
+        internal const string NoSampling = "-nosampling";
+        internal const string NoCSwitch = "-nocswitch";
 
 
 
@@ -335,6 +339,16 @@ public enum ExtractionOptions
         /// </summary>
         public int? Concurrency { get; private set; }
 
+        /// <summary>
+        /// -NoSampling Ignore CPU sampling data 
+        /// </summary>
+        public bool IgnoreCPUSampling { get; private set; }
+
+        /// <summary>
+        /// -NoCSwitch Ignore Context Switch data
+        /// </summary>
+        public bool IgnoreCSwitchData { get; private set; } 
+
 
         /// <summary>
         /// Create an extract command with given command line switches
@@ -418,6 +432,12 @@ public override void Parse()
                     case ChildArg: // -child
                         IsChildProcess = true;
                         break;
+                    case NoSampling:
+                        IgnoreCPUSampling = true;
+                        break;
+                    case NoCSwitch:
+                        IgnoreCSwitchData = true;
+                        break;
                     case AllCPUArg:   // -allcpu
                         ExtractAllCPUData = true;
                         break;
@@ -493,7 +513,7 @@ public override void Parse()
             }
 
             ConfigureExtractors(Extractors, myProcessingActionList);
-            SetExtractorFilters(Extractors, ExtractAllCPUData, DisableExceptionFilter, TimelineDataExtractionIntervalS, Concurrency, NoReady);
+            SetExtractorFilters(Extractors, ExtractAllCPUData, DisableExceptionFilter, TimelineDataExtractionIntervalS, Concurrency, NoReady, IgnoreCPUSampling, IgnoreCSwitchData);
 
         }
 
@@ -568,21 +588,25 @@ private static void SortDependantExtractors(List<ExtractorBase> extractors)
             }
         }
 
-        static void SetExtractorFilters(List<ExtractorBase> extractors, bool extractAllCpuData, bool disableExceptionFilter, float? timelineExtractionInterval, int ?concurrency, bool noReady)
+        static void SetExtractorFilters(List<ExtractorBase> extractors, bool extractAllCpuData, bool disableExceptionFilter, float? timelineExtractionInterval, int ?concurrency, bool noReady, bool noSampling, bool noCSwitch)
         {
             var cpu = extractors.OfType<CPUExtractor>().SingleOrDefault();
             if (cpu != null)
             {
                 cpu.ExtractAllCPUData = extractAllCpuData;
                 cpu.Concurrency = concurrency;
                 cpu.NoReady = noReady;
+                cpu.NoSampling = noSampling;
+                cpu.NoCSwitch = noCSwitch;
                 cpu.TimelineDataExtractionIntervalS = timelineExtractionInterval;
             }
 
             var stacktag = extractors.OfType<StackTagExtractor>().SingleOrDefault();
             if( stacktag != null)
             {
                 stacktag.Concurrency = concurrency;
+                stacktag.NoSampling = noSampling;
+                stacktag.NoCSwitch = noCSwitch;
             }
 
             var exception = extractors.OfType<ExceptionExtractor>().SingleOrDefault();

ETWAnalyzer/ETWAnalyzer.csproj

@@ -22,8 +22,10 @@
     <PackageRequireLicenseAcceptance>True</PackageRequireLicenseAcceptance>
     <PackageProjectUrl>https://github.com/Siemens-Healthineers/ETWAnalyzer</PackageProjectUrl>
     <PackageReadmeFile>ProgramaticAccess.md</PackageReadmeFile>
-    <Version>3.0.0.0</Version>
+    <Version>3.0.0.1</Version>
     <Platforms>x64</Platforms>
+    <ServerGarbageCollection>true</ServerGarbageCollection>
+    <GarbageCollectionAdaptationMode>1</GarbageCollectionAdaptationMode>
   </PropertyGroup>
   <PropertyGroup Condition="'$(TargetFramework)' == 'net8.0-windows'">
     <SuppressTrimAnalysisWarnings>false</SuppressTrimAnalysisWarnings>

ETWAnalyzer/EventDump/DumpBase_T.cs

@@ -35,6 +35,7 @@ abstract class DumpBase<T> : DumpBase
         protected string Col_StartTime = "Start Time";
         protected string Col_Time = "Time";
         protected string Col_Machine = "Machine";
+        protected string Col_AveragePriority = "Average Priority";
 
 
         public override void Execute()