[cdac] Implement NibbleMap lookup and tests

The execution manager uses a nibble map to quickly map program counter
pointers to the beginnings of the native code for the managed method.

Implement the lookup algorithm for a nibble map.

Start adding unit tests for the nibble map

Also for testing in MockMemorySpace simplify ReaderContext, there's nothing special about the descriptor HeapFragments anymore.  We can use a uniform reader.

docs/design/datacontracts/ExecutionManager.md

@@ -0,0 +1,21 @@
+# Contract ExecutionManager
+
+This contract is for mapping a PC adddress to information about the
+managed method corresponding to that address.
+
+
+## APIs of contract
+
+**TODO**
+
+## Version 1
+
+**TODO** Methods
+
+### NibbleMap
+
+Version 1 of this contract depends on a "nibble map" data structure
+that allows mapping of a code address in a contiguous subsection of
+the address space to the pointer to the start of that a code sequence.
+It takes advantage of the fact that the code starts are aligned and
+are spaced apart to represent their addresses as a 4-bit nibble value.

src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/DataType.cs

@@ -20,6 +20,7 @@ public enum DataType
     pointer,
 
     GCHandle,
+    CodePointer,
     Thread,
     ThreadStore,
     GCAllocContext,

src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/Target.cs

@@ -41,6 +41,13 @@ internal abstract class Target
     /// <returns>Pointer read from the target</returns>}
     public abstract TargetPointer ReadPointer(ulong address);
 
+    /// <summary>
+    /// Read a code pointer from the target in target endianness
+    /// </summary>
+    /// <param name="address">Address to start reading from</param>
+    /// <returns>Pointer read from the target</returns>}
+    public abstract TargetCodePointer ReadCodePointer(ulong address);
+
     /// <summary>
     /// Read some bytes from the target
     /// </summary>

src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetCodePointer.cs

@@ -0,0 +1,30 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+using System;
+
+namespace Microsoft.Diagnostics.DataContractReader;
+
+public readonly struct TargetCodePointer : IEquatable<TargetCodePointer>
+{
+    public static TargetCodePointer Null = new(0);
+    public readonly ulong Value;
+    public TargetCodePointer(ulong value) => Value = value;
+
+    public static implicit operator ulong(TargetCodePointer p) => p.Value;
+    public static implicit operator TargetCodePointer(ulong v) => new TargetCodePointer(v);
+
+    public static bool operator ==(TargetCodePointer left, TargetCodePointer right) => left.Value == right.Value;
+    public static bool operator !=(TargetCodePointer left, TargetCodePointer right) => left.Value != right.Value;
+
+    public override bool Equals(object? obj) => obj is TargetCodePointer pointer && Equals(pointer);
+    public bool Equals(TargetCodePointer other) => Value == other.Value;
+
+    public override int GetHashCode() => Value.GetHashCode();
+
+    public bool Equals(TargetCodePointer x, TargetCodePointer y) => x.Value == y.Value;
+    public int GetHashCode(TargetCodePointer obj) => obj.Value.GetHashCode();
+
+    public TargetPointer AsTargetPointer => new(Value);
+
+    public override string ToString() => $"0x{Value:x}";
+}

src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetNUInt.cs

@@ -1,11 +1,16 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 using System;
+using System.Diagnostics;
 
 namespace Microsoft.Diagnostics.DataContractReader;
 
+
+[DebuggerDisplay("{Hex}")]
 public readonly struct TargetNUInt
 {
     public readonly ulong Value;
     public TargetNUInt(ulong value) => Value = value;
+
+    internal string Hex => $"0x{Value:x}";
 }

src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetPointer.cs

@@ -23,4 +23,6 @@ namespace Microsoft.Diagnostics.DataContractReader;
     public bool Equals(TargetPointer other) => Value == other.Value;
 
     public override int GetHashCode() => Value.GetHashCode();
+
+    public override string ToString() => $"0x{Value:x}";
 }

src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs

@@ -0,0 +1,179 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Numerics;
+
+using MapUnit = uint;
+
+
+namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers;
+
+// Given a contiguous region of memory in which we lay out a collection of non-overlapping code blocks that are
+// not too small (so that two adjacent ones aren't too close together) and  where the start of each code block is preceeded by a code header aligned on some power of 2,
+// we can break up the whole memory space into buckets of a fixed size (32-bytes in the current implementation), where
+// each bucket either has a code block header or not.
+// Thinking of each code block header address as a hex number, we can view it as: [index, offset, zeros]
+// where each index gives us a bucket and the offset gives us the position of the header within the bucket.
+// We encode each offset into a 4-bit nibble, reserving the special value 0 to mark the places in the map where a method doesn't start.
+//
+// To find the start of a method given an address we first convert it into a bucket index (giving the map unit)
+// and an offset which we can then turn into the index of the nibble that covers that address.
+// If the nibble is non-zero, we have the start of a method and it is near the given address.
+// If the nibble is zero, we have to search backward first through the current map unit, and then through previous map
+// units until we find a non-zero nibble.
+#pragma warning disable SA1121 // Use built in alias
+internal sealed class NibbleMap
+{
+
+    public static NibbleMap Create(Target target)
+    {
+        uint codeHeaderSize = (uint)target.PointerSize;
+        return new NibbleMap(target, codeHeaderSize);
+    }
+
+    private readonly Target _target;
+    private readonly uint _codeHeaderSize;
+    private NibbleMap(Target target, uint codeHeaderSize)
+    {
+        _target = target;
+        _codeHeaderSize = codeHeaderSize;
+    }
+
+    // Shift the next nibble into the least significant position.
+    private static T NextNibble<T>(T n) where T : IBinaryInteger<T>
+    {
+        return n >>> 4;
+    }
+
+
+    private const uint MapUnitBytes = sizeof(MapUnit); // our units are going to be 32-bit integers
+    private const MapUnit NibbleMask = 0x0F;
+    private const ulong NibblesPerMapUnit = 2 * MapUnitBytes; // 2 nibbles per byte * N bytes per map unit
+
+    // we will partition the address space into buckets of this many bytes.
+    // There is at most one code block header per bucket.
+    // normally we would then need Log2(BytesPerBucket) bits to find the exact start address,
+    // but because code headers are aligned, we can store the offset in a 4-bit nibble instead and shift appropriately to compute
+    // the effective address
+    private const ulong BytesPerBucket = 8 * sizeof(MapUnit);
+
+
+    // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that
+    // nible in the least significant position.
+    private static int ComputeNibbleShift(ulong mapIdx)
+    {
+        // the low bits of the nibble index give us how many nibbles we have to shift by.
+        int nibbleOffsetInMapUnit = (int)(mapIdx & (NibblesPerMapUnit - 1));
+        return 28 - (nibbleOffsetInMapUnit * 4);  // bit shift - 4 bits per nibble
+    }
+
+    private static ulong ComputeByteOffset(ulong mapIdx, uint nibble)
+    {
+        return mapIdx * BytesPerBucket + (nibble - 1) * MapUnitBytes;
+    }
+    private static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, ulong mapIdx, uint nibble)
+    {
+        return baseAddress + ComputeByteOffset(mapIdx, nibble);
+    }
+
+    // Given a relative address, decompose it into
+    //  the bucket index and an offset within the bucket.
+    private static void DecomposeAddress(TargetNUInt relative, out ulong mapIdx, out uint bucketByteIndex)
+    {
+        mapIdx = relative.Value / BytesPerBucket;
+        bucketByteIndex = ((uint)(relative.Value & (BytesPerBucket - 1)) / MapUnitBytes) + 1;
+        System.Diagnostics.Debug.Assert(bucketByteIndex == (bucketByteIndex & NibbleMask));
+    }
+
+    private static TargetPointer GetMapUnitAddress(TargetPointer mapStart, ulong mapIdx)
+    {
+        return mapStart + (mapIdx / NibblesPerMapUnit) * MapUnitBytes;
+    }
+
+    internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPointer currentPC)
+    {
+        TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value);
+        DecomposeAddress(relativeAddress, out ulong mapIdx, out uint bucketByteIndex);
+        return mapBase + ComputeByteOffset(mapIdx, bucketByteIndex);
+    }
+
+    internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapStart, TargetCodePointer currentPC)
+    {
+        TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value);
+        DecomposeAddress(relativeAddress, out ulong mapIdx, out uint bucketByteIndex);
+
+        MapUnit t = _target.Read<MapUnit>(GetMapUnitAddress(mapStart, mapIdx));
+
+        // shift the nibble we want to the least significant position
+        t = t >>> ComputeNibbleShift(mapIdx);
+        uint nibble = t & NibbleMask;
+        if (nibble != 0 && nibble <= bucketByteIndex)
+        {
+            return GetAbsoluteAddress(mapBase, mapIdx, nibble);
+        }
+
+        // search backwards through the current map unit
+        // we processed the lsb nibble, move to the next one
+        t = NextNibble(t);
+
+        // if there's any nibble set in the current unit, find it
+        if (t != 0)
+        {
+            mapIdx--;
+            nibble = t & NibbleMask;
+            while (nibble == 0)
+            {
+                t = NextNibble(t);
+                mapIdx--;
+                nibble = t & NibbleMask;
+            }
+            return GetAbsoluteAddress(mapBase, mapIdx, nibble);
+        }
+
+        // if we were near the beginning of the address space, there is not enough space for the code header,
+        // so we can stop
+        if (mapIdx < NibblesPerMapUnit)
+        {
+            return TargetPointer.Null;
+        }
+
+        // We're now done with the current map index.
+        // Align the map index and move to the previous map unit, then move back one nibble.
+#pragma warning disable IDE0054 // use compound assignment
+        mapIdx = mapIdx & (~(NibblesPerMapUnit - 1));
+        mapIdx--;
+#pragma warning restore IDE0054 // use compound assignment
+
+        // read the map unit containing mapIdx and skip over it if it is all zeros
+        while (mapIdx >= NibblesPerMapUnit)
+        {
+            t = _target.Read<MapUnit>(GetMapUnitAddress(mapStart, mapIdx));
+            if (t != 0)
+                break;
+            mapIdx -= NibblesPerMapUnit;
+        }
+
+        // if we went all the way to the front, we didn't find a code header
+        if (mapIdx < NibblesPerMapUnit)
+        {
+            return TargetPointer.Null;
+        }
+
+        // move to the correct nibble in the map unit
+        while (mapIdx != 0 && (t & NibbleMask) == 0)
+        {
+            t = NextNibble(t);
+            mapIdx--;
+        }
+
+        if (mapIdx == 0 && t == 0)
+        {
+            return TargetPointer.Null;
+        }
+
+        nibble = t & NibbleMask;
+        return GetAbsoluteAddress(mapBase, mapIdx, nibble);
+    }
+
+}
+#pragma warning restore SA1121 // Use built in alias

src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader/ContractDescriptorTarget.cs

@@ -298,6 +298,20 @@ public override TargetPointer ReadPointerFromSpan(ReadOnlySpan<byte> bytes)
         }
     }
 
+    public override TargetCodePointer ReadCodePointer(ulong address)
+    {
+        TypeInfo codePointerTypeInfo = GetTypeInfo(DataType.CodePointer);
+        if (codePointerTypeInfo.Size is sizeof(uint))
+        {
+            return new TargetCodePointer(Read<uint>(address));
+        }
+        else if (codePointerTypeInfo.Size is sizeof(ulong))
+        {
+            return new TargetCodePointer(Read<ulong>(address));
+        }
+        throw new InvalidOperationException($"Failed to read code pointer at 0x{address:x8} because CodePointer size is not 4 or 8");
+    }
+
     public void ReadPointers(ulong address, Span<TargetPointer> buffer)
     {
         // TODO(cdac) - This could do a single read, and then swizzle in place if it is useful for performance

src/native/managed/cdacreader/tests/MockMemorySpace.cs

@@ -188,7 +188,7 @@ private string MakeContractsJson()
             return (json, pointerData);
         }
 
-        private ReadContext CreateContext()
+        private ReadContext CreateContext(out ulong contractDescriptorAddress)
         {
             if (_created)
                 throw new InvalidOperationException("Context already created");
@@ -198,14 +198,17 @@ private ReadContext CreateContext()
 
             HeapFragment descriptor = CreateContractDescriptor(json.Data.Length, pointerDataCount);
 
+            AddHeapFragment(descriptor);
+            AddHeapFragment(json);
+            if (pointerData.Data.Length > 0)
+                AddHeapFragment(pointerData);
+
             ReadContext context = new ReadContext
             {
-                ContractDescriptor = descriptor,
-                JsonDescriptor = json,
-                PointerData = pointerData,
                 HeapFragments = _heapFragments,
             };
             _created = true;
+            contractDescriptorAddress = descriptor.Address;
             return context;
         }
 
@@ -228,53 +231,20 @@ private bool FragmentFits(HeapFragment f)
 
         public bool TryCreateTarget([NotNullWhen(true)] out ContractDescriptorTarget? target)
         {
-            ReadContext context = CreateContext();
-            return ContractDescriptorTarget.TryCreate(context.ContractDescriptor.Address, context.ReadFromTarget, out target);
+            ReadContext context = CreateContext(out ulong contractDescriptorAddress);
+            return ContractDescriptorTarget.TryCreate(contractDescriptorAddress, context.ReadFromTarget, out target);
         }
     }
 
     // Used by ReadFromTarget to return the appropriate bytes
     internal class ReadContext
     {
-        public HeapFragment ContractDescriptor { get; init;}
-
-        public HeapFragment JsonDescriptor { get; init; }
-
-        public HeapFragment PointerData { get; init;}
         public IReadOnlyList<HeapFragment> HeapFragments { get; init; }
 
-        internal int ReadFromTarget(ulong address, Span<byte> span)
+        internal int ReadFromTarget(ulong address, Span<byte> buffer)
         {
             if (address == 0)
                 return -1;
-            // Populate the span with the requested portion of the contract descriptor
-            if (address >= ContractDescriptor.Address && address + (uint)span.Length <= ContractDescriptor.Address + (ulong)ContractDescriptor.Data.Length)
-            {
-                int offset = checked ((int)(address - ContractDescriptor.Address));
-                ContractDescriptor.Data.AsSpan(offset, span.Length).CopyTo(span);
-                return 0;
-            }
-
-            // Populate the span with the JSON descriptor - this assumes the product will read it all at once.
-            if (address == JsonDescriptor.Address)
-            {
-                JsonDescriptor.Data.AsSpan().CopyTo(span);
-                return 0;
-            }
-
-            // Populate the span with the requested portion of the pointer data
-            if (address >= PointerData.Address && address + (uint)span.Length <= PointerData.Address + (ulong)PointerData.Data.Length)
-            {
-                int offset = checked((int)(address - PointerData.Address));
-                PointerData.Data.AsSpan(offset, span.Length).CopyTo(span);
-                return 0;
-            }
-
-            return ReadFragment(address, span);
-        }
-
-        private int ReadFragment(ulong address, Span<byte> buffer)
-        {
             bool partialReadOcurred = false;
             HeapFragment lastHeapFragment = default;
             int availableLength = 0;