Skip to content

Commit

Permalink
Eval Details (#147)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sroyal-statsig
sroyal-statsig authored Jun 17, 2024
1 parent 5244a4e commit 053c3e8
Show file tree
Hide file tree
Showing 8 changed files with 94 additions and 27 deletions.
6 changes: 6 additions & 0 deletions dotnet-statsig-tests/Server/GetFeatureGateTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,16 @@ public async void GetFeatureGate()
Assert.True(gate.Value);
Assert.Equal("7w9rbTSffLT89pxqpyhuqK", gate.RuleID);
Assert.Equal(EvaluationReason.Network, gate.Reason);
Assert.Equal(EvaluationReason.Network, gate.EvaluationDetails?.Reason);
Assert.Equal(1631638014811, gate.EvaluationDetails?.ConfigSyncTime);
Assert.Equal(1631638014811, gate.EvaluationDetails?.InitTime);

var gate2 = StatsigServer.GetFeatureGateWithExposureLoggingDisabled(user, "fake_gate");
Assert.False(gate2.Value);
Assert.Equal(EvaluationReason.Unrecognized, gate2.Reason);
Assert.Equal(EvaluationReason.Unrecognized, gate2.EvaluationDetails?.Reason);
Assert.Equal(1631638014811, gate.EvaluationDetails?.ConfigSyncTime);
Assert.Equal(1631638014811, gate.EvaluationDetails?.InitTime);

await StatsigServer.Shutdown();

Expand Down
7 changes: 6 additions & 1 deletion dotnet-statsig/src/Statsig/DynamicConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using Newtonsoft.Json;
using Statsig.Lib;
using Statsig.Server;
using Statsig.Server.Evaluation;

namespace Statsig
{
Expand All @@ -26,6 +27,8 @@ public class DynamicConfig
[JsonProperty("is_user_in_experiment")]
public bool IsUserInExperiment { get; private set; }

public EvaluationDetails? EvaluationDetails { get; }


static DynamicConfig? _defaultConfig;

Expand All @@ -50,7 +53,8 @@ public DynamicConfig(
List<IReadOnlyDictionary<string, string>>? secondaryExposures = null,
List<string>? explicitParameters = null,
bool isInLayer = false,
bool isUserInExperiment = false
bool isUserInExperiment = false,
EvaluationDetails? details = null
)
{
ConfigName = configName ?? "";
Expand All @@ -61,6 +65,7 @@ public DynamicConfig(
ExplicitParameters = explicitParameters ?? new List<string>();
IsInLayer = isInLayer;
IsUserInExperiment = isUserInExperiment;
EvaluationDetails = details;
}

public T? Get<T>(string key, T? defaultValue = default(T))
Expand Down
4 changes: 3 additions & 1 deletion dotnet-statsig/src/Statsig/FeatureGate.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class FeatureGate
[JsonProperty("secondary_exposures")]
public List<IReadOnlyDictionary<string, string>> SecondaryExposures { get; }
public EvaluationReason? Reason { get; }
public EvaluationDetails? EvaluationDetails { get; }

static FeatureGate? _defaultConfig;

Expand All @@ -31,13 +32,14 @@ public static FeatureGate Default
}
}

public FeatureGate(string? name = null, bool value = false, string? ruleID = null, List<IReadOnlyDictionary<string, string>>? secondaryExposures = null, EvaluationReason? reason = null)
public FeatureGate(string? name = null, bool value = false, string? ruleID = null, List<IReadOnlyDictionary<string, string>>? secondaryExposures = null, EvaluationReason? reason = null, EvaluationDetails? details = null)
{
Name = name ?? "";
Value = value;
RuleID = ruleID ?? "";
SecondaryExposures = secondaryExposures ?? new List<IReadOnlyDictionary<string, string>>();
Reason = reason ?? EvaluationReason.Uninitialized;
EvaluationDetails = details;
}

internal static FeatureGate? FromJObject(string name, JObject? jobj)
Expand Down
7 changes: 6 additions & 1 deletion dotnet-statsig/src/Statsig/Layer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using Newtonsoft.Json;
using Statsig.Lib;
using Statsig.Server;
using Statsig.Server.Evaluation;

namespace Statsig
{
Expand Down Expand Up @@ -31,6 +32,8 @@ public class Layer

static Layer? _default;

public EvaluationDetails? EvaluationDetails { get; }

public static Layer Default
{
get
Expand All @@ -50,7 +53,8 @@ public Layer(string? name = null,
string? allocatedExperimentName = null,
List<string>? explicitParameters = null,
Action<Layer, string>? onExposure = null,
string? groupName = null)
string? groupName = null,
EvaluationDetails? details = null)
{
Name = name ?? "";
Value = value ?? new Dictionary<string, JToken>();
Expand All @@ -61,6 +65,7 @@ public Layer(string? name = null,
ExplicitParameters = explicitParameters ?? new List<string>();
AllocatedExperimentName = allocatedExperimentName ?? "";
GroupName = groupName;
EvaluationDetails = details;
}

public T? Get<T>(string key, T? defaultValue = default(T))
Expand Down
22 changes: 22 additions & 0 deletions dotnet-statsig/src/Statsig/Server/Evaluation/EvaluationDetails.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@

using System;
using System.Collections.Generic;

namespace Statsig.Server.Evaluation
{
public class EvaluationDetails
{
public EvaluationReason Reason { get; }
public long InitTime { get; }
public long ServerTime { get; }
public long ConfigSyncTime { get; }

public EvaluationDetails(EvaluationReason reason, long initTime, long configSyncTime)
{
Reason = reason;
this.InitTime = initTime;
this.ServerTime = DateTime.Now.Millisecond;
this.ConfigSyncTime = configSyncTime;
}
}
}
27 changes: 17 additions & 10 deletions dotnet-statsig/src/Statsig/Server/Evaluation/Evaluator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,13 @@ internal void OverrideLayer(string layerName, Dictionary<string, JToken> value,
if (user.UserID != null && overrides.ContainsKey(user.UserID))
{
return new ConfigEvaluation(EvaluationResult.Pass, EvaluationReason.LocalOverride,
new FeatureGate(gateName, overrides[user.UserID]!, "override", null, EvaluationReason.LocalOverride));
new FeatureGate(gateName, overrides[user.UserID]!, "override", null, EvaluationReason.LocalOverride, _store.GetEvaluationDetails(EvaluationReason.LocalOverride)));
}

if (overrides.ContainsKey(""))
{
return new ConfigEvaluation(EvaluationResult.Pass, EvaluationReason.LocalOverride,
new FeatureGate(gateName, overrides[""]!, "override", null, EvaluationReason.LocalOverride));
new FeatureGate(gateName, overrides[""]!, "override", null, EvaluationReason.LocalOverride, _store.GetEvaluationDetails(EvaluationReason.LocalOverride)));
}
return null;
}
Expand Down Expand Up @@ -142,13 +142,13 @@ internal void OverrideLayer(string layerName, Dictionary<string, JToken> value,
if (user.UserID != null && overrides.ContainsKey(user.UserID))
{
return new ConfigEvaluation(EvaluationResult.Pass, EvaluationReason.LocalOverride, null,
new DynamicConfig(configName, overrides[user.UserID]!, "override"));
new DynamicConfig(configName, overrides[user.UserID]!, "override", null, null, null, false, false, _store.GetEvaluationDetails(EvaluationReason.LocalOverride)));
}

if (overrides.ContainsKey(""))
{
return new ConfigEvaluation(EvaluationResult.Pass, EvaluationReason.LocalOverride, null,
new DynamicConfig(configName, overrides[""]!, "override"));
new DynamicConfig(configName, overrides[""]!, "override", null, null, null, false, false, _store.GetEvaluationDetails(EvaluationReason.LocalOverride)));
}
return null;
}
Expand Down Expand Up @@ -192,6 +192,11 @@ internal List<string> GetSpecNames(string type)
return _store.GetSpecNames(type);
}

internal EvaluationDetails GetEvaluationDetails(EvaluationReason? reason = null)
{
return _store.GetEvaluationDetails(reason);
}

internal Dictionary<string, Object>? GetAllEvaluations(StatsigUser user, string? clientSDKKey, string? hash, bool includeLocalOverrides = false)
{
if (_store.EvalReason == EvaluationReason.Uninitialized || _store.LastSyncTime == 0)
Expand Down Expand Up @@ -460,9 +465,9 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec, int depth)
(
EvaluationResult.Fail,
_store.EvalReason,
new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, "disabled", null, _store.EvalReason),
new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, "disabled", null, _store.EvalReason, _store.GetEvaluationDetails()),
new DynamicConfig(spec.Name, spec.DynamicConfigDefault.Value, "disabled", null, null,
spec.ExplicitParameters)
spec.ExplicitParameters, false, false, _store.GetEvaluationDetails())
);
}

Expand Down Expand Up @@ -490,7 +495,8 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec, int depth)
passPercentage ? rule.FeatureGateValue.Value : spec.FeatureGateDefault.Value,
rule.ID,
CleanExposures(secondaryExposures),
_store.EvalReason
_store.EvalReason,
_store.GetEvaluationDetails()
);
var configV = new DynamicConfig
(
Expand All @@ -501,7 +507,8 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec, int depth)
CleanExposures(secondaryExposures),
spec.ExplicitParameters,
spec.HasSharedParams,
IsUserAllocatedToExperiment(user, spec, rule.ID)
IsUserAllocatedToExperiment(user, spec, rule.ID),
_store.GetEvaluationDetails()
);
return new ConfigEvaluation(passPercentage ? EvaluationResult.Pass : EvaluationResult.Fail,
_store.EvalReason, gateV, configV);
Expand All @@ -515,9 +522,9 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec, int depth)
(
EvaluationResult.Fail,
_store.EvalReason,
new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, "default", CleanExposures(secondaryExposures)),
new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, "default", CleanExposures(secondaryExposures), _store.EvalReason, _store.GetEvaluationDetails()),
new DynamicConfig(spec.Name, spec.DynamicConfigDefault.Value, "default", null, CleanExposures(secondaryExposures),
spec.ExplicitParameters)
spec.ExplicitParameters, false, false, _store.GetEvaluationDetails())
);
}

Expand Down
10 changes: 10 additions & 0 deletions dotnet-statsig/src/Statsig/Server/Evaluation/SpecStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ class SpecStore
internal Dictionary<string, string> SDKKeysToAppIDs { get; private set; }
internal Dictionary<string, string> HashedSDKKeysToAppIDs { get; private set; }
internal readonly ConcurrentDictionary<string, IDList> _idLists;

internal long InitialUpdateTime { get; private set; }
private double _idListsSyncInterval;
private double _rulesetsSyncInterval;
private Func<IIDStore>? _idStoreFactory;
Expand All @@ -47,6 +49,7 @@ internal SpecStore(StatsigOptions options, RequestDispatcher dispatcher, string
_idListsSyncInterval = options.IDListsSyncInterval;
_rulesetsSyncInterval = options.RulesetsSyncInterval;
LastSyncTime = 0;
InitialUpdateTime = 0;
FeatureGates = new Dictionary<string, ConfigSpec>();
DynamicConfigs = new Dictionary<string, ConfigSpec>();
LayerConfigs = new Dictionary<string, ConfigSpec>();
Expand Down Expand Up @@ -84,6 +87,8 @@ internal async Task<InitializeResult> Initialize()
EvalReason = EvaluationReason.DataAdapter;
}

InitialUpdateTime = LastSyncTime;

await SyncIDLists().ConfigureAwait(false);

// Start background tasks to periodically refresh the store
Expand Down Expand Up @@ -136,6 +141,11 @@ internal List<string> GetSpecNames(string type)
};
}

internal EvaluationDetails GetEvaluationDetails(EvaluationReason? reason = null)
{
return new EvaluationDetails(reason ?? EvalReason, InitialUpdateTime, LastSyncTime);
}

private async Task BackgroundPeriodicSyncIDListsTask(CancellationToken cancellationToken)
{
var delayInterval = TimeSpan.FromSeconds(_idListsSyncInterval);
Expand Down
38 changes: 24 additions & 14 deletions dotnet-statsig/src/Statsig/Server/ServerDriver.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ public FeatureGate GetFeatureGate(StatsigUser user, string gateName)
return _errorBoundary.Capture(
"GetFeatureGate",
() => CheckGateImpl(user, gateName, shouldLogExposure: true),
() => new FeatureGate(gateName, false, null, null, EvaluationReason.Error)
() => new FeatureGate(gateName)
);
}

Expand All @@ -144,7 +144,7 @@ public FeatureGate GetFeatureGateWithExposureLoggingDisabled(StatsigUser user, s
return _errorBoundary.Capture(
"GetFeatureGateWithExposureLoggingDisabled",
() => CheckGateImpl(user, gateName, shouldLogExposure: false),
() => new FeatureGate(gateName, false, null, null, EvaluationReason.Error)
() => new FeatureGate(gateName)
);
}

Expand Down Expand Up @@ -453,30 +453,30 @@ private FeatureGate CheckGateImpl(StatsigUser user, string gateName, bool should
var isInitialized = EnsureInitialized();
if (!isInitialized)
{
return new FeatureGate(gateName, false, null, null, EvaluationReason.Uninitialized);
return new FeatureGate(gateName, false, null, null, EvaluationReason.Uninitialized, evaluator.GetEvaluationDetails(EvaluationReason.Uninitialized));
}
var userIsValid = ValidateUser(user);
if (!userIsValid)
{
return new FeatureGate(gateName, false, null, null, EvaluationReason.Error);
return new FeatureGate(gateName, false, null, null, EvaluationReason.Error, evaluator.GetEvaluationDetails(EvaluationReason.Error));
}
NormalizeUser(user);
var nameValid = ValidateNonEmptyArgument(gateName, "gateName");
if (!nameValid)
{
return new FeatureGate(gateName, false, null, null, EvaluationReason.Error);
return new FeatureGate(gateName, false, null, null, EvaluationReason.Error, evaluator.GetEvaluationDetails(EvaluationReason.Error));
}

var evaluation = evaluator.CheckGate(user, gateName);

if (evaluation.Result == EvaluationResult.Unsupported)
{
return new FeatureGate(gateName, false, null, null, EvaluationReason.Unsupported);
return new FeatureGate(gateName, false, null, null, EvaluationReason.Unsupported, evaluator.GetEvaluationDetails(EvaluationReason.Unsupported));
}

if (evaluation.Reason == EvaluationReason.Unrecognized)
{
var gateValue = new FeatureGate(gateName, false, null, null, EvaluationReason.Unrecognized);
var gateValue = new FeatureGate(gateName, false, null, null, EvaluationReason.Unrecognized, evaluator.GetEvaluationDetails(EvaluationReason.Unrecognized));
if (shouldLogExposure)
{
LogGateExposureImpl(user, gateName, gateValue, ExposureCause.Automatic, evaluation.Reason);
Expand Down Expand Up @@ -509,21 +509,31 @@ private DynamicConfig GetConfigImpl(StatsigUser user, string configName, bool sh
var userIsValid = ValidateUser(user);
if (!userIsValid)
{
return new DynamicConfig(configName);
return new DynamicConfig(configName, null, null, null, null, null, false, false, evaluator.GetEvaluationDetails(EvaluationReason.Error));
}
NormalizeUser(user);
var nameValid = ValidateNonEmptyArgument(configName, "configName");
if (!nameValid)
{
return new DynamicConfig(configName);
return new DynamicConfig(configName, null, null, null, null, null, false, false, evaluator.GetEvaluationDetails(EvaluationReason.Error));
}


var evaluation = evaluator.GetConfig(user, configName);

if (evaluation.Result == EvaluationResult.Unsupported)
{
return new DynamicConfig(configName);
return new DynamicConfig(configName, null, null, null, null, null, false, false, evaluator.GetEvaluationDetails(EvaluationReason.Unsupported));
}

if (evaluation.Reason == EvaluationReason.Unrecognized)
{
var configValue = new DynamicConfig(configName, null, null, null, null, null, false, false, evaluator.GetEvaluationDetails(EvaluationReason.Unrecognized));
if (shouldLogExposure)
{
LogConfigExposureImpl(user, configName, configValue, ExposureCause.Automatic, evaluation.Reason);
}
return configValue;
}

if (shouldLogExposure)
Expand Down Expand Up @@ -551,20 +561,20 @@ private Layer GetLayerImpl(StatsigUser user, string layerName, bool shouldLogExp
var userIsValid = ValidateUser(user);
if (!userIsValid)
{
return new Layer(layerName);
return new Layer(layerName, null, null, null, null, null, null, evaluator.GetEvaluationDetails(EvaluationReason.Error));
}
NormalizeUser(user);
var nameIsValid = ValidateNonEmptyArgument(layerName, "layerName");
if (!nameIsValid)
{
return new Layer(layerName);
return new Layer(layerName, null, null, null, null, null, null, evaluator.GetEvaluationDetails(EvaluationReason.Error));
}

var evaluation = evaluator.GetLayer(user, layerName);

if (evaluation.Result == EvaluationResult.Unsupported)
{
return new Layer(layerName);
return new Layer(layerName, null, null, null, null, null, null, evaluator.GetEvaluationDetails(EvaluationReason.Unsupported));
}

void OnExposure(Layer layer, string parameterName)
Expand All @@ -578,7 +588,7 @@ void OnExposure(Layer layer, string parameterName)
}

var dc = evaluation.ConfigValue;
return new Layer(layerName, dc.Value, dc.RuleID, evaluation.ConfigDelegate, evaluation.ExplicitParameters, OnExposure, dc.GroupName);
return new Layer(layerName, dc.Value, dc.RuleID, evaluation.ConfigDelegate, evaluation.ExplicitParameters, OnExposure, dc.GroupName, evaluator.GetEvaluationDetails(evaluation.Reason));
}

private void LogLayerParameterExposureImpl(
Expand Down

0 comments on commit 053c3e8

Please sign in to comment.