From 053c3e88977b62b6135bdfa70c2314c1de7f3f47 Mon Sep 17 00:00:00 2001 From: sroyal-statsig <76536058+sroyal-statsig@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:44:54 -0700 Subject: [PATCH] Eval Details (#147) --- .../Server/GetFeatureGateTest.cs | 6 +++ dotnet-statsig/src/Statsig/DynamicConfig.cs | 7 +++- dotnet-statsig/src/Statsig/FeatureGate.cs | 4 +- dotnet-statsig/src/Statsig/Layer.cs | 7 +++- .../Server/Evaluation/EvaluationDetails.cs | 22 +++++++++++ .../Statsig/Server/Evaluation/Evaluator.cs | 27 ++++++++----- .../Statsig/Server/Evaluation/SpecStore.cs | 10 +++++ .../src/Statsig/Server/ServerDriver.cs | 38 ++++++++++++------- 8 files changed, 94 insertions(+), 27 deletions(-) create mode 100644 dotnet-statsig/src/Statsig/Server/Evaluation/EvaluationDetails.cs diff --git a/dotnet-statsig-tests/Server/GetFeatureGateTest.cs b/dotnet-statsig-tests/Server/GetFeatureGateTest.cs index 1eb9424..63e5deb 100644 --- a/dotnet-statsig-tests/Server/GetFeatureGateTest.cs +++ b/dotnet-statsig-tests/Server/GetFeatureGateTest.cs @@ -86,10 +86,16 @@ public async void GetFeatureGate() Assert.True(gate.Value); Assert.Equal("7w9rbTSffLT89pxqpyhuqK", gate.RuleID); Assert.Equal(EvaluationReason.Network, gate.Reason); + Assert.Equal(EvaluationReason.Network, gate.EvaluationDetails?.Reason); + Assert.Equal(1631638014811, gate.EvaluationDetails?.ConfigSyncTime); + Assert.Equal(1631638014811, gate.EvaluationDetails?.InitTime); var gate2 = StatsigServer.GetFeatureGateWithExposureLoggingDisabled(user, "fake_gate"); Assert.False(gate2.Value); Assert.Equal(EvaluationReason.Unrecognized, gate2.Reason); + Assert.Equal(EvaluationReason.Unrecognized, gate2.EvaluationDetails?.Reason); + Assert.Equal(1631638014811, gate.EvaluationDetails?.ConfigSyncTime); + Assert.Equal(1631638014811, gate.EvaluationDetails?.InitTime); await StatsigServer.Shutdown(); diff --git a/dotnet-statsig/src/Statsig/DynamicConfig.cs b/dotnet-statsig/src/Statsig/DynamicConfig.cs index a19d198..26c386c 100644 --- a/dotnet-statsig/src/Statsig/DynamicConfig.cs +++ b/dotnet-statsig/src/Statsig/DynamicConfig.cs @@ -4,6 +4,7 @@ using Newtonsoft.Json; using Statsig.Lib; using Statsig.Server; +using Statsig.Server.Evaluation; namespace Statsig { @@ -26,6 +27,8 @@ public class DynamicConfig [JsonProperty("is_user_in_experiment")] public bool IsUserInExperiment { get; private set; } + public EvaluationDetails? EvaluationDetails { get; } + static DynamicConfig? _defaultConfig; @@ -50,7 +53,8 @@ public DynamicConfig( List>? secondaryExposures = null, List? explicitParameters = null, bool isInLayer = false, - bool isUserInExperiment = false + bool isUserInExperiment = false, + EvaluationDetails? details = null ) { ConfigName = configName ?? ""; @@ -61,6 +65,7 @@ public DynamicConfig( ExplicitParameters = explicitParameters ?? new List(); IsInLayer = isInLayer; IsUserInExperiment = isUserInExperiment; + EvaluationDetails = details; } public T? Get(string key, T? defaultValue = default(T)) diff --git a/dotnet-statsig/src/Statsig/FeatureGate.cs b/dotnet-statsig/src/Statsig/FeatureGate.cs index 3cd1ae5..9eb2ecb 100644 --- a/dotnet-statsig/src/Statsig/FeatureGate.cs +++ b/dotnet-statsig/src/Statsig/FeatureGate.cs @@ -16,6 +16,7 @@ public class FeatureGate [JsonProperty("secondary_exposures")] public List> SecondaryExposures { get; } public EvaluationReason? Reason { get; } + public EvaluationDetails? EvaluationDetails { get; } static FeatureGate? _defaultConfig; @@ -31,13 +32,14 @@ public static FeatureGate Default } } - public FeatureGate(string? name = null, bool value = false, string? ruleID = null, List>? secondaryExposures = null, EvaluationReason? reason = null) + public FeatureGate(string? name = null, bool value = false, string? ruleID = null, List>? secondaryExposures = null, EvaluationReason? reason = null, EvaluationDetails? details = null) { Name = name ?? ""; Value = value; RuleID = ruleID ?? ""; SecondaryExposures = secondaryExposures ?? new List>(); Reason = reason ?? EvaluationReason.Uninitialized; + EvaluationDetails = details; } internal static FeatureGate? FromJObject(string name, JObject? jobj) diff --git a/dotnet-statsig/src/Statsig/Layer.cs b/dotnet-statsig/src/Statsig/Layer.cs index 21d4da9..44353ab 100644 --- a/dotnet-statsig/src/Statsig/Layer.cs +++ b/dotnet-statsig/src/Statsig/Layer.cs @@ -4,6 +4,7 @@ using Newtonsoft.Json; using Statsig.Lib; using Statsig.Server; +using Statsig.Server.Evaluation; namespace Statsig { @@ -31,6 +32,8 @@ public class Layer static Layer? _default; + public EvaluationDetails? EvaluationDetails { get; } + public static Layer Default { get @@ -50,7 +53,8 @@ public Layer(string? name = null, string? allocatedExperimentName = null, List? explicitParameters = null, Action? onExposure = null, - string? groupName = null) + string? groupName = null, + EvaluationDetails? details = null) { Name = name ?? ""; Value = value ?? new Dictionary(); @@ -61,6 +65,7 @@ public Layer(string? name = null, ExplicitParameters = explicitParameters ?? new List(); AllocatedExperimentName = allocatedExperimentName ?? ""; GroupName = groupName; + EvaluationDetails = details; } public T? Get(string key, T? defaultValue = default(T)) diff --git a/dotnet-statsig/src/Statsig/Server/Evaluation/EvaluationDetails.cs b/dotnet-statsig/src/Statsig/Server/Evaluation/EvaluationDetails.cs new file mode 100644 index 0000000..6868f4d --- /dev/null +++ b/dotnet-statsig/src/Statsig/Server/Evaluation/EvaluationDetails.cs @@ -0,0 +1,22 @@ + +using System; +using System.Collections.Generic; + +namespace Statsig.Server.Evaluation +{ + public class EvaluationDetails + { + public EvaluationReason Reason { get; } + public long InitTime { get; } + public long ServerTime { get; } + public long ConfigSyncTime { get; } + + public EvaluationDetails(EvaluationReason reason, long initTime, long configSyncTime) + { + Reason = reason; + this.InitTime = initTime; + this.ServerTime = DateTime.Now.Millisecond; + this.ConfigSyncTime = configSyncTime; + } + } +} \ No newline at end of file diff --git a/dotnet-statsig/src/Statsig/Server/Evaluation/Evaluator.cs b/dotnet-statsig/src/Statsig/Server/Evaluation/Evaluator.cs index 55b0ccd..e4dc72d 100644 --- a/dotnet-statsig/src/Statsig/Server/Evaluation/Evaluator.cs +++ b/dotnet-statsig/src/Statsig/Server/Evaluation/Evaluator.cs @@ -97,13 +97,13 @@ internal void OverrideLayer(string layerName, Dictionary value, if (user.UserID != null && overrides.ContainsKey(user.UserID)) { return new ConfigEvaluation(EvaluationResult.Pass, EvaluationReason.LocalOverride, - new FeatureGate(gateName, overrides[user.UserID]!, "override", null, EvaluationReason.LocalOverride)); + new FeatureGate(gateName, overrides[user.UserID]!, "override", null, EvaluationReason.LocalOverride, _store.GetEvaluationDetails(EvaluationReason.LocalOverride))); } if (overrides.ContainsKey("")) { return new ConfigEvaluation(EvaluationResult.Pass, EvaluationReason.LocalOverride, - new FeatureGate(gateName, overrides[""]!, "override", null, EvaluationReason.LocalOverride)); + new FeatureGate(gateName, overrides[""]!, "override", null, EvaluationReason.LocalOverride, _store.GetEvaluationDetails(EvaluationReason.LocalOverride))); } return null; } @@ -142,13 +142,13 @@ internal void OverrideLayer(string layerName, Dictionary value, if (user.UserID != null && overrides.ContainsKey(user.UserID)) { return new ConfigEvaluation(EvaluationResult.Pass, EvaluationReason.LocalOverride, null, - new DynamicConfig(configName, overrides[user.UserID]!, "override")); + new DynamicConfig(configName, overrides[user.UserID]!, "override", null, null, null, false, false, _store.GetEvaluationDetails(EvaluationReason.LocalOverride))); } if (overrides.ContainsKey("")) { return new ConfigEvaluation(EvaluationResult.Pass, EvaluationReason.LocalOverride, null, - new DynamicConfig(configName, overrides[""]!, "override")); + new DynamicConfig(configName, overrides[""]!, "override", null, null, null, false, false, _store.GetEvaluationDetails(EvaluationReason.LocalOverride))); } return null; } @@ -192,6 +192,11 @@ internal List GetSpecNames(string type) return _store.GetSpecNames(type); } + internal EvaluationDetails GetEvaluationDetails(EvaluationReason? reason = null) + { + return _store.GetEvaluationDetails(reason); + } + internal Dictionary? GetAllEvaluations(StatsigUser user, string? clientSDKKey, string? hash, bool includeLocalOverrides = false) { if (_store.EvalReason == EvaluationReason.Uninitialized || _store.LastSyncTime == 0) @@ -460,9 +465,9 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec, int depth) ( EvaluationResult.Fail, _store.EvalReason, - new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, "disabled", null, _store.EvalReason), + new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, "disabled", null, _store.EvalReason, _store.GetEvaluationDetails()), new DynamicConfig(spec.Name, spec.DynamicConfigDefault.Value, "disabled", null, null, - spec.ExplicitParameters) + spec.ExplicitParameters, false, false, _store.GetEvaluationDetails()) ); } @@ -490,7 +495,8 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec, int depth) passPercentage ? rule.FeatureGateValue.Value : spec.FeatureGateDefault.Value, rule.ID, CleanExposures(secondaryExposures), - _store.EvalReason + _store.EvalReason, + _store.GetEvaluationDetails() ); var configV = new DynamicConfig ( @@ -501,7 +507,8 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec, int depth) CleanExposures(secondaryExposures), spec.ExplicitParameters, spec.HasSharedParams, - IsUserAllocatedToExperiment(user, spec, rule.ID) + IsUserAllocatedToExperiment(user, spec, rule.ID), + _store.GetEvaluationDetails() ); return new ConfigEvaluation(passPercentage ? EvaluationResult.Pass : EvaluationResult.Fail, _store.EvalReason, gateV, configV); @@ -515,9 +522,9 @@ private ConfigEvaluation Evaluate(StatsigUser user, ConfigSpec spec, int depth) ( EvaluationResult.Fail, _store.EvalReason, - new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, "default", CleanExposures(secondaryExposures)), + new FeatureGate(spec.Name, spec.FeatureGateDefault.Value, "default", CleanExposures(secondaryExposures), _store.EvalReason, _store.GetEvaluationDetails()), new DynamicConfig(spec.Name, spec.DynamicConfigDefault.Value, "default", null, CleanExposures(secondaryExposures), - spec.ExplicitParameters) + spec.ExplicitParameters, false, false, _store.GetEvaluationDetails()) ); } diff --git a/dotnet-statsig/src/Statsig/Server/Evaluation/SpecStore.cs b/dotnet-statsig/src/Statsig/Server/Evaluation/SpecStore.cs index 6f3123b..b0b37d4 100644 --- a/dotnet-statsig/src/Statsig/Server/Evaluation/SpecStore.cs +++ b/dotnet-statsig/src/Statsig/Server/Evaluation/SpecStore.cs @@ -31,6 +31,8 @@ class SpecStore internal Dictionary SDKKeysToAppIDs { get; private set; } internal Dictionary HashedSDKKeysToAppIDs { get; private set; } internal readonly ConcurrentDictionary _idLists; + + internal long InitialUpdateTime { get; private set; } private double _idListsSyncInterval; private double _rulesetsSyncInterval; private Func? _idStoreFactory; @@ -47,6 +49,7 @@ internal SpecStore(StatsigOptions options, RequestDispatcher dispatcher, string _idListsSyncInterval = options.IDListsSyncInterval; _rulesetsSyncInterval = options.RulesetsSyncInterval; LastSyncTime = 0; + InitialUpdateTime = 0; FeatureGates = new Dictionary(); DynamicConfigs = new Dictionary(); LayerConfigs = new Dictionary(); @@ -84,6 +87,8 @@ internal async Task Initialize() EvalReason = EvaluationReason.DataAdapter; } + InitialUpdateTime = LastSyncTime; + await SyncIDLists().ConfigureAwait(false); // Start background tasks to periodically refresh the store @@ -136,6 +141,11 @@ internal List GetSpecNames(string type) }; } + internal EvaluationDetails GetEvaluationDetails(EvaluationReason? reason = null) + { + return new EvaluationDetails(reason ?? EvalReason, InitialUpdateTime, LastSyncTime); + } + private async Task BackgroundPeriodicSyncIDListsTask(CancellationToken cancellationToken) { var delayInterval = TimeSpan.FromSeconds(_idListsSyncInterval); diff --git a/dotnet-statsig/src/Statsig/Server/ServerDriver.cs b/dotnet-statsig/src/Statsig/Server/ServerDriver.cs index c85c9c8..d5a561e 100644 --- a/dotnet-statsig/src/Statsig/Server/ServerDriver.cs +++ b/dotnet-statsig/src/Statsig/Server/ServerDriver.cs @@ -135,7 +135,7 @@ public FeatureGate GetFeatureGate(StatsigUser user, string gateName) return _errorBoundary.Capture( "GetFeatureGate", () => CheckGateImpl(user, gateName, shouldLogExposure: true), - () => new FeatureGate(gateName, false, null, null, EvaluationReason.Error) + () => new FeatureGate(gateName) ); } @@ -144,7 +144,7 @@ public FeatureGate GetFeatureGateWithExposureLoggingDisabled(StatsigUser user, s return _errorBoundary.Capture( "GetFeatureGateWithExposureLoggingDisabled", () => CheckGateImpl(user, gateName, shouldLogExposure: false), - () => new FeatureGate(gateName, false, null, null, EvaluationReason.Error) + () => new FeatureGate(gateName) ); } @@ -453,30 +453,30 @@ private FeatureGate CheckGateImpl(StatsigUser user, string gateName, bool should var isInitialized = EnsureInitialized(); if (!isInitialized) { - return new FeatureGate(gateName, false, null, null, EvaluationReason.Uninitialized); + return new FeatureGate(gateName, false, null, null, EvaluationReason.Uninitialized, evaluator.GetEvaluationDetails(EvaluationReason.Uninitialized)); } var userIsValid = ValidateUser(user); if (!userIsValid) { - return new FeatureGate(gateName, false, null, null, EvaluationReason.Error); + return new FeatureGate(gateName, false, null, null, EvaluationReason.Error, evaluator.GetEvaluationDetails(EvaluationReason.Error)); } NormalizeUser(user); var nameValid = ValidateNonEmptyArgument(gateName, "gateName"); if (!nameValid) { - return new FeatureGate(gateName, false, null, null, EvaluationReason.Error); + return new FeatureGate(gateName, false, null, null, EvaluationReason.Error, evaluator.GetEvaluationDetails(EvaluationReason.Error)); } var evaluation = evaluator.CheckGate(user, gateName); if (evaluation.Result == EvaluationResult.Unsupported) { - return new FeatureGate(gateName, false, null, null, EvaluationReason.Unsupported); + return new FeatureGate(gateName, false, null, null, EvaluationReason.Unsupported, evaluator.GetEvaluationDetails(EvaluationReason.Unsupported)); } if (evaluation.Reason == EvaluationReason.Unrecognized) { - var gateValue = new FeatureGate(gateName, false, null, null, EvaluationReason.Unrecognized); + var gateValue = new FeatureGate(gateName, false, null, null, EvaluationReason.Unrecognized, evaluator.GetEvaluationDetails(EvaluationReason.Unrecognized)); if (shouldLogExposure) { LogGateExposureImpl(user, gateName, gateValue, ExposureCause.Automatic, evaluation.Reason); @@ -509,13 +509,13 @@ private DynamicConfig GetConfigImpl(StatsigUser user, string configName, bool sh var userIsValid = ValidateUser(user); if (!userIsValid) { - return new DynamicConfig(configName); + return new DynamicConfig(configName, null, null, null, null, null, false, false, evaluator.GetEvaluationDetails(EvaluationReason.Error)); } NormalizeUser(user); var nameValid = ValidateNonEmptyArgument(configName, "configName"); if (!nameValid) { - return new DynamicConfig(configName); + return new DynamicConfig(configName, null, null, null, null, null, false, false, evaluator.GetEvaluationDetails(EvaluationReason.Error)); } @@ -523,7 +523,17 @@ private DynamicConfig GetConfigImpl(StatsigUser user, string configName, bool sh if (evaluation.Result == EvaluationResult.Unsupported) { - return new DynamicConfig(configName); + return new DynamicConfig(configName, null, null, null, null, null, false, false, evaluator.GetEvaluationDetails(EvaluationReason.Unsupported)); + } + + if (evaluation.Reason == EvaluationReason.Unrecognized) + { + var configValue = new DynamicConfig(configName, null, null, null, null, null, false, false, evaluator.GetEvaluationDetails(EvaluationReason.Unrecognized)); + if (shouldLogExposure) + { + LogConfigExposureImpl(user, configName, configValue, ExposureCause.Automatic, evaluation.Reason); + } + return configValue; } if (shouldLogExposure) @@ -551,20 +561,20 @@ private Layer GetLayerImpl(StatsigUser user, string layerName, bool shouldLogExp var userIsValid = ValidateUser(user); if (!userIsValid) { - return new Layer(layerName); + return new Layer(layerName, null, null, null, null, null, null, evaluator.GetEvaluationDetails(EvaluationReason.Error)); } NormalizeUser(user); var nameIsValid = ValidateNonEmptyArgument(layerName, "layerName"); if (!nameIsValid) { - return new Layer(layerName); + return new Layer(layerName, null, null, null, null, null, null, evaluator.GetEvaluationDetails(EvaluationReason.Error)); } var evaluation = evaluator.GetLayer(user, layerName); if (evaluation.Result == EvaluationResult.Unsupported) { - return new Layer(layerName); + return new Layer(layerName, null, null, null, null, null, null, evaluator.GetEvaluationDetails(EvaluationReason.Unsupported)); } void OnExposure(Layer layer, string parameterName) @@ -578,7 +588,7 @@ void OnExposure(Layer layer, string parameterName) } var dc = evaluation.ConfigValue; - return new Layer(layerName, dc.Value, dc.RuleID, evaluation.ConfigDelegate, evaluation.ExplicitParameters, OnExposure, dc.GroupName); + return new Layer(layerName, dc.Value, dc.RuleID, evaluation.ConfigDelegate, evaluation.ExplicitParameters, OnExposure, dc.GroupName, evaluator.GetEvaluationDetails(evaluation.Reason)); } private void LogLayerParameterExposureImpl(