Skip to content

Commit

Permalink
Filter segments from secondary exposures in gcir (#261)
Browse files Browse the repository at this point in the history 
![Screenshot 2024-05-15 at 3.27.09 PM.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/ak7zkUsSoHh2VQAhtjtL/51467ef0-404e-4fea-a548-9df5662da38b.png)
  • Loading branch information
@kenny-statsig
kenny-statsig authored May 17, 2024
1 parent 438da54 commit 50e5e8e
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 84 deletions.
2 changes: 2 additions & 0 deletions statsig/client_initialize_formatter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ def clean_exposures(exposures):
seen: Dict[str, bool] = {}
result = []
for exposure in exposures:
if exposure['gate'].startswith('segment:'):
continue
key = f"{exposure['gate']}|{exposure['gateValue']}|{exposure['ruleID']}"
if not seen.get(key, False):
seen[key] = True
Expand Down
135 changes: 51 additions & 84 deletions tests/test_server_sdk_consistency.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,18 @@
import time
import requests

from statsig.utils import HashingAlgorithm

TEST_URLS = [
"https://statsigapi.net/v1",
"https://staging.statsigapi.net/v1"
]


def _construct_statsig_user(user_values) -> StatsigUser:
statsig_user = StatsigUser(user_id=user_values.get("userID"), custom_ids=user_values.get("customIDs"))
statsig_user = StatsigUser(
user_id=user_values.get("userID"),
custom_ids=user_values.get("customIDs"))
statsig_user.app_version = user_values.get("appVersion")
statsig_user.user_agent = user_values.get("userAgent")
statsig_user.ip = user_values.get("ip")
Expand Down Expand Up @@ -57,130 +61,93 @@ def _test_consistency(self):
for entry in self.data:
for val in self.data[entry]:
statsig_user = _construct_statsig_user(val["user"])

self._test_gate_results(statsig_user, val["feature_gates_v2"])
self._test_config_results(statsig_user, val["dynamic_configs"])
self._test_layer_results(statsig_user, val["layer_configs"])
sdk_results = self.sdk._evaluator.get_client_initialize_response(
statsig_user, HashingAlgorithm.NONE)

self._test_gate_results(
statsig_user, val["feature_gates_v2"],
sdk_results["feature_gates"])
self._test_config_results(
statsig_user, val["dynamic_configs"],
sdk_results["dynamic_configs"])
self._test_layer_results(
statsig_user, val["layer_configs"],
sdk_results["layer_configs"])

print("[end]")

def _test_gate_results(self, statsig_user: StatsigUser, gates):
for name in gates:
if name == "inline_targeting_rules_exp":
continue
eval_result = self.sdk._evaluator.check_gate(
statsig_user, name)
sdk_result = self.sdk.check_gate(statsig_user, name)
server_result = gates[name]
if eval_result.boolean_value != server_result["value"]:
def _test_gate_results(self, statsig_user: StatsigUser, server, local):
for name in server:
sdk_result = local[name]
server_result = server[name]
if sdk_result["value"] != server_result["value"]:
print(
f'\nDifferent values for gate {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["value"]}, Actual: {eval_result.boolean_value}')
self.assertEqual(eval_result.boolean_value,
server_result["value"], f"Gate result mismatch [{name}]")
self.assertEqual(eval_result.boolean_value, sdk_result)
f'\nExpected: {server_result["value"]}, Actual: {sdk_result["value"]}')

if eval_result.rule_id != server_result["rule_id"]:
if sdk_result["rule_id"] != server_result["rule_id"]:
print(
f'\nDifferent rule_id for gate {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["rule_id"]}, Actual: {eval_result.rule_id}')
self.assertEqual(eval_result.rule_id,
server_result["rule_id"])
f'\nExpected: {server_result["rule_id"]}, Actual: {sdk_result["rule_id"]}')

if eval_result.secondary_exposures != server_result["secondary_exposures"]:
if sdk_result["secondary_exposures"] != server_result["secondary_exposures"]:
print(
f'\nDifferent secondary_exposures for gate {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["secondary_exposures"]}, Actual: {eval_result.secondary_exposures}')
self.assertEqual(eval_result.secondary_exposures,
server_result.get("secondary_exposures"))
print(".", end="")

def _test_config_results(self, statsig_user: StatsigUser, configs):
for name in configs:
if name == "inline_targeting_rules_exp":
continue
eval_result = self.sdk._evaluator.get_config(
statsig_user, name)
sdk_result = self.sdk.get_config(statsig_user, name)
server_result = configs[name]
if eval_result.json_value != server_result["value"]:
f'\nExpected: {server_result["secondary_exposures"]}, Actual: {sdk_result["secondary_exposures"]}')

def _test_config_results(self, statsig_user: StatsigUser, server, local):
for name in server:
sdk_result = local[name]
server_result = server[name]
if sdk_result["value"] != server_result["value"]:
print(
f'\nDifferent values for config {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["value"]}, Actual: {eval_result.json_value}')
self.assertEqual(eval_result.json_value,
server_result["value"])
self.assertEqual(eval_result.json_value,
sdk_result.get_value())
f'\nExpected: {server_result["value"]}, Actual: {sdk_result["value"]}')

if eval_result.rule_id != server_result["rule_id"]:
if sdk_result["rule_id"] != server_result["rule_id"]:
print(
f'\nDifferent rule_id for config {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["rule_id"]}, Actual: {eval_result.rule_id}')
self.assertEqual(eval_result.rule_id,
server_result["rule_id"])
f'\nExpected: {server_result["rule_id"]}, Actual: {sdk_result["rule_id"]}')

if eval_result.secondary_exposures != server_result["secondary_exposures"]:
if sdk_result["secondary_exposures"] != server_result["secondary_exposures"]:
print(
f'\nDifferent secondary_exposures for config {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["secondary_exposures"]}, Actual: {eval_result.secondary_exposures}')
self.assertEqual(eval_result.secondary_exposures,
server_result.get("secondary_exposures"))
print(".", end="")
f'\nExpected: {server_result["secondary_exposures"]}, Actual: {sdk_result["secondary_exposures"]}')

def _test_layer_results(self, statsig_user: StatsigUser, layers):
for name in layers:
if name == "inline_targeting_rules_exp":
continue
eval_result = self.sdk._evaluator.get_layer(statsig_user, name)
sdk_result = self.sdk.get_layer(statsig_user, name)
server_result = layers[name]
if eval_result.json_value != server_result["value"]:
def _test_layer_results(self, statsig_user: StatsigUser, server, local):
for name in server:
sdk_result = local[name]
server_result = server[name]
if sdk_result["value"] != server_result["value"]:
print(
f'\nDifferent values for layer {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["value"]}, Actual: {eval_result.json_value}')
self.assertEqual(eval_result.json_value,
server_result["value"])

count = 0
for key in eval_result.json_value:
count += 1
self.assertEqual(
eval_result.json_value[key], sdk_result.get(key))

self.assertEqual(len(eval_result.json_value), count)
f'\nExpected: {server_result["value"]}, Actual: {sdk_result["value"]}')

if eval_result.rule_id != server_result["rule_id"]:
if sdk_result["rule_id"] != server_result["rule_id"]:
print(
f'\nDifferent rule_id for layer {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["rule_id"]}, Actual: {eval_result.rule_id}')
self.assertEqual(eval_result.rule_id,
server_result["rule_id"])
f'\nExpected: {server_result["rule_id"]}, Actual: {sdk_result["rule_id"]}')

if eval_result.secondary_exposures != server_result["secondary_exposures"]:
if sdk_result["secondary_exposures"] != server_result["secondary_exposures"]:
print(
f'\nDifferent secondary_exposures for layer {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["secondary_exposures"]}, Actual: {eval_result.secondary_exposures}')
self.assertEqual(eval_result.secondary_exposures,
server_result.get("secondary_exposures"))
f'\nExpected: {server_result["secondary_exposures"]}, Actual: {sdk_result["secondary_exposures"]}')

if eval_result.undelegated_secondary_exposures != server_result[
"undelegated_secondary_exposures"]:
if sdk_result["undelegated_secondary_exposures"] != server_result[
"undelegated_secondary_exposures"]:
print(
f'\nDifferent undelegated_secondary_exposures for layer {name} user: {statsig_user.to_dict(True)}')
print(
f'\nExpected: {server_result["undelegated_secondary_exposures"]}, Actual: {eval_result.undelegated_secondary_exposures}')
self.assertEqual(eval_result.undelegated_secondary_exposures,
server_result.get("undelegated_secondary_exposures"))
print(".", end="")
f'\nExpected: {server_result["undelegated_secondary_exposures"]}, Actual: {sdk_result["undelegated_secondary_exposures"]}')


if __name__ == '__main__':
Expand Down

0 comments on commit 50e5e8e

Please sign in to comment.