From 08573dd6d5e672915a6f342516e69965818da17a Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Wed, 11 Dec 2024 16:11:48 +0100 Subject: [PATCH 01/21] Picking up ADM model type and using in summaries --- python/pdstools/adm/ADMDatamart.py | 10 ++++- python/pdstools/adm/Aggregates.py | 11 ++++- python/pdstools/reports/HealthCheck.qmd | 5 ++- python/pdstools/utils/cdh_utils.py | 1 + python/pdstools/utils/report_utils.py | 59 ++++++++++--------------- python/tests/test_Aggregates.py | 4 +- python/tests/test_datasets.py | 2 +- python/tests/test_end_to_end.py | 6 +-- 8 files changed, 52 insertions(+), 46 deletions(-) diff --git a/python/pdstools/adm/ADMDatamart.py b/python/pdstools/adm/ADMDatamart.py index 0005ef6e..abcaebb9 100644 --- a/python/pdstools/adm/ADMDatamart.py +++ b/python/pdstools/adm/ADMDatamart.py @@ -120,7 +120,9 @@ def __init__( self.aggregates = Aggregates(datamart=self) self.agb = AGB(datamart=self) self.generate = Reports(datamart=self) - self.cdh_guidelines = CDHGuidelines() + self.cdh_guidelines = ( + CDHGuidelines() + ) # not sure if this should be part of the ADM DM self.model_data = self._validate_model_data( model_df, query=query, extract_pyname_keys=extract_pyname_keys @@ -310,6 +312,11 @@ def _validate_model_data( if "Treatment" in schema.names(): self.context_keys.append("Treatment") + # Model technique (NaiveBayes or GradientBoost) added in '24 (US-648869 and related) + if "ModelTechnique" not in schema.names(): + df = df.with_columns( + ModelTechnique=pl.lit(None), + ) self.context_keys = [k for k in self.context_keys if k in schema.names()] df = df.with_columns( @@ -399,7 +406,6 @@ def apply_predictor_categorization( categorization() if callable(categorization) else categorization ) - if df is not None: return df.with_columns(PredictorCategory=categorization_expr) diff --git a/python/pdstools/adm/Aggregates.py b/python/pdstools/adm/Aggregates.py index c5f0dc15..67a11339 100644 --- a/python/pdstools/adm/Aggregates.py +++ b/python/pdstools/adm/Aggregates.py @@ -460,8 +460,7 @@ def name_normalizer(x): .agg( pl.col("SnapshotTime").min().cast(pl.Date).alias("DateRange Min"), pl.col("SnapshotTime").max().cast(pl.Date).alias("DateRange Max"), - pl.col("Positives").sum(), - pl.col("ResponseCount").sum(), + pl.sum(["Positives", "ResponseCount"]), (cdh_utils.weighted_performance_polars() * 100).alias("Performance"), pl.col("Configuration").cast(pl.Utf8), pl.col("Configuration") @@ -469,6 +468,12 @@ def name_normalizer(x): .str.to_uppercase() .is_in([x.upper() for x in self.cdh_guidelines.standard_configurations]) .alias("isNBADModelConfiguration"), + (pl.col("ModelTechnique") == "GradientBoost") + .any(ignore_nulls=False) + .alias("usesAGB"), + (pl.col("ModelTechnique") == "GradientBoost") + .all(ignore_nulls=False) + .alias("usesAGBOnly"), actionIdentifierExpr.drop_nulls() .n_unique() .alias("Total Number of Actions"), @@ -730,6 +735,8 @@ def overall_summary( # TODO there was something about OmniAdaptiveModel here - but I don't recall what was the issue pl.col("usesNBAD").any(), pl.col("usesNBADOnly").all(), + pl.col("usesAGB").any(), + pl.col("usesAGBOnly").all(), # pl.lit(usesNBAD).alias("usesNBAD"), # ((pl.len() > 0) & pl.lit(usesNBAD and usesNBADOnly)).alias( # "usesNBADOnly" diff --git a/python/pdstools/reports/HealthCheck.qmd b/python/pdstools/reports/HealthCheck.qmd index 5c29400b..4b89288d 100644 --- a/python/pdstools/reports/HealthCheck.qmd +++ b/python/pdstools/reports/HealthCheck.qmd @@ -1470,8 +1470,11 @@ if datamart.predictor_data is not None: } try: + # TODO see about this warning that we're getting + # The default of observed=False is deprecated... + fig = px.treemap( - missing.to_pandas(), + missing, path=path, color="Percentage without responses", template="pega", diff --git a/python/pdstools/utils/cdh_utils.py b/python/pdstools/utils/cdh_utils.py index a2277026..86730a84 100644 --- a/python/pdstools/utils/cdh_utils.py +++ b/python/pdstools/utils/cdh_utils.py @@ -562,6 +562,7 @@ def _capitalize(fields: Union[str, Iterable[str]]) -> List[str]: "Offline", "Update", "Strategy", + "ModelTechnique" ] if not isinstance(fields, list): fields = [fields] diff --git a/python/pdstools/utils/report_utils.py b/python/pdstools/utils/report_utils.py index 5a880d4d..835848d1 100644 --- a/python/pdstools/utils/report_utils.py +++ b/python/pdstools/utils/report_utils.py @@ -124,6 +124,15 @@ def table_standard_formatting( highlight_configurations: List[str] = [], rag_styler: callable = rag_background_styler, ): + def apply_style(gt, rag, rows): + style = rag_styler(rag) + if style is not None: + gt = gt.tab_style( + style=style, + locations=loc.body(columns=col_name, rows=rows), + ) + return gt + def apply_rag_styling(gt, col_name, metric): if col_name in source_table.collect_schema().names(): min_val = cdh_guidelines.min(metric) @@ -169,42 +178,11 @@ def apply_rag_styling(gt, col_name, metric): ] # TODO consider that bad / warning rows are exclusive - def apply_style(gt, rag, rows): - style = rag_styler(rag) - if style is not None: - gt = gt.tab_style( - style=style, - locations=loc.body(columns=col_name, rows=rows), - ) - return gt - gt = apply_style(gt, "green", good_rows) gt = apply_style(gt, "amber", warning_rows) gt = apply_style(gt, "red", bad_rows) return gt - def apply_standard_name_style(gt, col_name, standard_list): - if col_name in source_table.collect_schema().names(): - values = source_table[col_name].to_list() - non_standard_rows = [ - i for i, v in enumerate(values) if v not in standard_list - ] - gt = gt.tab_style( - style=rag_styler("yellow"), - locations=loc.body(columns=col_name, rows=non_standard_rows), - ) - return gt - - def apply_configuration_style(gt, col_name): - if col_name in source_table.collect_schema().names(): - values = source_table[col_name].to_list() - multiple_config_rows = [i for i, v in enumerate(values) if v.count(",") > 1] - gt = gt.tab_style( - style=rag_styler("yellow"), - locations=loc.body(columns=col_name, rows=multiple_config_rows), - ) - return gt - gt = ( GT(source_table, rowname_col=rowname_col, groupname_col=groupname_col) .tab_options(table_font_size=8) @@ -260,11 +238,22 @@ def metric_styling_default(gt, cols): case _: gt = metric_styling_default(gt, cols) - for metric in highlight_lists.keys(): - gt = apply_standard_name_style(gt, metric, highlight_lists[metric]) + # Highlight columns with non-standard values + # TODO consider stripping spaces and discarding case etc so email matches E-Mail + for col_name in highlight_lists.keys(): + if col_name in source_table.collect_schema().names(): + values = source_table[col_name].to_list() + non_standard_rows = [ + i for i, v in enumerate(values) if v not in highlight_lists[col_name] + ] + gt = apply_style(gt, "yellow", non_standard_rows) - for metric in highlight_configurations: - gt = apply_configuration_style(gt, metric) + # Highlight column with more than one element (assuming its a comma-separated string) + for col_name in highlight_configurations: + if col_name in source_table.collect_schema().names(): + values = source_table[col_name].to_list() + multiple_config_rows = [i for i, v in enumerate(values) if v.count(",") > 1] + gt = apply_style(gt, "yellow", multiple_config_rows) return gt diff --git a/python/tests/test_Aggregates.py b/python/tests/test_Aggregates.py index 40c9a484..8fc53561 100644 --- a/python/tests/test_Aggregates.py +++ b/python/tests/test_Aggregates.py @@ -43,13 +43,13 @@ def test_aggregate_predictor_counts(agg): def test_aggregate_summary_by_channel(agg): summary_by_channel = agg.summary_by_channel().collect() assert summary_by_channel.shape[0] == 3 - assert summary_by_channel.shape[1] == 21 + assert summary_by_channel.shape[1] == 23 assert summary_by_channel["Total Number of Actions"].to_list() == [24, 27, 19] def test_aggregate_overall_summary(agg): overall_summary = agg.overall_summary().collect() assert overall_summary.shape[0] == 1 - assert overall_summary.shape[1] == 18 + assert overall_summary.shape[1] == 20 assert overall_summary["Number of Valid Channels"].item() == 3 assert overall_summary["Total Number of Treatments"].item() == 0 diff --git a/python/tests/test_datasets.py b/python/tests/test_datasets.py index 01217fcc..4e9569e3 100644 --- a/python/tests/test_datasets.py +++ b/python/tests/test_datasets.py @@ -21,7 +21,7 @@ def __new__(cls, ldf: pl.LazyFrame): def test_import_CDHSample(): Sample = datasets.cdh_sample() - assert Sample.model_data.shape == (1047, 27) + assert Sample.model_data.shape == (1047, 28) def test_import_SampleTrees(): diff --git a/python/tests/test_end_to_end.py b/python/tests/test_end_to_end.py index 77bd07ae..f2545513 100644 --- a/python/tests/test_end_to_end.py +++ b/python/tests/test_end_to_end.py @@ -38,13 +38,13 @@ def __new__(cls, ldf: pl.LazyFrame): def test_end_to_end(sample: ADMDatamart): - assert sample.model_data.shape == (1047, 27) + assert sample.model_data.shape == (1047, 28) assert sample.predictor_data.shape == (70735, 39) - assert sample.combined_data.shape == (4576, 65) + assert sample.combined_data.shape == (4576, 66) - assert sample.aggregates.last().shape == (68, 27) + assert sample.aggregates.last().shape == (68, 28) assert sample.aggregates.last(table="predictor_data").shape == (4576, 39) assert sample.model_data.collect_schema()["SnapshotTime"] == pl.Datetime From 8472f064c7f430c6e15eae798aacc64865a3eb73 Mon Sep 17 00:00:00 2001 From: kass1 Date: Wed, 11 Dec 2024 16:24:23 +0100 Subject: [PATCH 02/21] Change optional dependency instruction --- examples/articles/GettingStarted.ipynb | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/examples/articles/GettingStarted.ipynb b/examples/articles/GettingStarted.ipynb index e7d0f4af..32c31a12 100644 --- a/examples/articles/GettingStarted.ipynb +++ b/examples/articles/GettingStarted.ipynb @@ -32,17 +32,11 @@ "If you do not have Python or no compatible version installed, `uv` will automatically install a compatible version.\n", "\n", "## Optional dependencies\n", - "As of V4 of pdstools, we have made a big effort to reduce the number of big and heavy core dependencies. This means that while initial installation is very fast, you may at some points run into import errors and will be required to install additional dependency groups. If using `uv`, these can be installed with the `--extra` argument. \n", + "As of V4 of pdstools, we have made a big effort to reduce the number of big and heavy core dependencies. This means that while initial installation is very fast, you may at some points run into import errors and will be required to install additional dependency groups. \n", "\n", - "For instance, to install the optional dependencies to use the Pega DX API client, you should run \n", - "\n", - "```bash\n", - "uv pip install pdstools --extra api\n", - "```\n", - "\n", - "The alternative (pip-compatible) syntax for optional dependencies is:\n", + "To install extra dependencies, you can put them in square brackets after a package name. So, for instance, to install pdstools alongside the optional dependencies for the Pega DX API client, you should run:\n", "```bash\n", - "pip install 'pdstools[api]'\n", + "uv pip install 'pdstools[api]'\n", "```\n", "\n", "For an overview of all optional dependencies and the dependency groups they will be installed for, see the table below:\n", From 9f7c0571bcf687db26e2e390f6f962820f8d710d Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Wed, 11 Dec 2024 18:36:14 +0100 Subject: [PATCH 03/21] AGB status in HC --- python/pdstools/reports/HealthCheck.qmd | 30 +++++++++---- python/pdstools/utils/report_utils.py | 57 +++++++++++-------------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/python/pdstools/reports/HealthCheck.qmd b/python/pdstools/reports/HealthCheck.qmd index 4b89288d..83a17ab3 100644 --- a/python/pdstools/reports/HealthCheck.qmd +++ b/python/pdstools/reports/HealthCheck.qmd @@ -241,13 +241,20 @@ The "OmniChannel" percentage is an indicator of the overlap of actions between c df_channel_overview = ( datamart.aggregates.summary_by_channel() .with_columns( - NBAD=pl.when(pl.col("usesNBAD")) - .then( - pl.when(pl.col("usesNBADOnly")) - .then(pl.lit("Yes")) - .otherwise(pl.lit("With additional configurations")) - ) - .otherwise(pl.lit("No")) + NBAD=pl.when(pl.col("usesNBAD").is_null()) + .then(pl.lit("?")) + .when(pl.col("usesNBADOnly")) + .then(pl.lit("Yes")) + .when(pl.col("usesNBAD")) + .then(pl.lit("With additional configurations")) + .otherwise(pl.lit("No")), + AGB=pl.when(pl.col("usesAGB").is_null()) + .then(pl.lit("?")) + .when(pl.col("usesAGBOnly")) + .then(pl.lit("Always")) + .when(pl.col("usesAGB")) + .then(pl.lit("Yes")) + .otherwise(pl.lit("No")), ) .drop( [ @@ -272,7 +279,7 @@ formatted_channel_overview = ( "Treatments": ["Total Number of Treatments", "Used Treatments"], "Issues": "Issues", "OmniChannel": "OmniChannel Actions", - "CTR" : "CTR", + "CTR": "CTR", }, highlight_lists={ "Channel": cdh_guidelines.standard_channels, @@ -289,7 +296,7 @@ formatted_channel_overview = ( ) .tab_spanner( label=html("ADM Models"), - columns=["Positives", "ResponseCount", "Performance", "Configuration"], + columns=["Positives", "ResponseCount", "Performance", "Configuration", "AGB"], ) .tab_spanner( label=html("NBAD Setup"), @@ -322,6 +329,8 @@ display( "isValid", "usesNBAD", "usesNBADOnly", + "usesAGB", + "usesAGBOnly", ] ) .tab_style( @@ -339,9 +348,12 @@ display( "Performance", "Configuration", "CTR", + "AGB", "isValid", "usesNBAD", "usesNBADOnly", + "usesAGB", + "usesAGBOnly", ] ).tab_style( style=style.text(decorate="line-through"), diff --git a/python/pdstools/utils/report_utils.py b/python/pdstools/utils/report_utils.py index 835848d1..ede0a3ae 100644 --- a/python/pdstools/utils/report_utils.py +++ b/python/pdstools/utils/report_utils.py @@ -192,51 +192,42 @@ def apply_rag_styling(gt, col_name, metric): if title is not None: gt = gt.tab_header(title=title, subtitle=subtitle) - def metric_styling_model_performance(gt, cols): - return gt.fmt_number( - decimals=2, - columns=cols, - ) - - def metric_styling_percentage(gt, cols): - return gt.fmt_percent( - decimals=0, - columns=cols, - ) - - def metric_styling_ctr(gt, cols): - return gt.fmt_percent( - decimals=3, - columns=cols, - ) - - def metric_styling_default(gt, cols): - return gt.fmt_number( - decimals=0, - compact=True, - columns=cols, - ) - for metric in highlight_limits.keys(): cols = highlight_limits[metric] if isinstance(cols, str): cols = [cols] + # Highlight colors for col_name in cols: gt = apply_rag_styling(gt, col_name=col_name, metric=metric) - # gt = gt.fmt_number( - # columns=col_name, decimals=0, compact=True - # ) # default number formatting applied to everything - consider being smarter, in config + + # Value formatting match metric: case "Model Performance": - gt = metric_styling_model_performance(gt, cols) + gt = gt.fmt_number( + decimals=2, + columns=cols, + ) case "Engagement Lift": - gt = metric_styling_percentage(gt, cols) + gt = gt.fmt_percent( + decimals=0, + columns=cols, + ) case "OmniChannel": - gt = metric_styling_percentage(gt, cols) + gt = gt.fmt_percent( + decimals=0, + columns=cols, + ) case "CTR": - gt = metric_styling_ctr(gt, cols) + gt = gt.fmt_percent( + decimals=3, + columns=cols, + ) case _: - gt = metric_styling_default(gt, cols) + gt = gt.fmt_number( + decimals=0, + compact=True, + columns=cols, + ) # Highlight columns with non-standard values # TODO consider stripping spaces and discarding case etc so email matches E-Mail From b6aafbb82c38a8d4e8aeb9ac1b8a112d9870c6f2 Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Thu, 12 Dec 2024 12:35:45 +0100 Subject: [PATCH 04/21] Small refactoring of HC code --- python/pdstools/adm/Aggregates.py | 69 ++++++++++++- python/pdstools/adm/Reports.py | 4 +- python/pdstools/reports/HealthCheck.qmd | 125 ++++-------------------- python/pdstools/reports/ModelReport.qmd | 2 + python/pdstools/utils/report_utils.py | 12 ++- python/tests/test_Aggregates.py | 5 + python/tests/test_healthcheck.py | 4 +- 7 files changed, 105 insertions(+), 116 deletions(-) diff --git a/python/pdstools/adm/Aggregates.py b/python/pdstools/adm/Aggregates.py index 67a11339..ef0b39a4 100644 --- a/python/pdstools/adm/Aggregates.py +++ b/python/pdstools/adm/Aggregates.py @@ -566,7 +566,70 @@ def name_normalizer(x): ) ) - def predictor_last_snapshot(self) -> Optional[pl.DataFrame]: + def summary_by_configuration(self) -> pl.DataFrame: + """ + Generates a summary of the ADM model configurations. + + Returns + ------- + pl.DataFrame + A Polars DataFrame containing the configuration summary. + """ + + action_dim_agg = [pl.col("Name").n_unique().alias("Actions")] + if "Treatment" in self.datamart.context_keys: + action_dim_agg += [ + pl.col("Treatment").n_unique().alias("Unique Treatments") + ] + else: + action_dim_agg += [pl.lit(0).alias("Unique Treatments")] + + if "Issue" in self.datamart.context_keys: + action_dim_agg += [ + pl.col("Issue").cast(pl.String).unique().alias("Used for (Issues)") + ] + + group_by_cols = ["Configuration"] + [ + c for c in ["Channel", "Direction"] if c in self.datamart.context_keys + ] + configuration_summary = ( + self.last(table="model_data") + .group_by(group_by_cols) + .agg( + [ + pl.when((pl.col("ModelTechnique") == "GradientBoost").any()) + .then(pl.lit("Yes")) + .when(pl.col("ModelTechnique").is_null().any()) + .then(pl.lit("Unknown")) + .otherwise(pl.lit("No")) + .alias("AGB") + ] + + [ + pl.col("ModelID").n_unique(), + ] + + action_dim_agg + + [pl.sum(["ResponseCount", "Positives"])], + ) + .with_columns( + [ + # pl.col("Configuration") + # .is_in(standardNBADNames.keys()) + # .alias("Standard in NBAD Framework"), + (pl.col("ModelID") / pl.col("Actions")) + .round(2) + .alias("ModelsPerAction"), + ] + ) + .sort(group_by_cols) + ) + if "Issue" in self.datamart.context_keys: + configuration_summary = configuration_summary.with_columns( + pl.col("Used for (Issues)").list.unique().list.sort().list.join(", ") + ) + + return configuration_summary + + def predictors_overview(self) -> Optional[pl.DataFrame]: """ Generate a summary of the last snapshot of predictor data. @@ -585,7 +648,7 @@ def predictor_last_snapshot(self) -> Optional[pl.DataFrame]: predictor_summary = ( self.last(table="predictor_data") - .filter(pl.col("PredictorName") != "Classifier") + .filter(pl.col("PredictorName") != "Classifier") # TODO not name, there is a type .join( self.last(table="model_data") .select(["ModelID"] + model_identifiers) @@ -629,7 +692,7 @@ def predictor_last_snapshot(self) -> Optional[pl.DataFrame]: ) return predictor_summary - except ValueError: + except ValueError: # really? swallowing? return None def overall_summary( diff --git a/python/pdstools/adm/Reports.py b/python/pdstools/adm/Reports.py index 4fa25f5c..b3626cef 100644 --- a/python/pdstools/adm/Reports.py +++ b/python/pdstools/adm/Reports.py @@ -565,8 +565,8 @@ def excel_report( } if self.datamart.predictor_data is not None: - tabs["predictor_last_snapshot"] = ( - self.datamart.aggregates.predictor_last_snapshot() + tabs["predictors_overview"] = ( + self.datamart.aggregates.predictors_overview() ) if predictor_binning and self.datamart.predictor_data is not None: diff --git a/python/pdstools/reports/HealthCheck.qmd b/python/pdstools/reports/HealthCheck.qmd index 83a17ab3..f05f9700 100644 --- a/python/pdstools/reports/HealthCheck.qmd +++ b/python/pdstools/reports/HealthCheck.qmd @@ -73,22 +73,6 @@ def fig_set_xaxis_modelperformance(fig, label="Model Performance"): .update_xaxes(title=label, showticklabels=True, visible=True) ) return fig - - -# def highlight_non_standard_channels(v): -# if v not in set(standardNBADNames.values()): -# color = "orange" -# else: -# color = "" -# return "background-color: %s" % color - -# def highlight_non_standard_configurations(v): -# if v not in set(standardNBADNames.keys()): -# color = "orange" -# else: -# color = "" -# return "background-color: %s" % color - ``` ```{python} @@ -100,7 +84,9 @@ def fig_set_xaxis_modelperformance(fig, label="Model Performance"): title = "ADM Model Overview" subtitle = "Sample data" -# Insert the paths to your data files here to run the notebook from your IDE +# Insert the paths to your data files here to run the notebook from your IDE. +# Edit the _quarto.yml to enable/disable specific sections of the quarto output. +# Parameters will be overriden by quarto when a parameters yaml is provided model_file_path = None prediction_file_path = None @@ -229,7 +215,7 @@ except Exception as e: The [Plotly](https://plotly.com/python/) charts have [user controls for panning, zooming etc](https://plotly.com/chart-studio-help/zoom-pan-hover-controls/) but note that these interactive plots do not render well in portals like Sharepoint or Box. It is preferable to view them from a browser. ::: -# Overview of Channels +# Overview of the Channels In a typical NBAD setup, treatments for a channels are modelled by a channel specific model configuration as well as a cross-channel *OmniAdaptiveModel* configuration. This cross-channel configuration is typically only used as a fall-back, and, additionally, for action-level insights. @@ -241,9 +227,7 @@ The "OmniChannel" percentage is an indicator of the overlap of actions between c df_channel_overview = ( datamart.aggregates.summary_by_channel() .with_columns( - NBAD=pl.when(pl.col("usesNBAD").is_null()) - .then(pl.lit("?")) - .when(pl.col("usesNBADOnly")) + NBAD=pl.when(pl.col("usesNBADOnly")) .then(pl.lit("Yes")) .when(pl.col("usesNBAD")) .then(pl.lit("With additional configurations")) @@ -251,9 +235,9 @@ df_channel_overview = ( AGB=pl.when(pl.col("usesAGB").is_null()) .then(pl.lit("?")) .when(pl.col("usesAGBOnly")) - .then(pl.lit("Always")) - .when(pl.col("usesAGB")) .then(pl.lit("Yes")) + .when(pl.col("usesAGB")) + .then(pl.lit("Partially")) .otherwise(pl.lit("No")), ) .drop( @@ -477,6 +461,10 @@ if prediction_file_path: "isValid", ] ) + .tab_style( + style=style.text(decorate="line-through"), + locations=loc.body(rows=pl.col("isValid").not_()), + ) ) display(gt) @@ -794,7 +782,7 @@ except Exception as e: report_utils.quarto_plot_exception("Success Rates over Time", e) ``` -# Overview of Adaptive Models +# Overview of the Adaptive Models ```{python} n_unique_models = len(last_data.select("ModelID").unique()) # TODO or uniqueN ? @@ -809,47 +797,11 @@ There are a total of **{n_unique_models}** Adaptive Models in the latest snapsho In the standard configuration there is one Adaptive model per treatment/action for a configuration. ```{python} -action_dim_agg = [pl.col("Name").n_unique().alias("Actions")] -if report_utils.polars_col_exists(last_data, "Treatment"): - action_dim_agg += [pl.col("Treatment").n_unique().alias("Unique Treatments")] -else: - action_dim_agg += [pl.lit(0).alias("Unique Treatments")] - -# TODO work this into a get_model_overview function in the ADMDatamart -# note there already is a perhaps useful model_summary in there which returns -# some but not all the info we need here -# datamart.model_summary(context_keys=["Configuration","Channel","Direction"]).collect() - -model_overview = ( - last_data.group_by( - ["Configuration"] - + report_utils.polars_subset_to_existing_cols( - datamart_all_columns, ["Channel", "Direction"] - ) - ) - .agg( - [ - pl.col("ModelID").n_unique(), - ] - + action_dim_agg - + [pl.sum("ResponseCount"), pl.sum("Positives")] - ) - .with_columns( - [ - # pl.col("Configuration") - # .is_in(standardNBADNames.keys()) - # .alias("Standard in NBAD Framework"), - (pl.col("ModelID") / pl.col("Actions")) - .round(2) - .alias("ModelsPerAction"), - ] - ) - .sort(["Configuration", "Channel", "Direction"]) -) +model_overview = datamart.aggregates.summary_by_configuration() display( report_utils.table_standard_formatting( - model_overview, + model_overview.collect(), title="Model Overview", cdh_guidelines=cdh_guidelines, highlight_limits={ @@ -872,6 +824,7 @@ display( ModelID="Number of Models", Actions="Unique Actions", ModelsPerAction="Average number of Models per Action", + ResponseCount="Total Responses", ) ) ``` @@ -884,11 +837,9 @@ If there are any model configurations that have fewer than {configuration_respon ) configuration_overview = ( - datamart.model_data - # first, take max per model ID - .group_by("Configuration").agg( - pl.max("ResponseCount").alias("Responses"), pl.max("Positives") - ) + datamart.aggregates.last(table="model_data") + .group_by("Configuration") + .agg(pl.sum("ResponseCount").alias("Responses"), pl.sum("Positives")) ).collect() all_configurations = configuration_overview.select(["Configuration"]).unique() @@ -1089,46 +1040,6 @@ This is something that can be configured when reading the data. By default it si You can customize this (when reading in the data) to add patterns to identify for example external scores. -## Number of Predictors per model configuration - -This shows the total number of predictors per model configuration (this includes both active and inactive predictors). - -Note that the total number of predictors in the model data does not always equate the data from the more detailed view split by category below. - -```{python} -if datamart.predictor_data is not None: - context_aggregations = [] - if report_utils.polars_col_exists(datamart.combined_data, "Channel"): - context_aggregations += [pl.col("Channel").unique().alias("Used in (Channels)")] - if report_utils.polars_col_exists(datamart.combined_data, "Issue"): - context_aggregations += [pl.col("Issue").unique().alias("Used for (Issues)")] - - predictors_per_configuration = ( - datamart.combined_data.filter(pl.col("EntryType") != "Classifier") - .group_by("Configuration") - .agg( - [pl.col("PredictorName").unique().count().alias("Predictor Count")] - + context_aggregations - ) - .sort("Configuration") - .collect() - ) - - gt = report_utils.table_standard_formatting( - predictors_per_configuration, "Number of Predictors per Configuration", - cdh_guidelines=cdh_guidelines, - ).tab_style( - style=style.text(weight="bold"), - locations=loc.body(columns="Predictor Count"), - ) - gt = report_utils.table_style_predictor_count(gt, ["Predictor Count"], cdh_guidelines) - - display(gt) - -else: - report_utils.quarto_callout_no_predictor_data_warning() - -``` ## Number of Predictors per Predictor Category diff --git a/python/pdstools/reports/ModelReport.qmd b/python/pdstools/reports/ModelReport.qmd index b4e677ff..84a62f36 100644 --- a/python/pdstools/reports/ModelReport.qmd +++ b/python/pdstools/reports/ModelReport.qmd @@ -49,6 +49,8 @@ from pdstools.utils import report_utils ```{python} # | tags: [parameters] +# Insert the paths to your data files here to run the notebook from your IDE. +# Edit the _quarto.yml to enable/disable specific sections of the quarto output. # Parameters will be overriden by quarto when a parameters yaml is provided title = "ADM Model Details" diff --git a/python/pdstools/utils/report_utils.py b/python/pdstools/utils/report_utils.py index ede0a3ae..5275c344 100644 --- a/python/pdstools/utils/report_utils.py +++ b/python/pdstools/utils/report_utils.py @@ -1,3 +1,4 @@ +import re import traceback from typing import Dict, List, Literal, Optional, Union from IPython.display import display, Markdown @@ -230,12 +231,19 @@ def apply_rag_styling(gt, col_name, metric): ) # Highlight columns with non-standard values - # TODO consider stripping spaces and discarding case etc so email matches E-Mail + def simplify_name(x: str) -> str: + if x is None: + return x + return re.sub("\\W", "", x, flags=re.IGNORECASE).upper() + for col_name in highlight_lists.keys(): if col_name in source_table.collect_schema().names(): + simplified_names = [simplify_name(x) for x in highlight_lists[col_name]] values = source_table[col_name].to_list() non_standard_rows = [ - i for i, v in enumerate(values) if v not in highlight_lists[col_name] + i + for i, v in enumerate(values) + if simplify_name(v) not in simplified_names ] gt = apply_style(gt, "yellow", non_standard_rows) diff --git a/python/tests/test_Aggregates.py b/python/tests/test_Aggregates.py index 8fc53561..85cb8876 100644 --- a/python/tests/test_Aggregates.py +++ b/python/tests/test_Aggregates.py @@ -53,3 +53,8 @@ def test_aggregate_overall_summary(agg): assert overall_summary.shape[1] == 20 assert overall_summary["Number of Valid Channels"].item() == 3 assert overall_summary["Total Number of Treatments"].item() == 0 + +def test_summary_by_configuration(agg): + configuration_summary = agg.summary_by_configuration().collect() + assert "AGB" in configuration_summary.columns + \ No newline at end of file diff --git a/python/tests/test_healthcheck.py b/python/tests/test_healthcheck.py index c6e8984c..a843b5f6 100644 --- a/python/tests/test_healthcheck.py +++ b/python/tests/test_healthcheck.py @@ -38,7 +38,7 @@ def test_ExportTables(sample: ADMDatamart): spreadsheet = ExcelFile(excel) assert list(spreadsheet.sheet_names) == [ "modeldata_last_snapshot", - "predictor_last_snapshot", + "predictors_overview", "predictor_binning", ] # TODO we could go further and check the size of the sheets @@ -54,7 +54,7 @@ def test_ExportTables_NoBinning(sample: ADMDatamart): spreadsheet = ExcelFile(excel) assert list(spreadsheet.sheet_names) == [ "modeldata_last_snapshot", - "predictor_last_snapshot", + "predictors_overview", ] # TODO we could go further and check the size of the sheets # spreadsheet = read_excel(excel, sheet_name=None) From 30ffc954793293122723777bb299132e5de9b09b Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Thu, 12 Dec 2024 14:28:51 +0100 Subject: [PATCH 05/21] Small refactoring of HC code --- python/pdstools/reports/HealthCheck.qmd | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/python/pdstools/reports/HealthCheck.qmd b/python/pdstools/reports/HealthCheck.qmd index f05f9700..359d5b27 100644 --- a/python/pdstools/reports/HealthCheck.qmd +++ b/python/pdstools/reports/HealthCheck.qmd @@ -1032,21 +1032,16 @@ if datamart.predictor_data is None: ) ``` -This analysis looks at the predictors that are driving the models. - -The predictors are categorized (by color) by their “source”. - -This is something that can be configured when reading the data. By default it simply takes the first part before the dot in the predictor name, so this typically distinguishes between e.g. *Customer*, *Account*, *IH* and parameterized (*Param.*) predictors. - -You can customize this (when reading in the data) to add patterns to identify for example external scores. - ## Number of Predictors per Predictor Category -Split by category (defaults to the string before the first dot, can be overridden when reading the data). +The Predictor Categories identify the source of the predictors. By default we split by the first dot, so this distinguishes between between e.g. *Customer*, *Account*, *IH* and parameterized (*Param.*) predictors. + +You can override this behavior when the data is read. The numbers here can differ from the totals above, these ones are leading. + ::: {.callout-tip title="Guidance"} - Total number of predictors per model 200 - 700 to stay within service limits - There should be some “IH” predictors but no more than ca 100 of them From 8594d8b070b11224520a98882d9a7c3f05929eec Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Fri, 13 Dec 2024 16:33:40 +0100 Subject: [PATCH 06/21] Initial version of conversion modeling on IH --- .../ih/Conversion_Modeling_Reporting.ipynb | 10151 ++++++++++++++++ 1 file changed, 10151 insertions(+) create mode 100644 examples/ih/Conversion_Modeling_Reporting.ipynb diff --git a/examples/ih/Conversion_Modeling_Reporting.ipynb b/examples/ih/Conversion_Modeling_Reporting.ipynb new file mode 100644 index 00000000..aefc4ab7 --- /dev/null +++ b/examples/ih/Conversion_Modeling_Reporting.ipynb @@ -0,0 +1,10151 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import polars as pl\n", + "from pdstools import read_ds_export\n", + "import re\n", + "\n", + "import plotly.io as pio\n", + "import plotly as plotly\n", + "import plotly.express as px\n", + "import plotly.graph_objs as go\n", + "from plotly.subplots import make_subplots\n", + "\n", + "plotly.offline.init_notebook_mode()\n", + "pio.renderers.default = \"vscode\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "def capitalize(fields: list) -> list:\n", + " \"\"\"Applies automatic capitalization.\n", + " Parameters\n", + " ----------\n", + " fields : list\n", + " A list of names\n", + "\n", + " Returns\n", + " -------\n", + " fields : list\n", + " The input list, but each value properly capitalized\n", + " \"\"\"\n", + " capitalize_end_words = [\n", + " \"ID\",\n", + " \"Key\",\n", + " \"Name\",\n", + " \"Treatment\",\n", + " \"Count\",\n", + " \"Category\",\n", + " \"Class\",\n", + " \"Time\",\n", + " \"DateTime\",\n", + " \"UpdateTime\",\n", + " \"Version\",\n", + " \"Rate\",\n", + " \"Ratio\",\n", + " \"Negatives\",\n", + " \"Positives\",\n", + " \"Threshold\",\n", + " \"Error\",\n", + " \"Importance\",\n", + " \"Type\",\n", + " \"Percentage\",\n", + " \"Index\",\n", + " \"Symbol\",\n", + " \"ResponseCount\",\n", + " \"ConfigurationName\",\n", + " \"Configuration\",\n", + " ]\n", + " if not isinstance(fields, list):\n", + " fields = [fields]\n", + " fields_new = [re.sub(\"^p([xyz])\", \"\", field) for field in fields]\n", + " seen = set(fields)\n", + " for i, item in enumerate(fields_new):\n", + " if item in seen:\n", + " fields_new[i] = fields[i]\n", + " for word in capitalize_end_words:\n", + " fields_new = [re.sub(word + '\\b', word, field, flags=re.I) for field in fields_new]\n", + " fields_new = [field[:1].upper() + field[1:] for field in fields_new]\n", + " return fields_new" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "ih = read_ds_export(\"Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\", path=\"../../data\")" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Schema([('ControlGroupValidityStart', String),\n", + " ('Stage', String),\n", + " ('Value', Float64),\n", + " ('Behaviour', String),\n", + " ('PropensitySource', String),\n", + " ('MktType', String),\n", + " ('ExternalAudienceId', String),\n", + " ('Response', String),\n", + " ('ControlGroupValidityEnd', String),\n", + " ('ReferrerUrl', String),\n", + " ('Propensity', Float64),\n", + " ('WorkID', String),\n", + " ('ActionContext', String),\n", + " ('Name', String),\n", + " ('Treatment', String),\n", + " ('BundleName', String),\n", + " ('Division', String),\n", + " ('ExternalID', String),\n", + " ('CustomerID', String),\n", + " ('PartitionKey', Int64),\n", + " ('PyCategory', String),\n", + " ('Category', String),\n", + " ('ExperimentGroup', String),\n", + " ('Latitude', Float64),\n", + " ('MaxBudget', String),\n", + " ('Group', String),\n", + " ('ChannelSubGroup', String),\n", + " ('Outcome', String),\n", + " ('Reason', String),\n", + " ('Rank', Int64),\n", + " ('OutcomeTime', String),\n", + " ('SubjectType', String),\n", + " ('Application', String),\n", + " ('ModelClass', String),\n", + " ('SubCategory', String),\n", + " ('IPAddress', String),\n", + " ('ModelControlGroup', String),\n", + " ('CustomerSegment', String),\n", + " ('UserAgent', String),\n", + " ('Interaction', String),\n", + " ('StreamPartition', String),\n", + " ('ChannelGroup', String),\n", + " ('Step', String),\n", + " ('Label', String),\n", + " ('Channel', String),\n", + " ('Fulfilled', String),\n", + " ('URI', String),\n", + " ('Issue', String),\n", + " ('Longitude', Float64),\n", + " ('OutcomeWeight', Float64),\n", + " ('Priority', Float64),\n", + " ('Operator', String),\n", + " ('SubjectID', String),\n", + " ('Strategy', String),\n", + " ('Organization', String),\n", + " ('Unit', String),\n", + " ('BundleHead', Boolean),\n", + " ('DecisionTime', String),\n", + " ('UpdateDateTime', Int64),\n", + " ('StreamPosition', String),\n", + " ('GroupID', String),\n", + " ('ISFactID', String),\n", + " ('TargetBudget', Float64),\n", + " ('InteractionID', String),\n", + " ('Revenue', Float64),\n", + " ('ApplicationVersion', String),\n", + " ('DeviceType', String),\n", + " ('AggregateCount', Int64),\n", + " ('Component', String),\n", + " ('PyWorkID', String),\n", + " ('MktValue', String),\n", + " ('BudgetedCost', Float64),\n", + " ('CustomerSubSegment', String),\n", + " ('Cost', Float64),\n", + " ('ConversionEventID', String),\n", + " ('FinalPropensity', Float64),\n", + " ('Weight', Float64),\n", + " ('EvaluationCriteria', String),\n", + " ('PlacementType', String),\n", + " ('Direction', String),\n", + " ('Journey', String),\n", + " ('Utm_medium', String),\n", + " ('FactID', String),\n", + " ('CampaignCode', String),\n", + " ('PyRevenue', String),\n", + " ('InternalTags', String)])" + ] + }, + "execution_count": 70, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dframe_columns = ih.collect_schema().names()\n", + "cols = capitalize(dframe_columns)\n", + "ih = ih.rename(dict(map(lambda i, j: (i, j), dframe_columns, cols)))\n", + "ih.collect_schema()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At first, take a look into the IH dataframe, explore the columns, outcome types and business structure" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (9, 92)
statisticControlGroupValidityStartStageValueBehaviourPropensitySourceMktTypeExternalAudienceIdResponseControlGroupValidityEndReferrerUrlPropensityWorkIDActionContextNameTreatmentBundleNameDivisionExternalIDCustomerIDPartitionKeyPyCategoryCategoryExperimentGroupLatitudeMaxBudgetGroupChannelSubGroupOutcomeReasonRankOutcomeTimeSubjectTypeApplicationModelClassSubCategoryIPAddress…OrganizationUnitBundleHeadDecisionTimeUpdateDateTimeStreamPositionGroupIDISFactIDTargetBudgetInteractionIDRevenueApplicationVersionDeviceTypeAggregateCountComponentPyWorkIDMktValueBudgetedCostCustomerSubSegmentCostConversionEventIDFinalPropensityWeightEvaluationCriteriaPlacementTypeDirectionJourneyUtm_mediumFactIDCampaignCodePyRevenueInternalTagsOutcomeDateTimeDayMonthYearQuarter
strstrstrf64strstrstrstrstrstrstrf64strstrstrstrstrstrstrstrf64strstrstrf64strstrstrstrstrf64strstrstrstrstrstr…strstrf64strf64strstrstrf64strf64strstrf64strstrstrf64strf64strf64f64strstrstrstrstrstrstrstrstrstrstrstrstrstr
"count""304784""304784"304784.0"304784""304784""304784""304784""304784""304784""304784"304784.0"304784""304784""304784""304784""304784""304784""304784""304784"304784.0"304784""304784""52854"304784.0"304784""304784""304784""304784""304784"304784.0"304784""304784""304784""304784""304784""304784"…"304784""304784"304784.0"304784"304784.0"304784""304784""304784"304784.0"304784"304784.0"304784""304784"304784.0"304784""304784""304784"304784.0"304784"304784.0"304784"304784.0304784.0"304784""304784""304784""304784""304784""304784""304784""304784""304784""304784""304784""304784""304784""304784"
"null_count""0""0"0.0"0""0""0""0""0""0""0"0.0"0""0""0""0""0""0""0""0"0.0"0""0""251930"0.0"0""0""0""0""0"0.0"0""0""0""0""0""0"…"0""0"0.0"0"0.0"0""0""0"0.0"0"0.0"0""0"0.0"0""0""0"0.0"0"0.0"0"0.00.0"0""0""0""0""0""0""0""0""0""0""0""0""0""0"
"mean"nullnull0.0nullnullnullnullnullnullnull0.146205nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull2.501184nullnullnullnullnullnull…nullnull0.0null1.7337e12nullnullnull0.0null0.0nullnull1.0nullnullnull0.0null0.0null0.1971370.0nullnullnullnullnullnullnullnullnull"2024-12-09 00:35:11.766000""2024-12-08 11:34:38.282000"nullnullnull
"std"nullnull0.0nullnullnullnullnullnullnull0.195219nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull3.501869nullnullnullnullnullnull…nullnullnullnull2.7342e8nullnullnull0.0null0.0nullnull0.0nullnullnull0.0null0.0null0.2823090.0nullnullnullnullnullnullnullnullnullnullnullnullnullnull
"min"""""0.0""""""""""""""0.0"""""AppleIPhone15128GB"""""""""""0.0""""""0.0"0""""""Clicked"""1.0"20241203T115401.223 GMT""Data-CDHCaptureResponse-SR""ConversionModel"""""""…""""0.0"20241203T112232.703 GMT"1.7332e12"0"""""0.0"-1808289819383917128"0.0"01.01.01"""1.0"ConversionAttribution"""""0.0""0.0""0.00.0"""""""""""0""""""""2024-12-03 11:54:01.223000""2024-12-03""2024-12""2024""2024_Q4"
"25%"nullnull0.0nullnullnullnullnullnullnull0.0nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull1.0nullnullnullnullnullnull…nullnullnullnull1.7334e12nullnullnull0.0null0.0nullnull1.0nullnullnull0.0null0.0null0.00.0nullnullnullnullnullnullnullnullnull"2024-12-05 19:26:36.015000""2024-12-05"nullnullnull
"50%"nullnull0.0nullnullnullnullnullnullnull0.090087nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull1.0nullnullnullnullnullnull…nullnullnullnull1.7335e12nullnullnull0.0null0.0nullnull1.0nullnullnull0.0null0.0null0.0979490.0nullnullnullnullnullnullnullnullnull"2024-12-06 09:18:47.649000""2024-12-06"nullnullnull
"75%"nullnull0.0nullnullnullnullnullnullnull0.196034nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull1.0nullnullnullnullnullnull…nullnullnullnull1.7340e12nullnullnull0.0null0.0nullnull1.0nullnullnull0.0null0.0null0.2026590.0nullnullnullnullnullnullnullnullnull"2024-12-12 07:20:04.558000""2024-12-12"nullnullnull
"max"""""0.0"""""PropensityPriority"""""""""1.0"""Customer""WalkieTalkie""WalkieHero""""""""D-ZZZM3AGC"0.0"""""Conversion-Test"0.0"0""Phones""""Product conversion""Control"32.0"20241213T090853.607 GMT""PegaMKT-Data-Customer""ConversionModel""Engagement"""""…""""0.0"20241213T090853.333 GMT"1.7341e12"999"""""0.0"8789841886479301898"0.0"01.01.01""desktop"1.0"Setting Rank""""NBAHealth_PropensityPriority"0.0""0.0"iPhone16Pro"1.00.0"Default""Hero""Inbound""""""0""""""""2024-12-13 09:08:53.607000""2024-12-13""2024-12""2024""2024_Q4"
" + ], + "text/plain": [ + "shape: (9, 92)\n", + "┌────────────┬────────────────┬────────┬──────────┬───┬───────────────┬─────────┬────────┬─────────┐\n", + "│ statistic ┆ ControlGroupVa ┆ Stage ┆ Value ┆ … ┆ Day ┆ Month ┆ Year ┆ Quarter │\n", + "│ --- ┆ lidityStart ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ --- ┆ str ┆ f64 ┆ ┆ str ┆ str ┆ str ┆ str │\n", + "│ ┆ str ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "╞════════════╪════════════════╪════════╪══════════╪═══╪═══════════════╪═════════╪════════╪═════════╡\n", + "│ count ┆ 304784 ┆ 304784 ┆ 304784.0 ┆ … ┆ 304784 ┆ 304784 ┆ 304784 ┆ 304784 │\n", + "│ null_count ┆ 0 ┆ 0 ┆ 0.0 ┆ … ┆ 0 ┆ 0 ┆ 0 ┆ 0 │\n", + "│ mean ┆ null ┆ null ┆ 0.0 ┆ … ┆ 2024-12-08 ┆ null ┆ null ┆ null │\n", + "│ ┆ ┆ ┆ ┆ ┆ 11:34:38.2820 ┆ ┆ ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ 00 ┆ ┆ ┆ │\n", + "│ std ┆ null ┆ null ┆ 0.0 ┆ … ┆ null ┆ null ┆ null ┆ null │\n", + "│ min ┆ ┆ ┆ 0.0 ┆ … ┆ 2024-12-03 ┆ 2024-12 ┆ 2024 ┆ 2024_Q4 │\n", + "│ 25% ┆ null ┆ null ┆ 0.0 ┆ … ┆ 2024-12-05 ┆ null ┆ null ┆ null │\n", + "│ 50% ┆ null ┆ null ┆ 0.0 ┆ … ┆ 2024-12-06 ┆ null ┆ null ┆ null │\n", + "│ 75% ┆ null ┆ null ┆ 0.0 ┆ … ┆ 2024-12-12 ┆ null ┆ null ┆ null │\n", + "│ max ┆ ┆ ┆ 0.0 ┆ … ┆ 2024-12-13 ┆ 2024-12 ┆ 2024 ┆ 2024_Q4 │\n", + "└────────────┴────────────────┴────────┴──────────┴───┴───────────────┴─────────┴────────┴─────────┘" + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ih = (\n", + " ih\n", + " .with_columns(\n", + " pl.col('OutcomeTime').str.strptime(pl.Datetime, \"%Y%m%dT%H%M%S%.3f %Z\").alias('OutcomeDateTime')\n", + " )\n", + " .with_columns(\n", + " [\n", + " pl.col(\"OutcomeDateTime\").dt.date().alias(\"Day\"),\n", + " (pl.col(\"OutcomeDateTime\").dt.strftime(\"%Y-%m\")).alias(\"Month\"),\n", + " pl.col(\"OutcomeDateTime\").dt.year().cast(str).alias(\"Year\"),\n", + " (pl.col(\"OutcomeDateTime\").dt.year().cast(str) + \"_Q\" + pl.col(\n", + " \"OutcomeDateTime\").dt.quarter().cast(\n", + " str)).alias(\"Quarter\")\n", + " ]\n", + " )\n", + " )\n", + "ih.describe()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Assuming conversion modelling setup folllows OOTB approach, so that IH contains Conversion outcome as positive result and Impression (for inbound channels) and Pending (for outbound channels) are treated as negative outcome. Each Conversion has corresponding Impression/Pending record, so to calculate correct Conversion Rate is count(Conversion) / (count(Impression/Pending) - 2 * count(Conversion))" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [ + "positive_model_response = [\"Conversion\"]\n", + "all_model_response = [\"Impression\", \"Pending\"]\n", + "group_by = [\"Day\", \"Month\", \"Year\", \"Quarter\", \"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", + "\n", + "ih = ih.filter(pl.col('ExperimentGroup').is_not_null())" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [], + "source": [ + "ih_analysis = (\n", + " ih.filter(\n", + " (pl.col(\"Outcome\").is_in(all_model_response + positive_model_response))\n", + " )\n", + " .with_columns([\n", + " pl.when(pl.col('Outcome').is_in(positive_model_response)).\n", + " then(1).otherwise(0).alias('Outcome_Binary')\n", + " ])\n", + " .group_by(group_by)\n", + " .agg([\n", + " pl.len().alias('Count'),\n", + " pl.sum(\"Outcome_Binary\").alias(\"Positives\")\n", + " ])\n", + " .with_columns([\n", + " (pl.col(\"Count\") - (2 * pl.col(\"Positives\"))).alias(\"Negatives\")\n", + " ])\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Showing results as a gauge plot across channel dimension to compare conversion rates inside specific channel between conversion and Engagement models. Set relevant reference data (baseline conversion rate). Delta from baseline is shown inside Gauge." + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "delta": { + "reference": 0.055, + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0, + 1 + ], + "y": [ + 0.575, + 1 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "bar": { + "color": "#EC9B00" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75, + "value": 0.055 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "Web Conversion-Control" + }, + "type": "indicator", + "value": 0.053588795954094534 + }, + { + "delta": { + "reference": 0.055, + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0, + 1 + ], + "y": [ + 0, + 0.425 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75, + "value": 0.055 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "Web Conversion-Test" + }, + "type": "indicator", + "value": 0.05898741473724661 + } + ], + "layout": { + "autosize": true, + "height": 540, + "margin": { + "b": 10, + "l": 10, + "r": 10, + "t": 120 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[CONV] Conversion (Channel/Model Type)" + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ChannelExperimentGroupNegativesPositivesCountConversionRateStdErrNameCName
0WebConversion-Control9731551108330.0535890.002221Web Conversion-ControlWeb_Conversion-Control
1WebConversion-Test9795614110230.0589870.002309Web Conversion-TestWeb_Conversion-Test
\n", + "
" + ], + "text/plain": [ + " Channel ExperimentGroup Negatives Positives Count ConversionRate \\\n", + "0 Web Conversion-Control 9731 551 10833 0.053589 \n", + "1 Web Conversion-Test 9795 614 11023 0.058987 \n", + "\n", + " StdErr Name CName \n", + "0 0.002221 Web Conversion-Control Web_Conversion-Control \n", + "1 0.002309 Web Conversion-Test Web_Conversion-Test " + ] + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gauge_group_by = [\"Channel\", \"ExperimentGroup\"]\n", + "reference = {'Web_Conversion-Test' : 0.055, 'Web_Conversion-Control' : 0.055}\n", + "gauge_data = (\n", + " ih_analysis.group_by(gauge_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\")\n", + " )\n", + " .with_columns(\n", + " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"ConversionRate\")]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", + " (\n", + " (\n", + " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", + " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", + " )\n", + " ** 0.5\n", + " )\n", + " ).alias(\"StdErr\")\n", + " ]\n", + " )\n", + " .sort(gauge_group_by, descending=False)\n", + " .collect()\n", + " )\n", + "\n", + "gauge_data = gauge_data.to_pandas()\n", + "\n", + "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", + "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", + "\n", + "gauge_data['Name'] = gauge_data[gauge_group_by].apply(lambda r: ' '.join(r.values.astype(str)), axis=1)\n", + "gauge_data['CName'] = gauge_data[gauge_group_by].apply(lambda r: '_'.join(r.values.astype(str)), axis=1)\n", + "\n", + "fig = make_subplots(rows=rows,\n", + " cols=cols,\n", + " specs=[[{\"type\": \"indicator\"} for c in range(cols)] for t in range(rows)]\n", + " )\n", + "fig.update_layout(\n", + " height=270 * rows,\n", + " autosize=True,\n", + " title='[CONV] Conversion (Channel/Model Type)',\n", + " margin=dict(b=10, t=120, l=10, r=10))\n", + "\n", + "for index, row in gauge_data.iterrows():\n", + " ref_value = reference.get(row['CName'], None)\n", + " gauge = {\n", + " 'axis': {'tickformat': ',.2%'},\n", + " 'threshold': {\n", + " 'line': {'color': \"red\", 'width': 2},\n", + " 'thickness': 0.75,\n", + " 'value': ref_value\n", + " }\n", + " }\n", + " if ref_value:\n", + " if row['ConversionRate'] < ref_value:\n", + " gauge = {\n", + " 'axis': {'tickformat': ',.2%'},\n", + " 'bar': {'color': '#EC5300' if row['ConversionRate'] < (0.75 * ref_value) else '#EC9B00'},\n", + " 'threshold': {\n", + " 'line': {'color': \"red\", 'width': 2},\n", + " 'thickness': 0.75,\n", + " 'value': ref_value\n", + " }\n", + " }\n", + "\n", + " trace1 = go.Indicator(mode=\"gauge+number+delta\",\n", + " number={'valueformat': \",.2%\"},\n", + " value=row['ConversionRate'],\n", + " delta={'reference': ref_value, 'valueformat': \",.2%\"},\n", + " title={'text': row['Name']},\n", + " gauge=gauge,\n", + " )\n", + " r, c = divmod(index, cols)\n", + " fig.add_trace(\n", + " trace1,\n", + " row=(r + 1), col=(c + 1)\n", + " )\n", + "fig.show()\n", + "gauge_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This plot provides detailed view on individual actions conversion rates and model type used." + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "branchvalues": "total", + "customdata": [ + [ + "0.0038969602362036787", + "199", + "3317", + 0.056598407281001135 + ], + [ + "0.0037745260830149003", + "206", + "3489", + 0.0557510148849797 + ], + [ + "0.0041720272869534515", + "125", + "2490", + 0.04780114722753346 + ], + [ + "0.025967220580682373", + "1", + "37", + 0.026315789473684206 + ], + [ + "0.019858547202852933", + "18", + "186", + 0.08823529411764706 + ], + [ + "0.004293640832369911", + "205", + "3022", + 0.0635264951967772 + ], + [ + "0.02063226798733351", + "10", + "138", + 0.06756756756756757 + ], + [ + "0.06405644848900469", + "2", + "19", + 0.09523809523809523 + ], + [ + "0.0", + "0", + "30", + 0 + ], + [ + "0.0", + "0", + "59", + 0 + ], + [ + "0.01514607976878216", + "4", + "126", + 0.03076923076923077 + ], + [ + "0.013976239729195046", + "9", + "201", + 0.04285714285714286 + ], + [ + "0.004783748419734918", + "112", + "2042", + 0.051996285979572884 + ], + [ + "0.004277411044689493", + "189", + "2926", + 0.060674157303370786 + ], + [ + "0.005961058344078005", + "83", + "1402", + 0.05589225589225589 + ], + [ + "0.03140223998671625", + "2", + "42", + 0.045454545454545456 + ], + [ + "(?)", + "(?)", + "(?)", + 0.05616436346050749 + ], + [ + "(?)", + "(?)", + "(?)", + 0.04749962547424086 + ], + [ + "(?)", + "(?)", + "(?)", + 0.06502768659262644 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07108371196768987 + ], + [ + "0.0", + "0", + "(?)", + 0 + ], + [ + "(?)", + "(?)", + "(?)", + 0.038268530336519 + ], + [ + "(?)", + "(?)", + "(?)", + 0.05714380606105013 + ], + [ + "(?)", + "(?)", + "(?)", + 0.05559477467779822 + ], + [ + "(?)", + "(?)", + "(?)", + 0.056357811460585466 + ], + [ + "(?)", + "(?)", + "(?)", + 0.056357811460585466 + ], + [ + "(?)", + "(?)", + "(?)", + 0.056357811460585466 + ], + [ + "(?)", + "(?)", + "(?)", + 0.056357811460585466 + ] + ], + "domain": { + "x": [ + 0, + 1 + ], + "y": [ + 0, + 1 + ] + }, + "hovertemplate": "labels=%{label}
Count=%{value}
parent=%{parent}
id=%{id}
StdErr=%{customdata[0]}
Positives=%{customdata[1]}
Negatives=%{customdata[2]}
ConversionRate=%{color}", + "ids": [ + "ALL/Web/Acquisition/Phones/AppleIPhone15128GB/Conversion-Control", + "ALL/Web/Acquisition/Phones/AppleIPhone15128GB/Conversion-Test", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Control", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB/Conversion-Control", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB/Conversion-Test", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB/Conversion-Control", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB/Conversion-Test", + "ALL/Web/Acquisition/Phones/BirthdayDiscount/Conversion-Control", + "ALL/Web/Acquisition/Phones/BirthdayDiscount/Conversion-Test", + "ALL/Web/Acquisition/Phones/IPhone16/Conversion-Control", + "ALL/Web/Acquisition/Phones/IPhone16/Conversion-Test", + "ALL/Web/Acquisition/Phones/SmartphoneXYZ/Conversion-Control", + "ALL/Web/Acquisition/Phones/SmartphoneXYZ/Conversion-Test", + "ALL/Web/Acquisition/Phones/WalkieTalkie/Conversion-Control", + "ALL/Web/Acquisition/Phones/WalkieTalkie/Conversion-Test", + "ALL/Web/Acquisition/Phones/AppleIPhone15128GB", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB", + "ALL/Web/Acquisition/Phones/BirthdayDiscount", + "ALL/Web/Acquisition/Phones/IPhone16", + "ALL/Web/Acquisition/Phones/SmartphoneXYZ", + "ALL/Web/Acquisition/Phones/WalkieTalkie", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition", + "ALL/Web", + "ALL" + ], + "labels": [ + "Conversion-Control", + "Conversion-Test", + "Conversion-Control", + "Conversion-Test", + "Conversion-Control", + "Conversion-Test", + "Conversion-Control", + "Conversion-Test", + "Conversion-Control", + "Conversion-Test", + "Conversion-Control", + "Conversion-Test", + "Conversion-Control", + "Conversion-Test", + "Conversion-Control", + "Conversion-Test", + "AppleIPhone15128GB", + "AppleIPhone1564GB", + "AppleIPhone16Pro1TB", + "AppleIPhone16Pro256GB", + "BirthdayDiscount", + "IPhone16", + "SmartphoneXYZ", + "WalkieTalkie", + "Phones", + "Acquisition", + "Web", + "ALL" + ], + "marker": { + "coloraxis": "coloraxis", + "colors": { + "bdata": "CSCOaHf6rD8yskyjZYusPz1pG1hkeag/J6+hvIbymj+XlpaWlpa2P3a2TLtFQ7A/whT5rBtMsT8YhmEYhmG4PwAAAAAAAAAAAAAAAAAAAAAg+IEf+IGfPxZf8RVf8aU/xmmX10Gfqj/xIanirhCvP4FVUObonaw/RhdddNFFpz9U7oZQk8GsP6jd9/PeUag/Xfc/i6elsD989C3KijKyPwAAAAAAAAAAcM2FzO6Xoz+oAGf080GtP9imIxbrdqw/rNcxWu7arD+s1zFa7tqsP6zXMVru2qw/rNcxWu7arD8=", + "dtype": "f8" + } + }, + "name": "", + "parents": [ + "ALL/Web/Acquisition/Phones/AppleIPhone15128GB", + "ALL/Web/Acquisition/Phones/AppleIPhone15128GB", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB", + "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB", + "ALL/Web/Acquisition/Phones/BirthdayDiscount", + "ALL/Web/Acquisition/Phones/BirthdayDiscount", + "ALL/Web/Acquisition/Phones/IPhone16", + "ALL/Web/Acquisition/Phones/IPhone16", + "ALL/Web/Acquisition/Phones/SmartphoneXYZ", + "ALL/Web/Acquisition/Phones/SmartphoneXYZ", + "ALL/Web/Acquisition/Phones/WalkieTalkie", + "ALL/Web/Acquisition/Phones/WalkieTalkie", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition", + "ALL/Web", + "ALL", + "" + ], + "textinfo": "label+value+percent parent+percent root", + "type": "treemap", + "values": { + "bdata": "AAAAAAAGrUAAAAAAAHquQAAAAAAAaKVAAAAAAACAQ0AAAAAAAMBrQAAAAAAA0KpAAAAAAADAY0AAAAAAAAA3QAAAAAAAAD5AAAAAAACATUAAAAAAAMBgQAAAAAAAYGtAAAAAAAC0oUAAAAAAANCpQAAAAAAAgJhAAAAAAAAAR0AAAAAAAMC9QAAAAAAAtqVAAAAAAACMrEAAAAAAAKBmQAAAAAAAQFZAAAAAAAAQdkAAAAAAAMK1QAAAAAAAOJlAAAAAAABY1UAAAAAAAFjVQAAAAAAAWNVAAAAAAABY1UA=", + "dtype": "f8" + } + } + ], + "layout": { + "coloraxis": { + "colorbar": { + "title": { + "text": "ConversionRate" + } + }, + "colorscale": [ + [ + 0, + "rgb(5,48,97)" + ], + [ + 0.1, + "rgb(33,102,172)" + ], + [ + 0.2, + "rgb(67,147,195)" + ], + [ + 0.3, + "rgb(146,197,222)" + ], + [ + 0.4, + "rgb(209,229,240)" + ], + [ + 0.5, + "rgb(247,247,247)" + ], + [ + 0.6, + "rgb(253,219,199)" + ], + [ + 0.7, + "rgb(244,165,130)" + ], + [ + 0.8, + "rgb(214,96,77)" + ], + [ + 0.9, + "rgb(178,24,43)" + ], + [ + 1, + "rgb(103,0,31)" + ] + ] + }, + "height": 640, + "legend": { + "tracegroupgap": 0 + }, + "margin": { + "b": 25, + "l": 25, + "r": 25, + "t": 50 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[BIZ] Conversion rate treemap" + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ChannelIssueGroupNameExperimentGroupNegativesPositivesCountConversionRateStdErr
0WebAcquisitionPhonesAppleIPhone15128GBConversion-Control331719937150.0565980.003897
1WebAcquisitionPhonesAppleIPhone15128GBConversion-Test348920639010.0557510.003775
2WebAcquisitionPhonesAppleIPhone1564GBConversion-Control249012527400.0478010.004172
3WebAcquisitionPhonesAppleIPhone1564GBConversion-Test371390.0263160.025967
4WebAcquisitionPhonesAppleIPhone16Pro1TBConversion-Control186182220.0882350.019859
5WebAcquisitionPhonesAppleIPhone16Pro1TBConversion-Test302220534320.0635260.004294
6WebAcquisitionPhonesAppleIPhone16Pro256GBConversion-Control138101580.0675680.020632
7WebAcquisitionPhonesAppleIPhone16Pro256GBConversion-Test192230.0952380.064056
8WebAcquisitionPhonesBirthdayDiscountConversion-Control300300.0000000.000000
9WebAcquisitionPhonesBirthdayDiscountConversion-Test590590.0000000.000000
10WebAcquisitionPhonesIPhone16Conversion-Control12641340.0307690.015146
11WebAcquisitionPhonesIPhone16Conversion-Test20192190.0428570.013976
12WebAcquisitionPhonesSmartphoneXYZConversion-Control204211222660.0519960.004784
13WebAcquisitionPhonesSmartphoneXYZConversion-Test292618933040.0606740.004277
14WebAcquisitionPhonesWalkieTalkieConversion-Control14028315680.0558920.005961
15WebAcquisitionPhonesWalkieTalkieConversion-Test422460.0454550.031402
\n", + "
" + ], + "text/plain": [ + " Channel Issue Group Name ExperimentGroup \\\n", + "0 Web Acquisition Phones AppleIPhone15128GB Conversion-Control \n", + "1 Web Acquisition Phones AppleIPhone15128GB Conversion-Test \n", + "2 Web Acquisition Phones AppleIPhone1564GB Conversion-Control \n", + "3 Web Acquisition Phones AppleIPhone1564GB Conversion-Test \n", + "4 Web Acquisition Phones AppleIPhone16Pro1TB Conversion-Control \n", + "5 Web Acquisition Phones AppleIPhone16Pro1TB Conversion-Test \n", + "6 Web Acquisition Phones AppleIPhone16Pro256GB Conversion-Control \n", + "7 Web Acquisition Phones AppleIPhone16Pro256GB Conversion-Test \n", + "8 Web Acquisition Phones BirthdayDiscount Conversion-Control \n", + "9 Web Acquisition Phones BirthdayDiscount Conversion-Test \n", + "10 Web Acquisition Phones IPhone16 Conversion-Control \n", + "11 Web Acquisition Phones IPhone16 Conversion-Test \n", + "12 Web Acquisition Phones SmartphoneXYZ Conversion-Control \n", + "13 Web Acquisition Phones SmartphoneXYZ Conversion-Test \n", + "14 Web Acquisition Phones WalkieTalkie Conversion-Control \n", + "15 Web Acquisition Phones WalkieTalkie Conversion-Test \n", + "\n", + " Negatives Positives Count ConversionRate StdErr \n", + "0 3317 199 3715 0.056598 0.003897 \n", + "1 3489 206 3901 0.055751 0.003775 \n", + "2 2490 125 2740 0.047801 0.004172 \n", + "3 37 1 39 0.026316 0.025967 \n", + "4 186 18 222 0.088235 0.019859 \n", + "5 3022 205 3432 0.063526 0.004294 \n", + "6 138 10 158 0.067568 0.020632 \n", + "7 19 2 23 0.095238 0.064056 \n", + "8 30 0 30 0.000000 0.000000 \n", + "9 59 0 59 0.000000 0.000000 \n", + "10 126 4 134 0.030769 0.015146 \n", + "11 201 9 219 0.042857 0.013976 \n", + "12 2042 112 2266 0.051996 0.004784 \n", + "13 2926 189 3304 0.060674 0.004277 \n", + "14 1402 83 1568 0.055892 0.005961 \n", + "15 42 2 46 0.045455 0.031402 " + ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "treemap_group_by = [\"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", + "\n", + "treemap_data = (\n", + " ih_analysis.group_by(treemap_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\")\n", + " )\n", + " .with_columns(\n", + " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"ConversionRate\")]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", + " (\n", + " (\n", + " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", + " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", + " )\n", + " ** 0.5\n", + " )\n", + " ).alias(\"StdErr\")\n", + " ]\n", + " )\n", + " .sort(treemap_group_by, descending=False)\n", + " .collect()\n", + " )\n", + "\n", + "treemap_data = treemap_data.to_pandas()\n", + "\n", + "fig = px.treemap(treemap_data, path=[px.Constant(\"ALL\")] + treemap_group_by, values='Count',\n", + " color=\"ConversionRate\",\n", + " color_continuous_scale=px.colors.sequential.RdBu_r,\n", + " title=\"[BIZ] Conversion rate treemap\",\n", + " hover_data=['StdErr', 'Positives', 'Negatives'],\n", + " height=640,\n", + " )\n", + "fig.update_traces(textinfo=\"label+value+percent parent+percent root\")\n", + "fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))\n", + "\n", + "fig.show()\n", + "treemap_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Detailed line/bar plot." + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ] + ], + "error_y": { + "array": { + "bdata": "RRJbW54SdT+CSTTWCbmAPw==", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", + "legendgroup": "Conversion-Control", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Control", + "offsetgroup": "Conversion-Control", + "orientation": "v", + "showlegend": true, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-12-12T00:00:00.000", + "2024-12-13T00:00:00.000" + ], + "xaxis": "x", + "y": { + "bdata": "GYej9nSiqz86pdV8uO2qPw==", + "dtype": "f8" + }, + "yaxis": "y" + }, + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ] + ], + "error_y": { + "array": { + "bdata": "GpTHPQzgdT94Iz5HkXaBPw==", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", + "legendgroup": "Conversion-Test", + "marker": { + "color": "#EF553B", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Test", + "offsetgroup": "Conversion-Test", + "orientation": "v", + "showlegend": true, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-12-12T00:00:00.000", + "2024-12-13T00:00:00.000" + ], + "xaxis": "x", + "y": { + "bdata": "wF7HtoEqrj+BxR9Pr0quPw==", + "dtype": "f8" + }, + "yaxis": "y" + } + ], + "layout": { + "annotations": [ + { + "font": {}, + "showarrow": false, + "text": "Web", + "textangle": 90, + "x": 0.98, + "xanchor": "left", + "xref": "paper", + "y": 0.5, + "yanchor": "middle", + "yref": "paper" + } + ], + "barmode": "group", + "height": 640, + "hovermode": "x unified", + "legend": { + "title": { + "text": "ExperimentGroup" + }, + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[CONV] Daily Conversion Rate with 95% confidence interval" + }, + "updatemenus": [ + { + "buttons": [ + { + "args": [ + "type", + "bar" + ], + "label": "Bar", + "method": "restyle" + }, + { + "args": [ + "type", + "line" + ], + "label": "Line", + "method": "restyle" + } + ], + "direction": "down", + "showactive": true + } + ], + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 0.98 + ], + "tickfont": { + "size": 10 + }, + "title": { + "text": "Day" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "tickformat": ",.2%", + "title": { + "text": "Conversion Rate" + } + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DayChannelExperimentGroupNegativesPositivesCountConversionRateCI
02024-12-12WebConversion-Control701140078110.0539740.005145
12024-12-12WebConversion-Test702844079080.0589180.005341
22024-12-13WebConversion-Control272015130220.0525950.008165
32024-12-13WebConversion-Test276717431150.0591640.008527
\n", + "
" + ], + "text/plain": [ + " Day Channel ExperimentGroup Negatives Positives Count \\\n", + "0 2024-12-12 Web Conversion-Control 7011 400 7811 \n", + "1 2024-12-12 Web Conversion-Test 7028 440 7908 \n", + "2 2024-12-13 Web Conversion-Control 2720 151 3022 \n", + "3 2024-12-13 Web Conversion-Test 2767 174 3115 \n", + "\n", + " ConversionRate CI \n", + "0 0.053974 0.005145 \n", + "1 0.058918 0.005341 \n", + "2 0.052595 0.008165 \n", + "3 0.059164 0.008527 " + ] + }, + "execution_count": 76, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", + "\n", + "line_data = (\n", + " ih_analysis.group_by(line_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\")\n", + " )\n", + " .with_columns(\n", + " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"ConversionRate\")]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", + " (\n", + " ((\n", + " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", + " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", + " )\n", + " ** 0.5) * 1.96\n", + " )\n", + " ).alias(\"CI\")\n", + " ]\n", + " )\n", + " .sort(line_group_by, descending=False)\n", + " .collect()\n", + " )\n", + "\n", + "line_data = line_data.to_pandas()\n", + "\n", + "if len(line_data[\"Day\"].unique()) < 30:\n", + " fig = px.bar(line_data,\n", + " x=\"Day\",\n", + " y=\"ConversionRate\",\n", + " color=\"ExperimentGroup\",\n", + " error_y='CI',\n", + " facet_row=\"Channel\",\n", + " barmode=\"group\",\n", + " title=\"[CONV] Daily Conversion Rate with 95% confidence interval\",\n", + " custom_data=[\"ExperimentGroup\"]\n", + " )\n", + " fig.update_layout(\n", + " updatemenus=[\n", + " dict(\n", + " buttons=list([\n", + " dict(\n", + " args=[\"type\", \"bar\"],\n", + " label=\"Bar\",\n", + " method=\"restyle\"\n", + " ),\n", + " dict(\n", + " args=[\"type\", \"line\"],\n", + " label=\"Line\",\n", + " method=\"restyle\"\n", + " )\n", + " ]),\n", + " direction=\"down\",\n", + " showactive=True,\n", + " ),\n", + " ]\n", + " )\n", + "else:\n", + " fig = px.line(\n", + " line_data,\n", + " x=\"Day\",\n", + " y=\"ConversionRate\",\n", + " color=\"ExperimentGroup\",\n", + " title=\"[CONV] Daily Conversion Rate\",\n", + " acet_row=\"Channel\",\n", + " custom_data=[\"ExperimentGroup\"]\n", + " )\n", + "\n", + "fig.update_xaxes(tickfont=dict(size=10))\n", + "fig.update_yaxes(tickformat=',.2%')\n", + "yaxis_names = ['yaxis'] + [axis_name for axis_name in fig.layout._subplotid_props if 'yaxis' in axis_name]\n", + "yaxis_layout_dict = {yaxis_name + \"_tickformat\": ',.2%' for yaxis_name in yaxis_names}\n", + "fig.update_layout(yaxis_layout_dict)\n", + "height = max(640, 300 * len(line_data[\"Channel\"].unique()))\n", + "fig.update_layout(\n", + " xaxis_title=\"Day\",\n", + " yaxis_title=\"Conversion Rate\",\n", + " hovermode=\"x unified\",\n", + " height=height\n", + " )\n", + "fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[1]))\n", + "fig = fig.update_traces(hovertemplate=\"Day\" + ' : %{x}' + '
' +\n", + " \"Experiment Group\" + ' : %{customdata[0]}' + '
' +\n", + " \"Conversion Rate\" + ' : %{y:.2%}' + '')\n", + "\n", + "fig.show()\n", + "line_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Engagement rates (CTR)" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "delta": { + "reference": 0.25, + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0, + 1 + ], + "y": [ + 0.575, + 1 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75, + "value": 0.25 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "Web Conversion-Control" + }, + "type": "indicator", + "value": 0.2543279517603579 + }, + { + "delta": { + "reference": 0.25, + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0, + 1 + ], + "y": [ + 0, + 0.425 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "bar": { + "color": "#EC9B00" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75, + "value": 0.25 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "Web Conversion-Test" + }, + "type": "indicator", + "value": 0.24440388125660487 + } + ], + "layout": { + "autosize": true, + "height": 540, + "margin": { + "b": 10, + "l": 10, + "r": 10, + "t": 120 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[ENG] Click-through rates (Channel/Model Type)" + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ChannelExperimentGroupNegativesPositivesCountCTRStdErrNameCName
0WebConversion-Control76672615128970.2543280.004295Web Conversion-ControlWeb_Conversion-Control
1WebConversion-Test78652544129530.2444040.004212Web Conversion-TestWeb_Conversion-Test
\n", + "
" + ], + "text/plain": [ + " Channel ExperimentGroup Negatives Positives Count CTR \\\n", + "0 Web Conversion-Control 7667 2615 12897 0.254328 \n", + "1 Web Conversion-Test 7865 2544 12953 0.244404 \n", + "\n", + " StdErr Name CName \n", + "0 0.004295 Web Conversion-Control Web_Conversion-Control \n", + "1 0.004212 Web Conversion-Test Web_Conversion-Test " + ] + }, + "execution_count": 77, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "positive_model_response = [\"Clicked\"]\n", + "all_model_response = [\"Impression\", \"Pending\"]\n", + "group_by = [\"Day\", \"Month\", \"Year\", \"Quarter\", \"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", + "\n", + "ih_analysis = (\n", + " ih.filter(\n", + " (pl.col(\"Outcome\").is_in(all_model_response + positive_model_response))\n", + " )\n", + " .with_columns([\n", + " pl.when(pl.col('Outcome').is_in(positive_model_response)).\n", + " then(1).otherwise(0).alias('Outcome_Binary')\n", + " ])\n", + " .group_by(group_by)\n", + " .agg([\n", + " pl.len().alias('Count'),\n", + " pl.sum(\"Outcome_Binary\").alias(\"Positives\")\n", + " ])\n", + " .with_columns([\n", + " (pl.col(\"Count\") - (2 * pl.col(\"Positives\"))).alias(\"Negatives\")\n", + " ])\n", + " )\n", + "gauge_group_by = [\"Channel\", \"ExperimentGroup\"]\n", + "reference = {'Web_Conversion-Test' : 0.25, 'Web_Conversion-Control' : 0.25}\n", + "gauge_data = (\n", + " ih_analysis.group_by(gauge_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\")\n", + " )\n", + " .with_columns(\n", + " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"CTR\")]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", + " (\n", + " (\n", + " (pl.col(\"CTR\") * (1 - pl.col(\"CTR\")))\n", + " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", + " )\n", + " ** 0.5\n", + " )\n", + " ).alias(\"StdErr\")\n", + " ]\n", + " )\n", + " .sort(gauge_group_by, descending=False)\n", + " .collect()\n", + " )\n", + "\n", + "gauge_data = gauge_data.to_pandas()\n", + "\n", + "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", + "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", + "\n", + "gauge_data['Name'] = gauge_data[gauge_group_by].apply(lambda r: ' '.join(r.values.astype(str)), axis=1)\n", + "gauge_data['CName'] = gauge_data[gauge_group_by].apply(lambda r: '_'.join(r.values.astype(str)), axis=1)\n", + "\n", + "fig = make_subplots(rows=rows,\n", + " cols=cols,\n", + " specs=[[{\"type\": \"indicator\"} for c in range(cols)] for t in range(rows)]\n", + " )\n", + "fig.update_layout(\n", + " height=270 * rows,\n", + " autosize=True,\n", + " title='[ENG] Click-through rates (Channel/Model Type)',\n", + " margin=dict(b=10, t=120, l=10, r=10))\n", + "\n", + "for index, row in gauge_data.iterrows():\n", + " ref_value = reference.get(row['CName'], None)\n", + " gauge = {\n", + " 'axis': {'tickformat': ',.2%'},\n", + " 'threshold': {\n", + " 'line': {'color': \"red\", 'width': 2},\n", + " 'thickness': 0.75,\n", + " 'value': ref_value\n", + " }\n", + " }\n", + " if ref_value:\n", + " if row['CTR'] < ref_value:\n", + " gauge = {\n", + " 'axis': {'tickformat': ',.2%'},\n", + " 'bar': {'color': '#EC5300' if row['CTR'] < (0.75 * ref_value) else '#EC9B00'},\n", + " 'threshold': {\n", + " 'line': {'color': \"red\", 'width': 2},\n", + " 'thickness': 0.75,\n", + " 'value': ref_value\n", + " }\n", + " }\n", + "\n", + " trace1 = go.Indicator(mode=\"gauge+number+delta\",\n", + " number={'valueformat': \",.2%\"},\n", + " value=row['CTR'],\n", + " delta={'reference': ref_value, 'valueformat': \",.2%\"},\n", + " title={'text': row['Name']},\n", + " gauge=gauge,\n", + " )\n", + " r, c = divmod(index, cols)\n", + " fig.add_trace(\n", + " trace1,\n", + " row=(r + 1), col=(c + 1)\n", + " )\n", + "fig.show()\n", + "gauge_data" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ] + ], + "error_y": { + "array": { + "bdata": "RF5y/PpbhD+1jAuA5jKQPw==", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", + "legendgroup": "Conversion-Control", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Control", + "offsetgroup": "Conversion-Control", + "orientation": "v", + "showlegend": true, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-12-12T00:00:00.000", + "2024-12-13T00:00:00.000" + ], + "xaxis": "x", + "y": { + "bdata": "NxhxcnVo0D8kX1PvnODPPw==", + "dtype": "f8" + }, + "yaxis": "y" + }, + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ] + ], + "error_y": { + "array": { + "bdata": "oiZ4fdQFhD+a6o3xiY2PPw==", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", + "legendgroup": "Conversion-Test", + "marker": { + "color": "#EF553B", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Test", + "offsetgroup": "Conversion-Test", + "orientation": "v", + "showlegend": true, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-12-12T00:00:00.000", + "2024-12-13T00:00:00.000" + ], + "xaxis": "x", + "y": { + "bdata": "vxNQKE6Szz+WQKIYiY3OPw==", + "dtype": "f8" + }, + "yaxis": "y" + } + ], + "layout": { + "annotations": [ + { + "font": {}, + "showarrow": false, + "text": "Web", + "textangle": 90, + "x": 0.98, + "xanchor": "left", + "xref": "paper", + "y": 0.5, + "yanchor": "middle", + "yref": "paper" + } + ], + "barmode": "group", + "height": 640, + "hovermode": "x unified", + "legend": { + "title": { + "text": "ExperimentGroup" + }, + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[ENG] Daily Click-through Rate with 95% confidence interval" + }, + "updatemenus": [ + { + "buttons": [ + { + "args": [ + "type", + "bar" + ], + "label": "Bar", + "method": "restyle" + }, + { + "args": [ + "type", + "line" + ], + "label": "Line", + "method": "restyle" + } + ], + "direction": "down", + "showactive": true + } + ], + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 0.98 + ], + "tickfont": { + "size": 10 + }, + "title": { + "text": "Day" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "tickformat": ",.2%", + "title": { + "text": "CTR" + } + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DayChannelExperimentGroupNegativesPositivesCountCTRCI
02024-12-12WebConversion-Control5511190093110.2563760.009941
12024-12-12WebConversion-Test5626184293100.2466520.009777
22024-12-13WebConversion-Control215671535860.2490420.015819
32024-12-13WebConversion-Test223970236430.2386940.015407
\n", + "
" + ], + "text/plain": [ + " Day Channel ExperimentGroup Negatives Positives Count \\\n", + "0 2024-12-12 Web Conversion-Control 5511 1900 9311 \n", + "1 2024-12-12 Web Conversion-Test 5626 1842 9310 \n", + "2 2024-12-13 Web Conversion-Control 2156 715 3586 \n", + "3 2024-12-13 Web Conversion-Test 2239 702 3643 \n", + "\n", + " CTR CI \n", + "0 0.256376 0.009941 \n", + "1 0.246652 0.009777 \n", + "2 0.249042 0.015819 \n", + "3 0.238694 0.015407 " + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", + "\n", + "line_data = (\n", + " ih_analysis.group_by(line_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\")\n", + " )\n", + " .with_columns(\n", + " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"CTR\")]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", + " (\n", + " ((\n", + " (pl.col(\"CTR\") * (1 - pl.col(\"CTR\")))\n", + " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", + " )\n", + " ** 0.5) * 1.96\n", + " )\n", + " ).alias(\"CI\")\n", + " ]\n", + " )\n", + " .sort(line_group_by, descending=False)\n", + " .collect()\n", + " )\n", + "\n", + "line_data = line_data.to_pandas()\n", + "\n", + "if len(line_data[\"Day\"].unique()) < 30:\n", + " fig = px.bar(line_data,\n", + " x=\"Day\",\n", + " y=\"CTR\",\n", + " color=\"ExperimentGroup\",\n", + " error_y='CI',\n", + " facet_row=\"Channel\",\n", + " barmode=\"group\",\n", + " title=\"[ENG] Daily Click-through Rate with 95% confidence interval\",\n", + " custom_data=[\"ExperimentGroup\"]\n", + " )\n", + " fig.update_layout(\n", + " updatemenus=[\n", + " dict(\n", + " buttons=list([\n", + " dict(\n", + " args=[\"type\", \"bar\"],\n", + " label=\"Bar\",\n", + " method=\"restyle\"\n", + " ),\n", + " dict(\n", + " args=[\"type\", \"line\"],\n", + " label=\"Line\",\n", + " method=\"restyle\"\n", + " )\n", + " ]),\n", + " direction=\"down\",\n", + " showactive=True,\n", + " ),\n", + " ]\n", + " )\n", + "else:\n", + " fig = px.line(\n", + " line_data,\n", + " x=\"Day\",\n", + " y=\"CTR\",\n", + " color=\"ExperimentGroup\",\n", + " title=\"[ENG] Daily Click-through Rate\",\n", + " acet_row=\"Channel\",\n", + " custom_data=[\"ExperimentGroup\"]\n", + " )\n", + "\n", + "fig.update_xaxes(tickfont=dict(size=10))\n", + "fig.update_yaxes(tickformat=',.2%')\n", + "yaxis_names = ['yaxis'] + [axis_name for axis_name in fig.layout._subplotid_props if 'yaxis' in axis_name]\n", + "yaxis_layout_dict = {yaxis_name + \"_tickformat\": ',.2%' for yaxis_name in yaxis_names}\n", + "fig.update_layout(yaxis_layout_dict)\n", + "height = max(640, 300 * len(line_data[\"Channel\"].unique()))\n", + "fig.update_layout(\n", + " xaxis_title=\"Day\",\n", + " yaxis_title=\"CTR\",\n", + " hovermode=\"x unified\",\n", + " height=height\n", + " )\n", + "fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[1]))\n", + "fig = fig.update_traces(hovertemplate=\"Day\" + ' : %{x}' + '
' +\n", + " \"Experiment Group\" + ' : %{customdata[0]}' + '
' +\n", + " \"CTR\" + ' : %{y:.2%}' + '')\n", + "\n", + "fig.show()\n", + "line_data" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From d7e3585860f339cd3f88b70692ea8636b46a9d36 Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Fri, 13 Dec 2024 16:56:40 +0100 Subject: [PATCH 07/21] Prep for data gen --- .../ih/Conversion_Modeling_Reporting.ipynb | 9524 +---------------- examples/ih/Example_IH_Analysis.ipynb | 58 +- 2 files changed, 68 insertions(+), 9514 deletions(-) diff --git a/examples/ih/Conversion_Modeling_Reporting.ipynb b/examples/ih/Conversion_Modeling_Reporting.ipynb index aefc4ab7..e76804ba 100644 --- a/examples/ih/Conversion_Modeling_Reporting.ipynb +++ b/examples/ih/Conversion_Modeling_Reporting.ipynb @@ -2,3902 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - " \n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import polars as pl\n", "from pdstools import read_ds_export\n", @@ -3915,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -3971,116 +78,32 @@ " return fields_new" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "TODO: see if we can generate such data rather than shipping it" + ] + }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "ih = read_ds_export(\"Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\", path=\"../../data\")" + "ih = read_ds_export(\"Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\", path=\".\")\n", + "\n", + "# we really only need a few columns\n", + "# Outcome outcomes: Conversionm, Impression, Pending\n", + "ih = ih.select([\"pyOutcome\", \"pxOutcomeTime\", \"pyChannel\", \"pyIssue\", \"pyGroup\", \"pyName\", \"ExperimentGroup\"])\n", + "ih.head().collect()" ] }, { "cell_type": "code", - "execution_count": 70, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Schema([('ControlGroupValidityStart', String),\n", - " ('Stage', String),\n", - " ('Value', Float64),\n", - " ('Behaviour', String),\n", - " ('PropensitySource', String),\n", - " ('MktType', String),\n", - " ('ExternalAudienceId', String),\n", - " ('Response', String),\n", - " ('ControlGroupValidityEnd', String),\n", - " ('ReferrerUrl', String),\n", - " ('Propensity', Float64),\n", - " ('WorkID', String),\n", - " ('ActionContext', String),\n", - " ('Name', String),\n", - " ('Treatment', String),\n", - " ('BundleName', String),\n", - " ('Division', String),\n", - " ('ExternalID', String),\n", - " ('CustomerID', String),\n", - " ('PartitionKey', Int64),\n", - " ('PyCategory', String),\n", - " ('Category', String),\n", - " ('ExperimentGroup', String),\n", - " ('Latitude', Float64),\n", - " ('MaxBudget', String),\n", - " ('Group', String),\n", - " ('ChannelSubGroup', String),\n", - " ('Outcome', String),\n", - " ('Reason', String),\n", - " ('Rank', Int64),\n", - " ('OutcomeTime', String),\n", - " ('SubjectType', String),\n", - " ('Application', String),\n", - " ('ModelClass', String),\n", - " ('SubCategory', String),\n", - " ('IPAddress', String),\n", - " ('ModelControlGroup', String),\n", - " ('CustomerSegment', String),\n", - " ('UserAgent', String),\n", - " ('Interaction', String),\n", - " ('StreamPartition', String),\n", - " ('ChannelGroup', String),\n", - " ('Step', String),\n", - " ('Label', String),\n", - " ('Channel', String),\n", - " ('Fulfilled', String),\n", - " ('URI', String),\n", - " ('Issue', String),\n", - " ('Longitude', Float64),\n", - " ('OutcomeWeight', Float64),\n", - " ('Priority', Float64),\n", - " ('Operator', String),\n", - " ('SubjectID', String),\n", - " ('Strategy', String),\n", - " ('Organization', String),\n", - " ('Unit', String),\n", - " ('BundleHead', Boolean),\n", - " ('DecisionTime', String),\n", - " ('UpdateDateTime', Int64),\n", - " ('StreamPosition', String),\n", - " ('GroupID', String),\n", - " ('ISFactID', String),\n", - " ('TargetBudget', Float64),\n", - " ('InteractionID', String),\n", - " ('Revenue', Float64),\n", - " ('ApplicationVersion', String),\n", - " ('DeviceType', String),\n", - " ('AggregateCount', Int64),\n", - " ('Component', String),\n", - " ('PyWorkID', String),\n", - " ('MktValue', String),\n", - " ('BudgetedCost', Float64),\n", - " ('CustomerSubSegment', String),\n", - " ('Cost', Float64),\n", - " ('ConversionEventID', String),\n", - " ('FinalPropensity', Float64),\n", - " ('Weight', Float64),\n", - " ('EvaluationCriteria', String),\n", - " ('PlacementType', String),\n", - " ('Direction', String),\n", - " ('Journey', String),\n", - " ('Utm_medium', String),\n", - " ('FactID', String),\n", - " ('CampaignCode', String),\n", - " ('PyRevenue', String),\n", - " ('InternalTags', String)])" - ] - }, - "execution_count": 70, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dframe_columns = ih.collect_schema().names()\n", "cols = capitalize(dframe_columns)\n", @@ -4097,48 +120,9 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (9, 92)
statisticControlGroupValidityStartStageValueBehaviourPropensitySourceMktTypeExternalAudienceIdResponseControlGroupValidityEndReferrerUrlPropensityWorkIDActionContextNameTreatmentBundleNameDivisionExternalIDCustomerIDPartitionKeyPyCategoryCategoryExperimentGroupLatitudeMaxBudgetGroupChannelSubGroupOutcomeReasonRankOutcomeTimeSubjectTypeApplicationModelClassSubCategoryIPAddress…OrganizationUnitBundleHeadDecisionTimeUpdateDateTimeStreamPositionGroupIDISFactIDTargetBudgetInteractionIDRevenueApplicationVersionDeviceTypeAggregateCountComponentPyWorkIDMktValueBudgetedCostCustomerSubSegmentCostConversionEventIDFinalPropensityWeightEvaluationCriteriaPlacementTypeDirectionJourneyUtm_mediumFactIDCampaignCodePyRevenueInternalTagsOutcomeDateTimeDayMonthYearQuarter
strstrstrf64strstrstrstrstrstrstrf64strstrstrstrstrstrstrstrf64strstrstrf64strstrstrstrstrf64strstrstrstrstrstr…strstrf64strf64strstrstrf64strf64strstrf64strstrstrf64strf64strf64f64strstrstrstrstrstrstrstrstrstrstrstrstrstr
"count""304784""304784"304784.0"304784""304784""304784""304784""304784""304784""304784"304784.0"304784""304784""304784""304784""304784""304784""304784""304784"304784.0"304784""304784""52854"304784.0"304784""304784""304784""304784""304784"304784.0"304784""304784""304784""304784""304784""304784"…"304784""304784"304784.0"304784"304784.0"304784""304784""304784"304784.0"304784"304784.0"304784""304784"304784.0"304784""304784""304784"304784.0"304784"304784.0"304784"304784.0304784.0"304784""304784""304784""304784""304784""304784""304784""304784""304784""304784""304784""304784""304784""304784"
"null_count""0""0"0.0"0""0""0""0""0""0""0"0.0"0""0""0""0""0""0""0""0"0.0"0""0""251930"0.0"0""0""0""0""0"0.0"0""0""0""0""0""0"…"0""0"0.0"0"0.0"0""0""0"0.0"0"0.0"0""0"0.0"0""0""0"0.0"0"0.0"0"0.00.0"0""0""0""0""0""0""0""0""0""0""0""0""0""0"
"mean"nullnull0.0nullnullnullnullnullnullnull0.146205nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull2.501184nullnullnullnullnullnull…nullnull0.0null1.7337e12nullnullnull0.0null0.0nullnull1.0nullnullnull0.0null0.0null0.1971370.0nullnullnullnullnullnullnullnullnull"2024-12-09 00:35:11.766000""2024-12-08 11:34:38.282000"nullnullnull
"std"nullnull0.0nullnullnullnullnullnullnull0.195219nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull3.501869nullnullnullnullnullnull…nullnullnullnull2.7342e8nullnullnull0.0null0.0nullnull0.0nullnullnull0.0null0.0null0.2823090.0nullnullnullnullnullnullnullnullnullnullnullnullnullnull
"min"""""0.0""""""""""""""0.0"""""AppleIPhone15128GB"""""""""""0.0""""""0.0"0""""""Clicked"""1.0"20241203T115401.223 GMT""Data-CDHCaptureResponse-SR""ConversionModel"""""""…""""0.0"20241203T112232.703 GMT"1.7332e12"0"""""0.0"-1808289819383917128"0.0"01.01.01"""1.0"ConversionAttribution"""""0.0""0.0""0.00.0"""""""""""0""""""""2024-12-03 11:54:01.223000""2024-12-03""2024-12""2024""2024_Q4"
"25%"nullnull0.0nullnullnullnullnullnullnull0.0nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull1.0nullnullnullnullnullnull…nullnullnullnull1.7334e12nullnullnull0.0null0.0nullnull1.0nullnullnull0.0null0.0null0.00.0nullnullnullnullnullnullnullnullnull"2024-12-05 19:26:36.015000""2024-12-05"nullnullnull
"50%"nullnull0.0nullnullnullnullnullnullnull0.090087nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull1.0nullnullnullnullnullnull…nullnullnullnull1.7335e12nullnullnull0.0null0.0nullnull1.0nullnullnull0.0null0.0null0.0979490.0nullnullnullnullnullnullnullnullnull"2024-12-06 09:18:47.649000""2024-12-06"nullnullnull
"75%"nullnull0.0nullnullnullnullnullnullnull0.196034nullnullnullnullnullnullnullnull0.0nullnullnull0.0nullnullnullnullnull1.0nullnullnullnullnullnull…nullnullnullnull1.7340e12nullnullnull0.0null0.0nullnull1.0nullnullnull0.0null0.0null0.2026590.0nullnullnullnullnullnullnullnullnull"2024-12-12 07:20:04.558000""2024-12-12"nullnullnull
"max"""""0.0"""""PropensityPriority"""""""""1.0"""Customer""WalkieTalkie""WalkieHero""""""""D-ZZZM3AGC"0.0"""""Conversion-Test"0.0"0""Phones""""Product conversion""Control"32.0"20241213T090853.607 GMT""PegaMKT-Data-Customer""ConversionModel""Engagement"""""…""""0.0"20241213T090853.333 GMT"1.7341e12"999"""""0.0"8789841886479301898"0.0"01.01.01""desktop"1.0"Setting Rank""""NBAHealth_PropensityPriority"0.0""0.0"iPhone16Pro"1.00.0"Default""Hero""Inbound""""""0""""""""2024-12-13 09:08:53.607000""2024-12-13""2024-12""2024""2024_Q4"
" - ], - "text/plain": [ - "shape: (9, 92)\n", - "┌────────────┬────────────────┬────────┬──────────┬───┬───────────────┬─────────┬────────┬─────────┐\n", - "│ statistic ┆ ControlGroupVa ┆ Stage ┆ Value ┆ … ┆ Day ┆ Month ┆ Year ┆ Quarter │\n", - "│ --- ┆ lidityStart ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ --- ┆ str ┆ f64 ┆ ┆ str ┆ str ┆ str ┆ str │\n", - "│ ┆ str ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", - "╞════════════╪════════════════╪════════╪══════════╪═══╪═══════════════╪═════════╪════════╪═════════╡\n", - "│ count ┆ 304784 ┆ 304784 ┆ 304784.0 ┆ … ┆ 304784 ┆ 304784 ┆ 304784 ┆ 304784 │\n", - "│ null_count ┆ 0 ┆ 0 ┆ 0.0 ┆ … ┆ 0 ┆ 0 ┆ 0 ┆ 0 │\n", - "│ mean ┆ null ┆ null ┆ 0.0 ┆ … ┆ 2024-12-08 ┆ null ┆ null ┆ null │\n", - "│ ┆ ┆ ┆ ┆ ┆ 11:34:38.2820 ┆ ┆ ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ 00 ┆ ┆ ┆ │\n", - "│ std ┆ null ┆ null ┆ 0.0 ┆ … ┆ null ┆ null ┆ null ┆ null │\n", - "│ min ┆ ┆ ┆ 0.0 ┆ … ┆ 2024-12-03 ┆ 2024-12 ┆ 2024 ┆ 2024_Q4 │\n", - "│ 25% ┆ null ┆ null ┆ 0.0 ┆ … ┆ 2024-12-05 ┆ null ┆ null ┆ null │\n", - "│ 50% ┆ null ┆ null ┆ 0.0 ┆ … ┆ 2024-12-06 ┆ null ┆ null ┆ null │\n", - "│ 75% ┆ null ┆ null ┆ 0.0 ┆ … ┆ 2024-12-12 ┆ null ┆ null ┆ null │\n", - "│ max ┆ ┆ ┆ 0.0 ┆ … ┆ 2024-12-13 ┆ 2024-12 ┆ 2024 ┆ 2024_Q4 │\n", - "└────────────┴────────────────┴────────┴──────────┴───┴───────────────┴─────────┴────────┴─────────┘" - ] - }, - "execution_count": 71, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ih = (\n", " ih\n", @@ -4168,7 +152,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -4181,7 +165,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -4213,966 +197,9 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "delta": { - "reference": 0.055, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0, - 1 - ], - "y": [ - 0.575, - 1 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "bar": { - "color": "#EC9B00" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.055 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "Web Conversion-Control" - }, - "type": "indicator", - "value": 0.053588795954094534 - }, - { - "delta": { - "reference": 0.055, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0, - 1 - ], - "y": [ - 0, - 0.425 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.055 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "Web Conversion-Test" - }, - "type": "indicator", - "value": 0.05898741473724661 - } - ], - "layout": { - "autosize": true, - "height": 540, - "margin": { - "b": 10, - "l": 10, - "r": 10, - "t": 120 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[CONV] Conversion (Channel/Model Type)" - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ChannelExperimentGroupNegativesPositivesCountConversionRateStdErrNameCName
0WebConversion-Control9731551108330.0535890.002221Web Conversion-ControlWeb_Conversion-Control
1WebConversion-Test9795614110230.0589870.002309Web Conversion-TestWeb_Conversion-Test
\n", - "
" - ], - "text/plain": [ - " Channel ExperimentGroup Negatives Positives Count ConversionRate \\\n", - "0 Web Conversion-Control 9731 551 10833 0.053589 \n", - "1 Web Conversion-Test 9795 614 11023 0.058987 \n", - "\n", - " StdErr Name CName \n", - "0 0.002221 Web Conversion-Control Web_Conversion-Control \n", - "1 0.002309 Web Conversion-Test Web_Conversion-Test " - ] - }, - "execution_count": 74, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "gauge_group_by = [\"Channel\", \"ExperimentGroup\"]\n", "reference = {'Web_Conversion-Test' : 0.055, 'Web_Conversion-Control' : 0.055}\n", @@ -5268,1443 +295,9 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "branchvalues": "total", - "customdata": [ - [ - "0.0038969602362036787", - "199", - "3317", - 0.056598407281001135 - ], - [ - "0.0037745260830149003", - "206", - "3489", - 0.0557510148849797 - ], - [ - "0.0041720272869534515", - "125", - "2490", - 0.04780114722753346 - ], - [ - "0.025967220580682373", - "1", - "37", - 0.026315789473684206 - ], - [ - "0.019858547202852933", - "18", - "186", - 0.08823529411764706 - ], - [ - "0.004293640832369911", - "205", - "3022", - 0.0635264951967772 - ], - [ - "0.02063226798733351", - "10", - "138", - 0.06756756756756757 - ], - [ - "0.06405644848900469", - "2", - "19", - 0.09523809523809523 - ], - [ - "0.0", - "0", - "30", - 0 - ], - [ - "0.0", - "0", - "59", - 0 - ], - [ - "0.01514607976878216", - "4", - "126", - 0.03076923076923077 - ], - [ - "0.013976239729195046", - "9", - "201", - 0.04285714285714286 - ], - [ - "0.004783748419734918", - "112", - "2042", - 0.051996285979572884 - ], - [ - "0.004277411044689493", - "189", - "2926", - 0.060674157303370786 - ], - [ - "0.005961058344078005", - "83", - "1402", - 0.05589225589225589 - ], - [ - "0.03140223998671625", - "2", - "42", - 0.045454545454545456 - ], - [ - "(?)", - "(?)", - "(?)", - 0.05616436346050749 - ], - [ - "(?)", - "(?)", - "(?)", - 0.04749962547424086 - ], - [ - "(?)", - "(?)", - "(?)", - 0.06502768659262644 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07108371196768987 - ], - [ - "0.0", - "0", - "(?)", - 0 - ], - [ - "(?)", - "(?)", - "(?)", - 0.038268530336519 - ], - [ - "(?)", - "(?)", - "(?)", - 0.05714380606105013 - ], - [ - "(?)", - "(?)", - "(?)", - 0.05559477467779822 - ], - [ - "(?)", - "(?)", - "(?)", - 0.056357811460585466 - ], - [ - "(?)", - "(?)", - "(?)", - 0.056357811460585466 - ], - [ - "(?)", - "(?)", - "(?)", - 0.056357811460585466 - ], - [ - "(?)", - "(?)", - "(?)", - 0.056357811460585466 - ] - ], - "domain": { - "x": [ - 0, - 1 - ], - "y": [ - 0, - 1 - ] - }, - "hovertemplate": "labels=%{label}
Count=%{value}
parent=%{parent}
id=%{id}
StdErr=%{customdata[0]}
Positives=%{customdata[1]}
Negatives=%{customdata[2]}
ConversionRate=%{color}", - "ids": [ - "ALL/Web/Acquisition/Phones/AppleIPhone15128GB/Conversion-Control", - "ALL/Web/Acquisition/Phones/AppleIPhone15128GB/Conversion-Test", - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Control", - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB/Conversion-Control", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB/Conversion-Test", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB/Conversion-Control", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB/Conversion-Test", - "ALL/Web/Acquisition/Phones/BirthdayDiscount/Conversion-Control", - "ALL/Web/Acquisition/Phones/BirthdayDiscount/Conversion-Test", - "ALL/Web/Acquisition/Phones/IPhone16/Conversion-Control", - "ALL/Web/Acquisition/Phones/IPhone16/Conversion-Test", - "ALL/Web/Acquisition/Phones/SmartphoneXYZ/Conversion-Control", - "ALL/Web/Acquisition/Phones/SmartphoneXYZ/Conversion-Test", - "ALL/Web/Acquisition/Phones/WalkieTalkie/Conversion-Control", - "ALL/Web/Acquisition/Phones/WalkieTalkie/Conversion-Test", - "ALL/Web/Acquisition/Phones/AppleIPhone15128GB", - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB", - "ALL/Web/Acquisition/Phones/BirthdayDiscount", - "ALL/Web/Acquisition/Phones/IPhone16", - "ALL/Web/Acquisition/Phones/SmartphoneXYZ", - "ALL/Web/Acquisition/Phones/WalkieTalkie", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition", - "ALL/Web", - "ALL" - ], - "labels": [ - "Conversion-Control", - "Conversion-Test", - "Conversion-Control", - "Conversion-Test", - "Conversion-Control", - "Conversion-Test", - "Conversion-Control", - "Conversion-Test", - "Conversion-Control", - "Conversion-Test", - "Conversion-Control", - "Conversion-Test", - "Conversion-Control", - "Conversion-Test", - "Conversion-Control", - "Conversion-Test", - "AppleIPhone15128GB", - "AppleIPhone1564GB", - "AppleIPhone16Pro1TB", - "AppleIPhone16Pro256GB", - "BirthdayDiscount", - "IPhone16", - "SmartphoneXYZ", - "WalkieTalkie", - "Phones", - "Acquisition", - "Web", - "ALL" - ], - "marker": { - "coloraxis": "coloraxis", - "colors": { - "bdata": "CSCOaHf6rD8yskyjZYusPz1pG1hkeag/J6+hvIbymj+XlpaWlpa2P3a2TLtFQ7A/whT5rBtMsT8YhmEYhmG4PwAAAAAAAAAAAAAAAAAAAAAg+IEf+IGfPxZf8RVf8aU/xmmX10Gfqj/xIanirhCvP4FVUObonaw/RhdddNFFpz9U7oZQk8GsP6jd9/PeUag/Xfc/i6elsD989C3KijKyPwAAAAAAAAAAcM2FzO6Xoz+oAGf080GtP9imIxbrdqw/rNcxWu7arD+s1zFa7tqsP6zXMVru2qw/rNcxWu7arD8=", - "dtype": "f8" - } - }, - "name": "", - "parents": [ - "ALL/Web/Acquisition/Phones/AppleIPhone15128GB", - "ALL/Web/Acquisition/Phones/AppleIPhone15128GB", - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro1TB", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB", - "ALL/Web/Acquisition/Phones/AppleIPhone16Pro256GB", - "ALL/Web/Acquisition/Phones/BirthdayDiscount", - "ALL/Web/Acquisition/Phones/BirthdayDiscount", - "ALL/Web/Acquisition/Phones/IPhone16", - "ALL/Web/Acquisition/Phones/IPhone16", - "ALL/Web/Acquisition/Phones/SmartphoneXYZ", - "ALL/Web/Acquisition/Phones/SmartphoneXYZ", - "ALL/Web/Acquisition/Phones/WalkieTalkie", - "ALL/Web/Acquisition/Phones/WalkieTalkie", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition", - "ALL/Web", - "ALL", - "" - ], - "textinfo": "label+value+percent parent+percent root", - "type": "treemap", - "values": { - "bdata": "AAAAAAAGrUAAAAAAAHquQAAAAAAAaKVAAAAAAACAQ0AAAAAAAMBrQAAAAAAA0KpAAAAAAADAY0AAAAAAAAA3QAAAAAAAAD5AAAAAAACATUAAAAAAAMBgQAAAAAAAYGtAAAAAAAC0oUAAAAAAANCpQAAAAAAAgJhAAAAAAAAAR0AAAAAAAMC9QAAAAAAAtqVAAAAAAACMrEAAAAAAAKBmQAAAAAAAQFZAAAAAAAAQdkAAAAAAAMK1QAAAAAAAOJlAAAAAAABY1UAAAAAAAFjVQAAAAAAAWNVAAAAAAABY1UA=", - "dtype": "f8" - } - } - ], - "layout": { - "coloraxis": { - "colorbar": { - "title": { - "text": "ConversionRate" - } - }, - "colorscale": [ - [ - 0, - "rgb(5,48,97)" - ], - [ - 0.1, - "rgb(33,102,172)" - ], - [ - 0.2, - "rgb(67,147,195)" - ], - [ - 0.3, - "rgb(146,197,222)" - ], - [ - 0.4, - "rgb(209,229,240)" - ], - [ - 0.5, - "rgb(247,247,247)" - ], - [ - 0.6, - "rgb(253,219,199)" - ], - [ - 0.7, - "rgb(244,165,130)" - ], - [ - 0.8, - "rgb(214,96,77)" - ], - [ - 0.9, - "rgb(178,24,43)" - ], - [ - 1, - "rgb(103,0,31)" - ] - ] - }, - "height": 640, - "legend": { - "tracegroupgap": 0 - }, - "margin": { - "b": 25, - "l": 25, - "r": 25, - "t": 50 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[BIZ] Conversion rate treemap" - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ChannelIssueGroupNameExperimentGroupNegativesPositivesCountConversionRateStdErr
0WebAcquisitionPhonesAppleIPhone15128GBConversion-Control331719937150.0565980.003897
1WebAcquisitionPhonesAppleIPhone15128GBConversion-Test348920639010.0557510.003775
2WebAcquisitionPhonesAppleIPhone1564GBConversion-Control249012527400.0478010.004172
3WebAcquisitionPhonesAppleIPhone1564GBConversion-Test371390.0263160.025967
4WebAcquisitionPhonesAppleIPhone16Pro1TBConversion-Control186182220.0882350.019859
5WebAcquisitionPhonesAppleIPhone16Pro1TBConversion-Test302220534320.0635260.004294
6WebAcquisitionPhonesAppleIPhone16Pro256GBConversion-Control138101580.0675680.020632
7WebAcquisitionPhonesAppleIPhone16Pro256GBConversion-Test192230.0952380.064056
8WebAcquisitionPhonesBirthdayDiscountConversion-Control300300.0000000.000000
9WebAcquisitionPhonesBirthdayDiscountConversion-Test590590.0000000.000000
10WebAcquisitionPhonesIPhone16Conversion-Control12641340.0307690.015146
11WebAcquisitionPhonesIPhone16Conversion-Test20192190.0428570.013976
12WebAcquisitionPhonesSmartphoneXYZConversion-Control204211222660.0519960.004784
13WebAcquisitionPhonesSmartphoneXYZConversion-Test292618933040.0606740.004277
14WebAcquisitionPhonesWalkieTalkieConversion-Control14028315680.0558920.005961
15WebAcquisitionPhonesWalkieTalkieConversion-Test422460.0454550.031402
\n", - "
" - ], - "text/plain": [ - " Channel Issue Group Name ExperimentGroup \\\n", - "0 Web Acquisition Phones AppleIPhone15128GB Conversion-Control \n", - "1 Web Acquisition Phones AppleIPhone15128GB Conversion-Test \n", - "2 Web Acquisition Phones AppleIPhone1564GB Conversion-Control \n", - "3 Web Acquisition Phones AppleIPhone1564GB Conversion-Test \n", - "4 Web Acquisition Phones AppleIPhone16Pro1TB Conversion-Control \n", - "5 Web Acquisition Phones AppleIPhone16Pro1TB Conversion-Test \n", - "6 Web Acquisition Phones AppleIPhone16Pro256GB Conversion-Control \n", - "7 Web Acquisition Phones AppleIPhone16Pro256GB Conversion-Test \n", - "8 Web Acquisition Phones BirthdayDiscount Conversion-Control \n", - "9 Web Acquisition Phones BirthdayDiscount Conversion-Test \n", - "10 Web Acquisition Phones IPhone16 Conversion-Control \n", - "11 Web Acquisition Phones IPhone16 Conversion-Test \n", - "12 Web Acquisition Phones SmartphoneXYZ Conversion-Control \n", - "13 Web Acquisition Phones SmartphoneXYZ Conversion-Test \n", - "14 Web Acquisition Phones WalkieTalkie Conversion-Control \n", - "15 Web Acquisition Phones WalkieTalkie Conversion-Test \n", - "\n", - " Negatives Positives Count ConversionRate StdErr \n", - "0 3317 199 3715 0.056598 0.003897 \n", - "1 3489 206 3901 0.055751 0.003775 \n", - "2 2490 125 2740 0.047801 0.004172 \n", - "3 37 1 39 0.026316 0.025967 \n", - "4 186 18 222 0.088235 0.019859 \n", - "5 3022 205 3432 0.063526 0.004294 \n", - "6 138 10 158 0.067568 0.020632 \n", - "7 19 2 23 0.095238 0.064056 \n", - "8 30 0 30 0.000000 0.000000 \n", - "9 59 0 59 0.000000 0.000000 \n", - "10 126 4 134 0.030769 0.015146 \n", - "11 201 9 219 0.042857 0.013976 \n", - "12 2042 112 2266 0.051996 0.004784 \n", - "13 2926 189 3304 0.060674 0.004277 \n", - "14 1402 83 1568 0.055892 0.005961 \n", - "15 42 2 46 0.045455 0.031402 " - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "treemap_group_by = [\"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", "\n", @@ -6760,1055 +353,9 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ] - ], - "error_y": { - "array": { - "bdata": "RRJbW54SdT+CSTTWCbmAPw==", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", - "legendgroup": "Conversion-Control", - "marker": { - "color": "#636efa", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Control", - "offsetgroup": "Conversion-Control", - "orientation": "v", - "showlegend": true, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-12-12T00:00:00.000", - "2024-12-13T00:00:00.000" - ], - "xaxis": "x", - "y": { - "bdata": "GYej9nSiqz86pdV8uO2qPw==", - "dtype": "f8" - }, - "yaxis": "y" - }, - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ] - ], - "error_y": { - "array": { - "bdata": "GpTHPQzgdT94Iz5HkXaBPw==", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", - "legendgroup": "Conversion-Test", - "marker": { - "color": "#EF553B", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Test", - "offsetgroup": "Conversion-Test", - "orientation": "v", - "showlegend": true, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-12-12T00:00:00.000", - "2024-12-13T00:00:00.000" - ], - "xaxis": "x", - "y": { - "bdata": "wF7HtoEqrj+BxR9Pr0quPw==", - "dtype": "f8" - }, - "yaxis": "y" - } - ], - "layout": { - "annotations": [ - { - "font": {}, - "showarrow": false, - "text": "Web", - "textangle": 90, - "x": 0.98, - "xanchor": "left", - "xref": "paper", - "y": 0.5, - "yanchor": "middle", - "yref": "paper" - } - ], - "barmode": "group", - "height": 640, - "hovermode": "x unified", - "legend": { - "title": { - "text": "ExperimentGroup" - }, - "tracegroupgap": 0 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[CONV] Daily Conversion Rate with 95% confidence interval" - }, - "updatemenus": [ - { - "buttons": [ - { - "args": [ - "type", - "bar" - ], - "label": "Bar", - "method": "restyle" - }, - { - "args": [ - "type", - "line" - ], - "label": "Line", - "method": "restyle" - } - ], - "direction": "down", - "showactive": true - } - ], - "xaxis": { - "anchor": "y", - "domain": [ - 0, - 0.98 - ], - "tickfont": { - "size": 10 - }, - "title": { - "text": "Day" - } - }, - "yaxis": { - "anchor": "x", - "domain": [ - 0, - 1 - ], - "tickformat": ",.2%", - "title": { - "text": "Conversion Rate" - } - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DayChannelExperimentGroupNegativesPositivesCountConversionRateCI
02024-12-12WebConversion-Control701140078110.0539740.005145
12024-12-12WebConversion-Test702844079080.0589180.005341
22024-12-13WebConversion-Control272015130220.0525950.008165
32024-12-13WebConversion-Test276717431150.0591640.008527
\n", - "
" - ], - "text/plain": [ - " Day Channel ExperimentGroup Negatives Positives Count \\\n", - "0 2024-12-12 Web Conversion-Control 7011 400 7811 \n", - "1 2024-12-12 Web Conversion-Test 7028 440 7908 \n", - "2 2024-12-13 Web Conversion-Control 2720 151 3022 \n", - "3 2024-12-13 Web Conversion-Test 2767 174 3115 \n", - "\n", - " ConversionRate CI \n", - "0 0.053974 0.005145 \n", - "1 0.058918 0.005341 \n", - "2 0.052595 0.008165 \n", - "3 0.059164 0.008527 " - ] - }, - "execution_count": 76, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", "\n", @@ -7913,966 +460,9 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "delta": { - "reference": 0.25, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0, - 1 - ], - "y": [ - 0.575, - 1 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.25 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "Web Conversion-Control" - }, - "type": "indicator", - "value": 0.2543279517603579 - }, - { - "delta": { - "reference": 0.25, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0, - 1 - ], - "y": [ - 0, - 0.425 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "bar": { - "color": "#EC9B00" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.25 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "Web Conversion-Test" - }, - "type": "indicator", - "value": 0.24440388125660487 - } - ], - "layout": { - "autosize": true, - "height": 540, - "margin": { - "b": 10, - "l": 10, - "r": 10, - "t": 120 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[ENG] Click-through rates (Channel/Model Type)" - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ChannelExperimentGroupNegativesPositivesCountCTRStdErrNameCName
0WebConversion-Control76672615128970.2543280.004295Web Conversion-ControlWeb_Conversion-Control
1WebConversion-Test78652544129530.2444040.004212Web Conversion-TestWeb_Conversion-Test
\n", - "
" - ], - "text/plain": [ - " Channel ExperimentGroup Negatives Positives Count CTR \\\n", - "0 Web Conversion-Control 7667 2615 12897 0.254328 \n", - "1 Web Conversion-Test 7865 2544 12953 0.244404 \n", - "\n", - " StdErr Name CName \n", - "0 0.004295 Web Conversion-Control Web_Conversion-Control \n", - "1 0.004212 Web Conversion-Test Web_Conversion-Test " - ] - }, - "execution_count": 77, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "positive_model_response = [\"Clicked\"]\n", "all_model_response = [\"Impression\", \"Pending\"]\n", @@ -8982,1055 +572,9 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ] - ], - "error_y": { - "array": { - "bdata": "RF5y/PpbhD+1jAuA5jKQPw==", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", - "legendgroup": "Conversion-Control", - "marker": { - "color": "#636efa", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Control", - "offsetgroup": "Conversion-Control", - "orientation": "v", - "showlegend": true, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-12-12T00:00:00.000", - "2024-12-13T00:00:00.000" - ], - "xaxis": "x", - "y": { - "bdata": "NxhxcnVo0D8kX1PvnODPPw==", - "dtype": "f8" - }, - "yaxis": "y" - }, - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ] - ], - "error_y": { - "array": { - "bdata": "oiZ4fdQFhD+a6o3xiY2PPw==", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", - "legendgroup": "Conversion-Test", - "marker": { - "color": "#EF553B", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Test", - "offsetgroup": "Conversion-Test", - "orientation": "v", - "showlegend": true, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-12-12T00:00:00.000", - "2024-12-13T00:00:00.000" - ], - "xaxis": "x", - "y": { - "bdata": "vxNQKE6Szz+WQKIYiY3OPw==", - "dtype": "f8" - }, - "yaxis": "y" - } - ], - "layout": { - "annotations": [ - { - "font": {}, - "showarrow": false, - "text": "Web", - "textangle": 90, - "x": 0.98, - "xanchor": "left", - "xref": "paper", - "y": 0.5, - "yanchor": "middle", - "yref": "paper" - } - ], - "barmode": "group", - "height": 640, - "hovermode": "x unified", - "legend": { - "title": { - "text": "ExperimentGroup" - }, - "tracegroupgap": 0 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[ENG] Daily Click-through Rate with 95% confidence interval" - }, - "updatemenus": [ - { - "buttons": [ - { - "args": [ - "type", - "bar" - ], - "label": "Bar", - "method": "restyle" - }, - { - "args": [ - "type", - "line" - ], - "label": "Line", - "method": "restyle" - } - ], - "direction": "down", - "showactive": true - } - ], - "xaxis": { - "anchor": "y", - "domain": [ - 0, - 0.98 - ], - "tickfont": { - "size": 10 - }, - "title": { - "text": "Day" - } - }, - "yaxis": { - "anchor": "x", - "domain": [ - 0, - 1 - ], - "tickformat": ",.2%", - "title": { - "text": "CTR" - } - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
DayChannelExperimentGroupNegativesPositivesCountCTRCI
02024-12-12WebConversion-Control5511190093110.2563760.009941
12024-12-12WebConversion-Test5626184293100.2466520.009777
22024-12-13WebConversion-Control215671535860.2490420.015819
32024-12-13WebConversion-Test223970236430.2386940.015407
\n", - "
" - ], - "text/plain": [ - " Day Channel ExperimentGroup Negatives Positives Count \\\n", - "0 2024-12-12 Web Conversion-Control 5511 1900 9311 \n", - "1 2024-12-12 Web Conversion-Test 5626 1842 9310 \n", - "2 2024-12-13 Web Conversion-Control 2156 715 3586 \n", - "3 2024-12-13 Web Conversion-Test 2239 702 3643 \n", - "\n", - " CTR CI \n", - "0 0.256376 0.009941 \n", - "1 0.246652 0.009777 \n", - "2 0.249042 0.015819 \n", - "3 0.238694 0.015407 " - ] - }, - "execution_count": 78, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", "\n", @@ -10143,7 +687,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.1" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/examples/ih/Example_IH_Analysis.ipynb b/examples/ih/Example_IH_Analysis.ipynb index 49c257d7..2430ac13 100644 --- a/examples/ih/Example_IH_Analysis.ipynb +++ b/examples/ih/Example_IH_Analysis.ipynb @@ -4,7 +4,19 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'cdhtools'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[1], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msys\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcdhtools\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mIHanalysis\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcdhtools\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcdh_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m readDSExport\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'cdhtools'" + ] + } + ], "source": [ "import pandas as pd\n", "import sys\n", @@ -18,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -35,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -51,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -274,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -548,7 +560,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -578,7 +590,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -607,7 +619,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -638,7 +650,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -668,7 +680,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -699,7 +711,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -728,7 +740,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -780,7 +792,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -809,7 +821,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -826,7 +838,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -856,7 +868,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -886,7 +898,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -895,7 +907,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -932,7 +944,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -968,11 +980,9 @@ } ], "metadata": { - "interpreter": { - "hash": "0c5c31b7614ab5f7bbff6555bdc6f3ec4cea8754d51936ee45052251e94c1071" - }, "kernelspec": { - "display_name": "Python 3.9.4 64-bit ('newvfenv': conda)", + "display_name": ".venv", + "language": "python", "name": "python3" }, "language_info": { @@ -985,7 +995,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.9" + "version": "3.12.3" } }, "nbformat": 4, From 28af59e3ef9f77fce7799e030689090bfd4aec28 Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Mon, 16 Dec 2024 13:10:35 +0100 Subject: [PATCH 08/21] First cut IH conversion reporting --- examples/ih/Conversion_Modeling_Reporting.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/ih/Conversion_Modeling_Reporting.ipynb b/examples/ih/Conversion_Modeling_Reporting.ipynb index e76804ba..8f436adf 100644 --- a/examples/ih/Conversion_Modeling_Reporting.ipynb +++ b/examples/ih/Conversion_Modeling_Reporting.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -96,7 +96,7 @@ "# we really only need a few columns\n", "# Outcome outcomes: Conversionm, Impression, Pending\n", "ih = ih.select([\"pyOutcome\", \"pxOutcomeTime\", \"pyChannel\", \"pyIssue\", \"pyGroup\", \"pyName\", \"ExperimentGroup\"])\n", - "ih.head().collect()" + "ih.collect()" ] }, { From ab90dc12e818460a462addeca1c8cb4321747ce6 Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Mon, 16 Dec 2024 15:12:30 +0100 Subject: [PATCH 09/21] Added IH data generator --- .../ih/Conversion_Modeling_Reporting.ipynb | 141 +++++------------- examples/ih/ih_helper.py | 82 ++++++++++ python/pdstools/utils/cdh_utils.py | 131 ++++++++-------- 3 files changed, 187 insertions(+), 167 deletions(-) create mode 100644 examples/ih/ih_helper.py diff --git a/examples/ih/Conversion_Modeling_Reporting.ipynb b/examples/ih/Conversion_Modeling_Reporting.ipynb index 8f436adf..3896f3c5 100644 --- a/examples/ih/Conversion_Modeling_Reporting.ipynb +++ b/examples/ih/Conversion_Modeling_Reporting.ipynb @@ -8,7 +8,8 @@ "source": [ "import polars as pl\n", "from pdstools import read_ds_export\n", - "import re\n", + "from pdstools.utils import cdh_utils\n", + "from ih_helper import ih_generator\n", "\n", "import plotly.io as pio\n", "import plotly as plotly\n", @@ -20,95 +21,26 @@ "pio.renderers.default = \"vscode\"\n" ] }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "def capitalize(fields: list) -> list:\n", - " \"\"\"Applies automatic capitalization.\n", - " Parameters\n", - " ----------\n", - " fields : list\n", - " A list of names\n", - "\n", - " Returns\n", - " -------\n", - " fields : list\n", - " The input list, but each value properly capitalized\n", - " \"\"\"\n", - " capitalize_end_words = [\n", - " \"ID\",\n", - " \"Key\",\n", - " \"Name\",\n", - " \"Treatment\",\n", - " \"Count\",\n", - " \"Category\",\n", - " \"Class\",\n", - " \"Time\",\n", - " \"DateTime\",\n", - " \"UpdateTime\",\n", - " \"Version\",\n", - " \"Rate\",\n", - " \"Ratio\",\n", - " \"Negatives\",\n", - " \"Positives\",\n", - " \"Threshold\",\n", - " \"Error\",\n", - " \"Importance\",\n", - " \"Type\",\n", - " \"Percentage\",\n", - " \"Index\",\n", - " \"Symbol\",\n", - " \"ResponseCount\",\n", - " \"ConfigurationName\",\n", - " \"Configuration\",\n", - " ]\n", - " if not isinstance(fields, list):\n", - " fields = [fields]\n", - " fields_new = [re.sub(\"^p([xyz])\", \"\", field) for field in fields]\n", - " seen = set(fields)\n", - " for i, item in enumerate(fields_new):\n", - " if item in seen:\n", - " fields_new[i] = fields[i]\n", - " for word in capitalize_end_words:\n", - " fields_new = [re.sub(word + '\\b', word, field, flags=re.I) for field in fields_new]\n", - " fields_new = [field[:1].upper() + field[1:] for field in fields_new]\n", - " return fields_new" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "TODO: see if we can generate such data rather than shipping it" - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "ih = read_ds_export(\"Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\", path=\".\")\n", + "from pathlib import Path\n", "\n", "# we really only need a few columns\n", "# Outcome outcomes: Conversionm, Impression, Pending\n", - "ih = ih.select([\"pyOutcome\", \"pxOutcomeTime\", \"pyChannel\", \"pyIssue\", \"pyGroup\", \"pyName\", \"ExperimentGroup\"])\n", - "ih.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dframe_columns = ih.collect_schema().names()\n", - "cols = capitalize(dframe_columns)\n", - "ih = ih.rename(dict(map(lambda i, j: (i, j), dframe_columns, cols)))\n", - "ih.collect_schema()" + "ih_cols = [\"pyOutcome\", \"pxOutcomeTime\", \"pyChannel\", \"pyIssue\", \"pyGroup\", \"pyName\", \"ExperimentGroup\"]\n", + "\n", + "ih_export_file = Path(\"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\")\n", + "if not ih_export_file.exists():\n", + " ih = ih_generator().generate(100000).select(ih_cols)\n", + "else:\n", + " ih = read_ds_export(ih_export_file).select(ih_cols)\n", + "ih = cdh_utils._polars_capitalize(ih)\n", + "ih = ih.filter(pl.col('ExperimentGroup').is_not_null())\n", + "ih.collect().group_by(\"Outcome\").agg(pl.len())" ] }, { @@ -152,20 +84,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "positive_model_response = [\"Conversion\"]\n", "all_model_response = [\"Impression\", \"Pending\"]\n", "group_by = [\"Day\", \"Month\", \"Year\", \"Quarter\", \"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", - "\n", - "ih = ih.filter(pl.col('ExperimentGroup').is_not_null())" + "\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -220,8 +151,7 @@ " (\n", " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " )\n", - " ** 0.5\n", + " ).sqrt()\n", " )\n", " ).alias(\"StdErr\")\n", " ]\n", @@ -230,13 +160,13 @@ " .collect()\n", " )\n", "\n", - "gauge_data = gauge_data.to_pandas()\n", - "\n", "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", "\n", - "gauge_data['Name'] = gauge_data[gauge_group_by].apply(lambda r: ' '.join(r.values.astype(str)), axis=1)\n", - "gauge_data['CName'] = gauge_data[gauge_group_by].apply(lambda r: '_'.join(r.values.astype(str)), axis=1)\n", + "gauge_data = gauge_data.with_columns(\n", + " pl.concat_str(gauge_group_by).alias('Name'),\n", + " pl.concat_str(gauge_group_by, separator = \"_\").alias('CName'),\n", + " )\n", "\n", "fig = make_subplots(rows=rows,\n", " cols=cols,\n", @@ -247,8 +177,8 @@ " autosize=True,\n", " title='[CONV] Conversion (Channel/Model Type)',\n", " margin=dict(b=10, t=120, l=10, r=10))\n", - "\n", - "for index, row in gauge_data.iterrows():\n", + "index = 0\n", + "for row in gauge_data.iter_rows(named=True):\n", " ref_value = reference.get(row['CName'], None)\n", " gauge = {\n", " 'axis': {'tickformat': ',.2%'},\n", @@ -282,6 +212,8 @@ " trace1,\n", " row=(r + 1), col=(c + 1)\n", " )\n", + " index = index + 1\n", + "\n", "fig.show()\n", "gauge_data" ] @@ -328,7 +260,7 @@ " .collect()\n", " )\n", "\n", - "treemap_data = treemap_data.to_pandas()\n", + "treemap_data = treemap_data\n", "\n", "fig = px.treemap(treemap_data, path=[px.Constant(\"ALL\")] + treemap_group_by, values='Count',\n", " color=\"ConversionRate\",\n", @@ -386,7 +318,7 @@ " .collect()\n", " )\n", "\n", - "line_data = line_data.to_pandas()\n", + "line_data = line_data\n", "\n", "if len(line_data[\"Day\"].unique()) < 30:\n", " fig = px.bar(line_data,\n", @@ -514,13 +446,16 @@ " .collect()\n", " )\n", "\n", - "gauge_data = gauge_data.to_pandas()\n", + "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", + "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", "\n", "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", "\n", - "gauge_data['Name'] = gauge_data[gauge_group_by].apply(lambda r: ' '.join(r.values.astype(str)), axis=1)\n", - "gauge_data['CName'] = gauge_data[gauge_group_by].apply(lambda r: '_'.join(r.values.astype(str)), axis=1)\n", + "gauge_data = gauge_data.with_columns(\n", + " pl.concat_str(gauge_group_by).alias('Name'),\n", + " pl.concat_str(gauge_group_by, separator = \"_\").alias('CName'),\n", + " )\n", "\n", "fig = make_subplots(rows=rows,\n", " cols=cols,\n", @@ -529,10 +464,10 @@ "fig.update_layout(\n", " height=270 * rows,\n", " autosize=True,\n", - " title='[ENG] Click-through rates (Channel/Model Type)',\n", + " title='[CONV] Conversion (Channel/Model Type)',\n", " margin=dict(b=10, t=120, l=10, r=10))\n", - "\n", - "for index, row in gauge_data.iterrows():\n", + "index = 0\n", + "for row in gauge_data.iter_rows(named=True):\n", " ref_value = reference.get(row['CName'], None)\n", " gauge = {\n", " 'axis': {'tickformat': ',.2%'},\n", @@ -566,6 +501,8 @@ " trace1,\n", " row=(r + 1), col=(c + 1)\n", " )\n", + " index = index + 1\n", + " \n", "fig.show()\n", "gauge_data" ] @@ -605,7 +542,7 @@ " .collect()\n", " )\n", "\n", - "line_data = line_data.to_pandas()\n", + "line_data = line_data\n", "\n", "if len(line_data[\"Day\"].unique()) < 30:\n", " fig = px.bar(line_data,\n", diff --git a/examples/ih/ih_helper.py b/examples/ih/ih_helper.py new file mode 100644 index 00000000..38e8c5c5 --- /dev/null +++ b/examples/ih/ih_helper.py @@ -0,0 +1,82 @@ +import datetime +import random +import polars as pl +from pdstools.utils import cdh_utils + +# Some day will move into a proper IH class + +class ih_generator: + interactions_period_days = 21 + accept_rate = 0.2 + accept_avg_duration_minutes = 10 + convert_over_accept_rate_test = 0.5 + convert_over_accept_rate_control = 0.3 + convert_avg_duration_days = 2 + + def generate(self, n): + now = datetime.datetime.now() + + + def _interpolate(min, max, i, n): + return min + (max - min) * i / (n - 1) + + + def to_prpc_time_str(timestamp): + return cdh_utils.to_prpc_date_time(timestamp)[0:15] + + + ih_fake_impressions = pl.DataFrame( + { + "InteractionID": [str(int(1e9 + i)) for i in range(n)], + "TimeStamp": [ + (now - datetime.timedelta(days=i * self.interactions_period_days / n)) + for i in range(n) + ], + "AcceptDurationMinutes": [ + random.uniform(0, 2 * self.accept_avg_duration_minutes) for i in range(n) + ], + "ConvertDurationDays": [ + random.uniform(0, 2 * self.convert_avg_duration_days) for i in range(n) + ], + "pyChannel": random.choices(["Web"], k=n), # random.choices(["Web", "Email"], k=n), + "pyIssue": "Acquisition", + "pyGroup": "Phones", + "pyName": "AppleIPhone1564GB", + "ExperimentGroup": ["Conversion-Test", "Conversion-Control"] * int(n / 2), + "pyOutcome": None, + } + ).with_columns( + pyOutcome=pl.when(pl.col.pyChannel == "Web") + .then(pl.lit("Impression")) + .otherwise(pl.lit("Pending")) + ) + ih_fake_accepts = ih_fake_impressions.sample(fraction=self.accept_rate).with_columns( + pl.col.TimeStamp + pl.duration(minutes=pl.col("AcceptDurationMinutes")), + pyOutcome=pl.when(pl.col.pyChannel == "Web") + .then(pl.lit("Clicked")) + .otherwise(pl.lit("Accepted")), + ) + ih_fake_converts_test = ih_fake_accepts.filter(pl.col.ExperimentGroup=="Conversion-Test").sample( + fraction=self.convert_over_accept_rate_test + ).with_columns( + pl.col.TimeStamp + pl.duration(days=pl.col("ConvertDurationDays")), + pyOutcome=pl.lit("Conversion"), + ) + ih_fake_converts_control = ih_fake_accepts.filter(pl.col.ExperimentGroup=="Conversion-Control").sample( + fraction=self.convert_over_accept_rate_control + ).with_columns( + pl.col.TimeStamp + pl.duration(days=pl.col("ConvertDurationDays")), + pyOutcome=pl.lit("Conversion"), + ) + + ih_data=pl.concat([ih_fake_impressions, ih_fake_accepts, ih_fake_converts_test, ih_fake_converts_control]).with_columns( + pxOutcomeTime=pl.col("TimeStamp").map_elements( + to_prpc_time_str, return_dtype=pl.String + ), + ).filter(pl.col("TimeStamp") < pl.lit(now)).drop( + ["AcceptDurationMinutes", "ConvertDurationDays", "TimeStamp"] + ).sort( + "InteractionID", "pxOutcomeTime" + ).lazy() + + return ih_data \ No newline at end of file diff --git a/python/pdstools/utils/cdh_utils.py b/python/pdstools/utils/cdh_utils.py index 86730a84..f26d728b 100644 --- a/python/pdstools/utils/cdh_utils.py +++ b/python/pdstools/utils/cdh_utils.py @@ -216,8 +216,7 @@ def _extract_keys( .alias(c) for c in overlap ] - ) - .drop([f"{c}_decoded" for c in overlap]) + ).drop([f"{c}_decoded" for c in overlap]) ) @@ -488,81 +487,81 @@ def _capitalize(fields: Union[str, Iterable[str]]) -> List[str]: The input list, but each value properly capitalized """ capitalize_endwords = [ - "ID", - "Key", - "Name", - "Treatment", - "Count", + "Active", + "BinNegatives", + "BinPositives", + "BinResponseCount", + "BinSymbol", + "Bins", + "Cap", "Category", "Class", - "Time", + "Component", + "Configuration", + "ConfigurationName", + "Context", + "Control", + "Count", + "Date", "DateTime", - "UpdateTime", - "ToClass", - "Version", - "Predictor", - "Predictors", - "Rate", - "Ratio", - "Negatives", - "Positives", - "Threshold", + "Description", + "Email", + "Enabled", "Error", + "Evidence", + "Execution", + "Group", + "GroupIndex", + "Hash", + "ID", + "Identifier", "Importance", - "Type", - "Percentage", "Index", - "Symbol", + "Issue", + "Key", + "Limit", "LowerBound", - "UpperBound", - "Bins", - "GroupIndex", - "ResponseCount", + "Message", + "ModelTechnique", + "Name", + "Negatives", "NegativesPercentage", - "PositivesPercentage", - "BinPositives", - "BinNegatives", - "BinResponseCount", - "BinSymbol", - "ResponseCountPercentage", - "ConfigurationName", - "Configuration", - "SMS", - "Relevant", - "Proposition", - "Active", - "Description", - "Reference", - "Date", + "Offline", + "Omni", + "Outcome", + "Paid", + "Percentage", "Performance", - "Identifier", - "Component", + "Positives", + "PositivesPercentage", "Prediction", - "Outcome", - "Hash", - "URL", - "Cap", - "Template", - "Issue", - "Group", - "Control", - "Evidence", + "Predictor", + "Predictors", "Propensity", - "Paid", - "Subject", - "Email", - "Web", - "Context", - "Limit", + "Proposition", + "Rate", + "Ratio", + "Reference", + "Relevant", + "ResponseCount", + "ResponseCountPercentage", + "SMS", "Stage", - "Omni", - "Execution", - "Enabled", - "Message", - "Offline", - "Update", "Strategy", - "ModelTechnique" + "Subject", + "Symbol", + "Template", + "Threshold", + "Time", + "ToClass", + "Treatment", + "Type", + "URL", + "Update", + "UpdateTime", + "UpperBound", + "Version", + "Web", ] if not isinstance(fields, list): fields = [fields] @@ -806,7 +805,9 @@ def lift_impl(bin_pos, bin_neg, total_pos, total_neg): # TODO not sure how polars (mis)behaves when there are no positives at all # I would hope for a NaN but base python doesn't do that. Polars perhaps. # Stijn: It does have proper None value support, may work like you say - bin_pos * (total_pos + total_neg) / ((bin_pos + bin_neg) * total_pos) + bin_pos + * (total_pos + total_neg) + / ((bin_pos + bin_neg) * total_pos) ).alias("Lift") return lift_impl(pos_col, neg_col, pos_col.sum(), neg_col.sum()) From e842d2b84d5be1cda99b6afddfbca1c186b13fb8 Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Mon, 16 Dec 2024 16:50:21 +0100 Subject: [PATCH 10/21] Small refactorings --- .../ih/Conversion_Modeling_Reporting.ipynb | 9968 ++++++++++++++++- examples/ih/ih_helper.py | 17 +- 2 files changed, 9915 insertions(+), 70 deletions(-) diff --git a/examples/ih/Conversion_Modeling_Reporting.ipynb b/examples/ih/Conversion_Modeling_Reporting.ipynb index 3896f3c5..644c6a04 100644 --- a/examples/ih/Conversion_Modeling_Reporting.ipynb +++ b/examples/ih/Conversion_Modeling_Reporting.ipynb @@ -2,14 +2,3907 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import polars as pl\n", "from pdstools import read_ds_export\n", "from pdstools.utils import cdh_utils\n", - "from ih_helper import ih_generator\n", + "from ih_helper import interaction_history\n", "\n", "import plotly.io as pio\n", "import plotly as plotly\n", @@ -23,9 +3916,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (4, 7)
ChannelExperimentGroupClickedPendingConversionImpressionAccepted
strstru32u32u32u32u32
"Web""Conversion-Test"5013null230824968null
"Email""Conversion-Control"null249571370null4937
"Email""Conversion-Test"null250312303null4966
"Web""Conversion-Control"5075null140525043null
" + ], + "text/plain": [ + "shape: (4, 7)\n", + "┌─────────┬────────────────────┬─────────┬─────────┬────────────┬────────────┬──────────┐\n", + "│ Channel ┆ ExperimentGroup ┆ Clicked ┆ Pending ┆ Conversion ┆ Impression ┆ Accepted │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ u32 ┆ u32 ┆ u32 ┆ u32 ┆ u32 │\n", + "╞═════════╪════════════════════╪═════════╪═════════╪════════════╪════════════╪══════════╡\n", + "│ Web ┆ Conversion-Test ┆ 5013 ┆ null ┆ 2308 ┆ 24968 ┆ null │\n", + "│ Email ┆ Conversion-Control ┆ null ┆ 24957 ┆ 1370 ┆ null ┆ 4937 │\n", + "│ Email ┆ Conversion-Test ┆ null ┆ 25031 ┆ 2303 ┆ null ┆ 4966 │\n", + "│ Web ┆ Conversion-Control ┆ 5075 ┆ null ┆ 1405 ┆ 25043 ┆ null │\n", + "└─────────┴────────────────────┴─────────┴─────────┴────────────┴────────────┴──────────┘" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from pathlib import Path\n", "\n", @@ -33,14 +3957,14 @@ "# Outcome outcomes: Conversionm, Impression, Pending\n", "ih_cols = [\"pyOutcome\", \"pxOutcomeTime\", \"pyChannel\", \"pyIssue\", \"pyGroup\", \"pyName\", \"ExperimentGroup\"]\n", "\n", - "ih_export_file = Path(\"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\")\n", + "ih_export_file = Path(\"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip \")\n", "if not ih_export_file.exists():\n", - " ih = ih_generator().generate(100000).select(ih_cols)\n", + " ih = interaction_history().generate(100000).select(ih_cols)\n", "else:\n", " ih = read_ds_export(ih_export_file).select(ih_cols)\n", "ih = cdh_utils._polars_capitalize(ih)\n", "ih = ih.filter(pl.col('ExperimentGroup').is_not_null())\n", - "ih.collect().group_by(\"Outcome\").agg(pl.len())" + "ih.collect().group_by([\"Channel\", \"ExperimentGroup\", \"Outcome\"]).agg(pl.len()).pivot(on=\"Outcome\",index=[\"Channel\",\"ExperimentGroup\"])" ] }, { @@ -52,26 +3976,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (9, 13)
statisticOutcomeOutcomeTimeChannelIssueGroupNameExperimentGroupOutcomeDateTimeDayMonthYearQuarter
strstrstrstrstrstrstrstrstrstrstrstrstr
"count""127376""127376""127376""127376""127376""127376""127376""127376""127376""127376""127376""127376"
"null_count""0""0""0""0""0""0""0""0""0""0""0""0"
"mean"nullnullnullnullnullnullnull"2024-12-06 05:42:04.989000""2024-12-05 17:42:58.470000"nullnullnull
"std"nullnullnullnullnullnullnullnullnullnullnullnull
"min""Accepted""20241125T164124""Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control""2024-11-25 16:41:24""2024-11-25""2024-11""2024""2024_Q4"
"25%"nullnullnullnullnullnullnull"2024-12-01 00:13:56""2024-12-01"nullnullnull
"50%"nullnullnullnullnullnullnull"2024-12-06 05:43:42""2024-12-06"nullnullnull
"75%"nullnullnullnullnullnullnull"2024-12-11 11:42:22""2024-12-11"nullnullnull
"max""Pending""20241216T164048""Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test""2024-12-16 16:40:48""2024-12-16""2024-12""2024""2024_Q4"
" + ], + "text/plain": [ + "shape: (9, 13)\n", + "┌────────────┬──────────┬───────────────┬─────────┬───┬───────────────┬─────────┬────────┬─────────┐\n", + "│ statistic ┆ Outcome ┆ OutcomeTime ┆ Channel ┆ … ┆ Day ┆ Month ┆ Year ┆ Quarter │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ str ┆ str ┆ ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════╪═══════════════╪═════════╪═══╪═══════════════╪═════════╪════════╪═════════╡\n", + "│ count ┆ 127376 ┆ 127376 ┆ 127376 ┆ … ┆ 127376 ┆ 127376 ┆ 127376 ┆ 127376 │\n", + "│ null_count ┆ 0 ┆ 0 ┆ 0 ┆ … ┆ 0 ┆ 0 ┆ 0 ┆ 0 │\n", + "│ mean ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-05 ┆ null ┆ null ┆ null │\n", + "│ ┆ ┆ ┆ ┆ ┆ 17:42:58.4700 ┆ ┆ ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ 00 ┆ ┆ ┆ │\n", + "│ std ┆ null ┆ null ┆ null ┆ … ┆ null ┆ null ┆ null ┆ null │\n", + "│ min ┆ Accepted ┆ 20241125T1641 ┆ Email ┆ … ┆ 2024-11-25 ┆ 2024-11 ┆ 2024 ┆ 2024_Q4 │\n", + "│ ┆ ┆ 24 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ 25% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-01 ┆ null ┆ null ┆ null │\n", + "│ 50% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-06 ┆ null ┆ null ┆ null │\n", + "│ 75% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-11 ┆ null ┆ null ┆ null │\n", + "│ max ┆ Pending ┆ 20241216T1640 ┆ Web ┆ … ┆ 2024-12-16 ┆ 2024-12 ┆ 2024 ┆ 2024_Q4 │\n", + "│ ┆ ┆ 48 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "└────────────┴──────────┴───────────────┴─────────┴───┴───────────────┴─────────┴────────┴─────────┘" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ih = (\n", - " ih\n", - " .with_columns(\n", - " pl.col('OutcomeTime').str.strptime(pl.Datetime, \"%Y%m%dT%H%M%S%.3f %Z\").alias('OutcomeDateTime')\n", - " )\n", - " .with_columns(\n", - " [\n", - " pl.col(\"OutcomeDateTime\").dt.date().alias(\"Day\"),\n", - " (pl.col(\"OutcomeDateTime\").dt.strftime(\"%Y-%m\")).alias(\"Month\"),\n", - " pl.col(\"OutcomeDateTime\").dt.year().cast(str).alias(\"Year\"),\n", - " (pl.col(\"OutcomeDateTime\").dt.year().cast(str) + \"_Q\" + pl.col(\n", - " \"OutcomeDateTime\").dt.quarter().cast(\n", - " str)).alias(\"Quarter\")\n", - " ]\n", - " )\n", - " )\n", + "ih = ih.with_columns(\n", + " pl.col(\"OutcomeTime\")\n", + " .str.strptime(pl.Datetime, \"%Y%m%dT%H%M%S%.3f %Z\")\n", + " .alias(\"OutcomeDateTime\")\n", + ").with_columns(\n", + " [\n", + " pl.col(\"OutcomeDateTime\").dt.date().alias(\"Day\"),\n", + " (pl.col(\"OutcomeDateTime\").dt.strftime(\"%Y-%m\")).alias(\"Month\"),\n", + " pl.col(\"OutcomeDateTime\").dt.year().cast(str).alias(\"Year\"),\n", + " (\n", + " pl.col(\"OutcomeDateTime\").dt.year().cast(str)\n", + " + \"_Q\"\n", + " + pl.col(\"OutcomeDateTime\").dt.quarter().cast(str)\n", + " ).alias(\"Quarter\"),\n", + " ]\n", + ")\n", "ih.describe()" ] }, @@ -101,22 +4065,19 @@ "outputs": [], "source": [ "ih_analysis = (\n", - " ih.filter(\n", - " (pl.col(\"Outcome\").is_in(all_model_response + positive_model_response))\n", - " )\n", - " .with_columns([\n", - " pl.when(pl.col('Outcome').is_in(positive_model_response)).\n", - " then(1).otherwise(0).alias('Outcome_Binary')\n", - " ])\n", - " .group_by(group_by)\n", - " .agg([\n", - " pl.len().alias('Count'),\n", - " pl.sum(\"Outcome_Binary\").alias(\"Positives\")\n", - " ])\n", - " .with_columns([\n", - " (pl.col(\"Count\") - (2 * pl.col(\"Positives\"))).alias(\"Negatives\")\n", - " ])\n", - " )" + " ih.filter((pl.col(\"Outcome\").is_in(all_model_response + positive_model_response)))\n", + " .with_columns(\n", + " [\n", + " pl.when(pl.col(\"Outcome\").is_in(positive_model_response))\n", + " .then(1)\n", + " .otherwise(0)\n", + " .alias(\"Outcome_Binary\")\n", + " ]\n", + " )\n", + " .group_by(group_by)\n", + " .agg([pl.len().alias(\"Count\"), pl.sum(\"Outcome_Binary\").alias(\"Positives\")])\n", + " .with_columns([(pl.col(\"Count\") - (2 * pl.col(\"Positives\"))).alias(\"Negatives\")])\n", + ")" ] }, { @@ -128,18 +4089,47 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (4, 7)
ChannelExperimentGroupNegativesPositivesCountConversionRateStdErr
strstru32i32u32f64f64
"Email""Conversion-Control"235871370263270.0548940.001442
"Email""Conversion-Test"227282303273340.0920060.001827
"Web""Conversion-Control"236381405264480.0561040.001454
"Web""Conversion-Test"226602308272760.0924380.001833
" + ], + "text/plain": [ + "shape: (4, 7)\n", + "┌─────────┬────────────────────┬───────────┬───────────┬───────┬────────────────┬──────────┐\n", + "│ Channel ┆ ExperimentGroup ┆ Negatives ┆ Positives ┆ Count ┆ ConversionRate ┆ StdErr │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ u32 ┆ i32 ┆ u32 ┆ f64 ┆ f64 │\n", + "╞═════════╪════════════════════╪═══════════╪═══════════╪═══════╪════════════════╪══════════╡\n", + "│ Email ┆ Conversion-Control ┆ 23587 ┆ 1370 ┆ 26327 ┆ 0.054894 ┆ 0.001442 │\n", + "│ Email ┆ Conversion-Test ┆ 22728 ┆ 2303 ┆ 27334 ┆ 0.092006 ┆ 0.001827 │\n", + "│ Web ┆ Conversion-Control ┆ 23638 ┆ 1405 ┆ 26448 ┆ 0.056104 ┆ 0.001454 │\n", + "│ Web ┆ Conversion-Test ┆ 22660 ┆ 2308 ┆ 27276 ┆ 0.092438 ┆ 0.001833 │\n", + "└─────────┴────────────────────┴───────────┴───────────┴───────┴────────────────┴──────────┘" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "gauge_group_by = [\"Channel\", \"ExperimentGroup\"]\n", - "reference = {'Web_Conversion-Test' : 0.055, 'Web_Conversion-Control' : 0.055}\n", + "reference = {'WebConversion-Test' : 0.055, 'WebConversion-Control' : 0.055}\n", "gauge_data = (\n", " ih_analysis.group_by(gauge_group_by)\n", " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\")\n", + " pl.sum([\"Negatives\", \"Positives\", \"Count\"])\n", " )\n", " .with_columns(\n", " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"ConversionRate\")]\n", @@ -159,13 +4149,972 @@ " .sort(gauge_group_by, descending=False)\n", " .collect()\n", " )\n", - "\n", + "gauge_data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "delta": { + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0, + 0.45 + ], + "y": [ + 0.575, + 1 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "EmailConversion-Control" + }, + "type": "indicator", + "value": 0.05489441839964739 + }, + { + "delta": { + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0.55, + 1 + ], + "y": [ + 0.575, + 1 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "EmailConversion-Test" + }, + "type": "indicator", + "value": 0.09200591266829132 + }, + { + "delta": { + "reference": 0.055, + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0, + 0.45 + ], + "y": [ + 0, + 0.425 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75, + "value": 0.055 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "WebConversion-Control" + }, + "type": "indicator", + "value": 0.056103501976600245 + }, + { + "delta": { + "reference": 0.055, + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0.55, + 1 + ], + "y": [ + 0, + 0.425 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75, + "value": 0.055 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "WebConversion-Test" + }, + "type": "indicator", + "value": 0.09243832105094521 + } + ], + "layout": { + "autosize": true, + "height": 540, + "margin": { + "b": 10, + "l": 10, + "r": 10, + "t": 120 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[CONV] Conversion (Channel/Model Type)" + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", "\n", "gauge_data = gauge_data.with_columns(\n", " pl.concat_str(gauge_group_by).alias('Name'),\n", - " pl.concat_str(gauge_group_by, separator = \"_\").alias('CName'),\n", + " # pl.concat_str(gauge_group_by, separator = \"_\").alias('CName'),\n", " )\n", "\n", "fig = make_subplots(rows=rows,\n", @@ -179,7 +5128,7 @@ " margin=dict(b=10, t=120, l=10, r=10))\n", "index = 0\n", "for row in gauge_data.iter_rows(named=True):\n", - " ref_value = reference.get(row['CName'], None)\n", + " ref_value = reference.get(row['Name'], None)\n", " gauge = {\n", " 'axis': {'tickformat': ',.2%'},\n", " 'threshold': {\n", @@ -214,8 +5163,7 @@ " )\n", " index = index + 1\n", "\n", - "fig.show()\n", - "gauge_data" + "fig.show()" ] }, { @@ -227,9 +5175,1055 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "branchvalues": "total", + "customdata": [ + [ + "0.0018330411793641235", + "2308", + "22660", + 0.09243832105094521 + ], + [ + "0.0014541660330504179", + "1405", + "23638", + 0.05610350197660025 + ], + [ + "0.001826881083957284", + "2303", + "22728", + 0.09200591266829132 + ], + [ + "0.0014418101169569632", + "1370", + "23587", + 0.05489441839964739 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07379838188037106 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07455090956114035 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07379838188037106 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07455090956114035 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07455090956114035 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07379838188037106 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07455090956114035 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07379838188037106 + ], + [ + "(?)", + "(?)", + "(?)", + 0.07417486646501184 + ] + ], + "domain": { + "x": [ + 0, + 1 + ], + "y": [ + 0, + 1 + ] + }, + "hovertemplate": "labels=%{label}
Count=%{value}
parent=%{parent}
id=%{id}
StdErr=%{customdata[0]}
Positives=%{customdata[1]}
Negatives=%{customdata[2]}
ConversionRate=%{color}", + "ids": [ + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Control", + "ALL/Email/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", + "ALL/Email/Acquisition/Phones/AppleIPhone1564GB/Conversion-Control", + "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Email/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Web/Acquisition", + "ALL/Email/Acquisition", + "ALL/Web", + "ALL/Email", + "ALL" + ], + "labels": [ + "Conversion-Test", + "Conversion-Control", + "Conversion-Test", + "Conversion-Control", + "AppleIPhone1564GB", + "AppleIPhone1564GB", + "Phones", + "Phones", + "Acquisition", + "Acquisition", + "Web", + "Email", + "ALL" + ], + "marker": { + "coloraxis": "coloraxis", + "colors": { + "bdata": "8prPrQmqtz+Zc1wkmbmsP/Pt8hGzjbc/g9OEBx8brD+Lhaxkc+SyPzTAc7bEFbM/i4WsZHPksj80wHO2xBWzPzTAc7bEFbM/i4WsZHPksj80wHO2xBWzP4uFrGRz5LI/OgOnwR/9sj8=", + "dtype": "f8" + } + }, + "name": "", + "parents": [ + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Email/Acquisition/Phones", + "ALL/Web/Acquisition/Phones", + "ALL/Email/Acquisition", + "ALL/Web/Acquisition", + "ALL/Web", + "ALL/Email", + "ALL", + "ALL", + "" + ], + "textinfo": "label+value+percent parent+percent root", + "type": "treemap", + "values": { + "bdata": "AAAAAACj2kAAAAAAANTZQAAAAACAsdpAAAAAAMC12UAAAAAAoDPqQAAAAACAO+pAAAAAAKAz6kAAAAAAgDvqQAAAAACAO+pAAAAAAKAz6kAAAAAAgDvqQAAAAACgM+pAAAAAAJA3+kA=", + "dtype": "f8" + } + } + ], + "layout": { + "coloraxis": { + "colorbar": { + "title": { + "text": "ConversionRate" + } + }, + "colorscale": [ + [ + 0, + "rgb(5,48,97)" + ], + [ + 0.1, + "rgb(33,102,172)" + ], + [ + 0.2, + "rgb(67,147,195)" + ], + [ + 0.3, + "rgb(146,197,222)" + ], + [ + 0.4, + "rgb(209,229,240)" + ], + [ + 0.5, + "rgb(247,247,247)" + ], + [ + 0.6, + "rgb(253,219,199)" + ], + [ + 0.7, + "rgb(244,165,130)" + ], + [ + 0.8, + "rgb(214,96,77)" + ], + [ + 0.9, + "rgb(178,24,43)" + ], + [ + 1, + "rgb(103,0,31)" + ] + ] + }, + "height": 640, + "legend": { + "tracegroupgap": 0 + }, + "margin": { + "b": 25, + "l": 25, + "r": 25, + "t": 50 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[BIZ] Conversion rate treemap" + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (4, 10)
ChannelIssueGroupNameExperimentGroupNegativesPositivesCountConversionRateStdErr
strstrstrstrstru32i32u32f64f64
"Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control"235871370263270.0548940.001442
"Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test"227282303273340.0920060.001827
"Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control"236381405264480.0561040.001454
"Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test"226602308272760.0924380.001833
" + ], + "text/plain": [ + "shape: (4, 10)\n", + "┌─────────┬─────────────┬────────┬───────────────┬───┬───────────┬───────┬──────────────┬──────────┐\n", + "│ Channel ┆ Issue ┆ Group ┆ Name ┆ … ┆ Positives ┆ Count ┆ ConversionRa ┆ StdErr │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ te ┆ --- │\n", + "│ str ┆ str ┆ str ┆ str ┆ ┆ i32 ┆ u32 ┆ --- ┆ f64 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ f64 ┆ │\n", + "╞═════════╪═════════════╪════════╪═══════════════╪═══╪═══════════╪═══════╪══════════════╪══════════╡\n", + "│ Email ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 1370 ┆ 26327 ┆ 0.054894 ┆ 0.001442 │\n", + "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", + "│ Email ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 2303 ┆ 27334 ┆ 0.092006 ┆ 0.001827 │\n", + "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", + "│ Web ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 1405 ┆ 26448 ┆ 0.056104 ┆ 0.001454 │\n", + "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", + "│ Web ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 2308 ┆ 27276 ┆ 0.092438 ┆ 0.001833 │\n", + "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", + "└─────────┴─────────────┴────────┴───────────────┴───┴───────────┴───────┴──────────────┴──────────┘" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "treemap_group_by = [\"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", "\n", @@ -285,9 +6279,1443 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ] + ], + "error_y": { + "array": { + "bdata": "DWL/DCcahz/F8zxNjF9/P3Tu/vQ4OYU/+UB9PBWRiT+uJCCKJ5GLP6qc0gMkLos/sOqXzYhbiz8ZE5WuVMqNP56dBfW5eIs/F5Um+TLRjD8Fw6EaqcCIPxLWvK84AY0/Z97fcm1dij8UsGe1gmaMP8PEyHdeFow/74I3ik+NiD/vxqe8smuKP1LwvmDI0ok/QhYikYr1iz8Jb1zA7KCMP4LbYtJ1+Ik/ofVLZUkMkj8=", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", + "legendgroup": "Conversion-Control", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Control", + "offsetgroup": "Conversion-Control", + "orientation": "v", + "showlegend": true, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-11-25", + "2024-11-26", + "2024-11-27", + "2024-11-28", + "2024-11-29", + "2024-11-30", + "2024-12-01", + "2024-12-02", + "2024-12-03", + "2024-12-04", + "2024-12-05", + "2024-12-06", + "2024-12-07", + "2024-12-08", + "2024-12-09", + "2024-12-10", + "2024-12-11", + "2024-12-12", + "2024-12-13", + "2024-12-14", + "2024-12-15", + "2024-12-16" + ], + "xaxis": "x2", + "y": { + "bdata": "5RMxm0uHij+/9pDLioGSPxLtoxHto6E/l0aBCI5dqj8W01lMZzGtP/7GZFg+5K0/kgUzd932rD8hfqm6EJyxPw4y9WgEjK0/YPS0eH8fsT+rbDhi2hOpPxP6ZVtIWbE/ZNmNB6eJrD/+8Iw4XBiwP//5ZMZ8sa4/inzWDabIpz+/IeU6YvGrP3oEW+WM26o/+Se1t8jkrz+mtSf/zuWwP4ex2vrbA6s/k5E04TRasj8=", + "dtype": "f8" + }, + "yaxis": "y2" + }, + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ] + ], + "error_y": { + "array": { + "bdata": "BhvaZK6lgD9g9NQqhHt8P/YRPcBHs4Q/sJTFsjZTiT+a85LlufOJPzWSWY6BKI0/8hDzKlyBjD9fSRI8cv2LP033NblDn4k/DpFd3Yb1jT/dzOvMjjCKP88YkQCLqIk/qRi32GTaiz/5QH08FZGJP/TAqvbfuYs/Tw9FoNZ2jD/IGXKbok6NPxJNgzS0NI0/eDXPHhYKjD9ZfQYKmTmMP08JQRaHhYw/C+4jlOOwkD8=", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", + "legendgroup": "Conversion-Control", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Control", + "offsetgroup": "Conversion-Control", + "orientation": "v", + "showlegend": false, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-11-25", + "2024-11-26", + "2024-11-27", + "2024-11-28", + "2024-11-29", + "2024-11-30", + "2024-12-01", + "2024-12-02", + "2024-12-03", + "2024-12-04", + "2024-12-05", + "2024-12-06", + "2024-12-07", + "2024-12-08", + "2024-12-09", + "2024-12-10", + "2024-12-11", + "2024-12-12", + "2024-12-13", + "2024-12-14", + "2024-12-15", + "2024-12-16" + ], + "xaxis": "x", + "y": { + "bdata": "GBgYGBgYeD/F3aDW1uuPPwzh77wHxKA/QlBnbfE6qT8UHD3B0ROsP7Ot4dknHLE/5SfEWfkJsT9K8QKZFC+wP16QlH/o26o/Eh/xER/xsT/tdPyDC5OqP9DOOiH4y6k/m4eRDM9trj+XRoEIjl2qPweQagCpBrA/fAfxHcR3sD+R2myR2myxP5AWqmSUI7E/xY+jkP9Rrz+DmZN+3oivP14uwx0SZLA/J3ZiJ3Zirz8=", + "dtype": "f8" + }, + "yaxis": "y" + }, + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ] + ], + "error_y": { + "array": { + "bdata": "4mc+p4tchj9Cj+n8AbyCP8nNFnqwD4w/fY7rs6vYjj+JF9Do/p2RP+xmu3n90pE/FKn+7DDZkT+DPChQUwuSP2TR+IhRKpE/wNa4/iQxkT8FbpkqvNOPPy1lVHUFPZI/hcF8vFVfkT8ThJ0vaF6QP0VkZ+LqN5I/QTq09Wd0kD+XDKATrDuSP4c9NmO4uJE/a3oK2u2pkT+LPTfBflCRP9Y+P6IhypE/KNT1ORgBkz8=", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", + "legendgroup": "Conversion-Test", + "marker": { + "color": "#EF553B", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Test", + "offsetgroup": "Conversion-Test", + "orientation": "v", + "showlegend": true, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-11-25", + "2024-11-26", + "2024-11-27", + "2024-11-28", + "2024-11-29", + "2024-11-30", + "2024-12-01", + "2024-12-02", + "2024-12-03", + "2024-12-04", + "2024-12-05", + "2024-12-06", + "2024-12-07", + "2024-12-08", + "2024-12-09", + "2024-12-10", + "2024-12-11", + "2024-12-12", + "2024-12-13", + "2024-12-14", + "2024-12-15", + "2024-12-16" + ], + "xaxis": "x2", + "y": { + "bdata": "F2DyFmDyhj8mGn5L09WbP6rPtDX+bq4/FBQUFBQUtD+kFeTexve5P+PtRg9SaLo/HwK6BUNcuz+BHfJv6VW7PzaU11BeQ7k/28o1p7qguD9XbnC2X4a1P/SSmlYfzLw/ENZvgKnmuD/EBqDxi562P925c+fOnbs/wPXPbtehtj9DUmfH3aO7PyiYmGLvA7o/3i/w4lt/uj/h2+MBvdS5P57xjGc847k/VVVVVVVVtT8=", + "dtype": "f8" + }, + "yaxis": "y2" + }, + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ] + ], + "error_y": { + "array": { + "bdata": "TKR2mfHQkT+RLdJBoUqCP7NgpqHuEYw/M2KzEJVAkj9regra7amRPxYh4UTMA44/uLl9++sAkj8wW7FCpE2RPweqaj+7vZE/p2Oz92GQkD85Qk+VPgGSP1AJwmjc4pA/CxY4ce4xkT80L7vWISCSP9X1Qg+XNJE/3qBgEugtkT+k0wSEcauQP7DcEYv6HZE/4S8o21V+kj8qDgjB8g2QP95H2xEmQpI/gXtejFBylT8=", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", + "legendgroup": "Conversion-Test", + "marker": { + "color": "#EF553B", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Test", + "offsetgroup": "Conversion-Test", + "orientation": "v", + "showlegend": false, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-11-25", + "2024-11-26", + "2024-11-27", + "2024-11-28", + "2024-11-29", + "2024-11-30", + "2024-12-01", + "2024-12-02", + "2024-12-03", + "2024-12-04", + "2024-12-05", + "2024-12-06", + "2024-12-07", + "2024-12-08", + "2024-12-09", + "2024-12-10", + "2024-12-11", + "2024-12-12", + "2024-12-13", + "2024-12-14", + "2024-12-15", + "2024-12-16" + ], + "xaxis": "x", + "y": { + "bdata": "6k1vetObnj/EzUolSnKZP36daxhijrA/Qd+DvLxvuz/eL/DiW3+6P5WYSkwlprI/qi6f2QcKuz/lLmRKk125P3MJaSr77bk/+Jphqivntj/tdPyDC5O6P1J6ZEk8X7c/lUDHa5syuT+btVmbtVm7P0dwXBJdU7k/16BiTZd+uD/yHeekJ563P1SNWjId97g/f206zln/vD+BLF8IibW0P53yR53yR70/oxZz/Vckuj8=", + "dtype": "f8" + }, + "yaxis": "y" + } + ], + "layout": { + "annotations": [ + { + "font": {}, + "showarrow": false, + "text": "Web", + "textangle": 90, + "x": 0.98, + "xanchor": "left", + "xref": "paper", + "y": 0.2425, + "yanchor": "middle", + "yref": "paper" + }, + { + "font": {}, + "showarrow": false, + "text": "Email", + "textangle": 90, + "x": 0.98, + "xanchor": "left", + "xref": "paper", + "y": 0.7575000000000001, + "yanchor": "middle", + "yref": "paper" + } + ], + "barmode": "group", + "height": 640, + "hovermode": "x unified", + "legend": { + "title": { + "text": "ExperimentGroup" + }, + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[CONV] Daily Conversion Rate with 95% confidence interval" + }, + "updatemenus": [ + { + "buttons": [ + { + "args": [ + "type", + "bar" + ], + "label": "Bar", + "method": "restyle" + }, + { + "args": [ + "type", + "line" + ], + "label": "Line", + "method": "restyle" + } + ], + "direction": "down", + "showactive": true + } + ], + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 0.98 + ], + "tickfont": { + "size": 10 + }, + "title": { + "text": "Day" + } + }, + "xaxis2": { + "anchor": "y2", + "domain": [ + 0, + 0.98 + ], + "matches": "x", + "showticklabels": false, + "tickfont": { + "size": 10 + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 0.485 + ], + "tickformat": ",.2%", + "title": { + "text": "Conversion Rate" + } + }, + "yaxis2": { + "anchor": "x2", + "domain": [ + 0.515, + 1 + ], + "matches": "y", + "tickformat": ",.2%", + "title": { + "text": "ConversionRate" + } + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (88, 8)
DayChannelExperimentGroupNegativesPositivesCountConversionRateCI
datestrstru32i32u32f64f64
2024-11-25"Email""Conversion-Control"38153910.0129530.01128
2024-11-25"Email""Conversion-Test"35343610.0112040.010919
2024-11-25"Web""Conversion-Control"33823420.0058820.008129
2024-11-25"Web""Conversion-Test"357113790.0298910.017399
2024-11-26"Email""Conversion-Control"11412111830.0180720.007659
……………………
2024-12-15"Web""Conversion-Test"108414013640.1143790.01783
2024-12-16"Email""Conversion-Control"764598820.0716890.017625
2024-12-16"Email""Conversion-Test"781719230.0833330.018559
2024-12-16"Web""Conversion-Control"781518830.0612980.0163
2024-12-16"Web""Conversion-Test"721828850.1021170.020944
" + ], + "text/plain": [ + "shape: (88, 8)\n", + "┌────────────┬─────────┬────────────────┬───────────┬───────────┬───────┬───────────────┬──────────┐\n", + "│ Day ┆ Channel ┆ ExperimentGrou ┆ Negatives ┆ Positives ┆ Count ┆ ConversionRat ┆ CI │\n", + "│ --- ┆ --- ┆ p ┆ --- ┆ --- ┆ --- ┆ e ┆ --- │\n", + "│ date ┆ str ┆ --- ┆ u32 ┆ i32 ┆ u32 ┆ --- ┆ f64 │\n", + "│ ┆ ┆ str ┆ ┆ ┆ ┆ f64 ┆ │\n", + "╞════════════╪═════════╪════════════════╪═══════════╪═══════════╪═══════╪═══════════════╪══════════╡\n", + "│ 2024-11-25 ┆ Email ┆ Conversion-Con ┆ 381 ┆ 5 ┆ 391 ┆ 0.012953 ┆ 0.01128 │\n", + "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", + "│ 2024-11-25 ┆ Email ┆ Conversion-Tes ┆ 353 ┆ 4 ┆ 361 ┆ 0.011204 ┆ 0.010919 │\n", + "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", + "│ 2024-11-25 ┆ Web ┆ Conversion-Con ┆ 338 ┆ 2 ┆ 342 ┆ 0.005882 ┆ 0.008129 │\n", + "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", + "│ 2024-11-25 ┆ Web ┆ Conversion-Tes ┆ 357 ┆ 11 ┆ 379 ┆ 0.029891 ┆ 0.017399 │\n", + "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", + "│ 2024-11-26 ┆ Email ┆ Conversion-Con ┆ 1141 ┆ 21 ┆ 1183 ┆ 0.018072 ┆ 0.007659 │\n", + "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2024-12-15 ┆ Web ┆ Conversion-Tes ┆ 1084 ┆ 140 ┆ 1364 ┆ 0.114379 ┆ 0.01783 │\n", + "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", + "│ 2024-12-16 ┆ Email ┆ Conversion-Con ┆ 764 ┆ 59 ┆ 882 ┆ 0.071689 ┆ 0.017625 │\n", + "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", + "│ 2024-12-16 ┆ Email ┆ Conversion-Tes ┆ 781 ┆ 71 ┆ 923 ┆ 0.083333 ┆ 0.018559 │\n", + "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", + "│ 2024-12-16 ┆ Web ┆ Conversion-Con ┆ 781 ┆ 51 ┆ 883 ┆ 0.061298 ┆ 0.0163 │\n", + "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", + "│ 2024-12-16 ┆ Web ┆ Conversion-Tes ┆ 721 ┆ 82 ┆ 885 ┆ 0.102117 ┆ 0.020944 │\n", + "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", + "└────────────┴─────────┴────────────────┴───────────┴───────────┴───────┴───────────────┴──────────┘" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", "\n", @@ -392,9 +7820,1004 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "delta": { + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0, + 0.45 + ], + "y": [ + 0.575, + 1 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "EmailConversion-Control" + }, + "type": "indicator", + "value": 0 + }, + { + "delta": { + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0.55, + 1 + ], + "y": [ + 0.575, + 1 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "EmailConversion-Test" + }, + "type": "indicator", + "value": 0 + }, + { + "delta": { + "reference": 0.25, + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0, + 0.45 + ], + "y": [ + 0, + 0.425 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "bar": { + "color": "#EC9B00" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75, + "value": 0.25 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "WebConversion-Control" + }, + "type": "indicator", + "value": 0.20265143952401868 + }, + { + "delta": { + "reference": 0.25, + "valueformat": ",.2%" + }, + "domain": { + "x": [ + 0.55, + 1 + ], + "y": [ + 0, + 0.425 + ] + }, + "gauge": { + "axis": { + "tickformat": ",.2%" + }, + "bar": { + "color": "#EC9B00" + }, + "threshold": { + "line": { + "color": "red", + "width": 2 + }, + "thickness": 0.75, + "value": 0.25 + } + }, + "mode": "gauge+number+delta", + "number": { + "valueformat": ",.2%" + }, + "title": { + "text": "WebConversion-Test" + }, + "type": "indicator", + "value": 0.2007769945530279 + } + ], + "layout": { + "autosize": true, + "height": 540, + "margin": { + "b": 10, + "l": 10, + "r": 10, + "t": 120 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[CONV] Conversion (Channel/Model Type)" + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (4, 9)
ChannelExperimentGroupNegativesPositivesCountCTRStdErrNameCName
strstru32i32u32f64f64strstr
"Email""Conversion-Control"249570249570.00.0"EmailConversion-Control""Email_Conversion-Control"
"Email""Conversion-Test"250310250310.00.0"EmailConversion-Test""Email_Conversion-Test"
"Web""Conversion-Control"199685075301180.2026510.00254"WebConversion-Control""Web_Conversion-Control"
"Web""Conversion-Test"199555013299810.2007770.002535"WebConversion-Test""Web_Conversion-Test"
" + ], + "text/plain": [ + "shape: (4, 9)\n", + "┌─────────┬────────────┬───────────┬───────────┬───┬──────────┬──────────┬────────────┬────────────┐\n", + "│ Channel ┆ Experiment ┆ Negatives ┆ Positives ┆ … ┆ CTR ┆ StdErr ┆ Name ┆ CName │\n", + "│ --- ┆ Group ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ --- ┆ u32 ┆ i32 ┆ ┆ f64 ┆ f64 ┆ str ┆ str │\n", + "│ ┆ str ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "╞═════════╪════════════╪═══════════╪═══════════╪═══╪══════════╪══════════╪════════════╪════════════╡\n", + "│ Email ┆ Conversion ┆ 24957 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ EmailConve ┆ Email_Conv │\n", + "│ ┆ -Control ┆ ┆ ┆ ┆ ┆ ┆ rsion-Cont ┆ ersion-Con │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ rol ┆ trol │\n", + "│ Email ┆ Conversion ┆ 25031 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ EmailConve ┆ Email_Conv │\n", + "│ ┆ -Test ┆ ┆ ┆ ┆ ┆ ┆ rsion-Test ┆ ersion-Tes │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ t │\n", + "│ Web ┆ Conversion ┆ 19968 ┆ 5075 ┆ … ┆ 0.202651 ┆ 0.00254 ┆ WebConvers ┆ Web_Conver │\n", + "│ ┆ -Control ┆ ┆ ┆ ┆ ┆ ┆ ion-Contro ┆ sion-Contr │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ l ┆ ol │\n", + "│ Web ┆ Conversion ┆ 19955 ┆ 5013 ┆ … ┆ 0.200777 ┆ 0.002535 ┆ WebConvers ┆ Web_Conver │\n", + "│ ┆ -Test ┆ ┆ ┆ ┆ ┆ ┆ ion-Test ┆ sion-Test │\n", + "└─────────┴────────────┴───────────┴───────────┴───┴──────────┴──────────┴────────────┴────────────┘" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "positive_model_response = [\"Clicked\"]\n", "all_model_response = [\"Impression\", \"Pending\"]\n", @@ -509,9 +8932,1432 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ] + ], + "error_y": { + "array": { + "bdata": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", + "legendgroup": "Conversion-Control", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Control", + "offsetgroup": "Conversion-Control", + "orientation": "v", + "showlegend": true, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-11-25", + "2024-11-26", + "2024-11-27", + "2024-11-28", + "2024-11-29", + "2024-11-30", + "2024-12-01", + "2024-12-02", + "2024-12-03", + "2024-12-04", + "2024-12-05", + "2024-12-06", + "2024-12-07", + "2024-12-08", + "2024-12-09", + "2024-12-10", + "2024-12-11", + "2024-12-12", + "2024-12-13", + "2024-12-14", + "2024-12-15", + "2024-12-16" + ], + "xaxis": "x2", + "y": { + "bdata": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "dtype": "f8" + }, + "yaxis": "y2" + }, + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ], + [ + "Conversion-Control" + ] + ], + "error_y": { + "array": { + "bdata": "EnayRxUfpj+/qpINNPeWP8dIXiOeyZc/7qkrsYdemD836iN6tHWXP61dFbdTHJc//F8A635dlz8hTrDO6EKXP2dAV13pH5c/7QiiTOYLlz+I3ZTNW82WP15RQCmUJZc/0pJ7kYDhlj9C0FFlggeYPyDl8Rxx75c/DCQFd1pVlT/GXPuVkaqXPzoVEK6HEZc/ZxO5+6jYlj+N2VEilqyXP1XBEqTi7Jc/Fm4m19LDnD8=", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", + "legendgroup": "Conversion-Control", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Control", + "offsetgroup": "Conversion-Control", + "orientation": "v", + "showlegend": false, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-11-25", + "2024-11-26", + "2024-11-27", + "2024-11-28", + "2024-11-29", + "2024-11-30", + "2024-12-01", + "2024-12-02", + "2024-12-03", + "2024-12-04", + "2024-12-05", + "2024-12-06", + "2024-12-07", + "2024-12-08", + "2024-12-09", + "2024-12-10", + "2024-12-11", + "2024-12-12", + "2024-12-13", + "2024-12-14", + "2024-12-15", + "2024-12-16" + ], + "xaxis": "x", + "y": { + "bdata": "u7q6urq6yj/0UmdcGITJP7MpkujKMMs/2M4x0/2ZzD+q26G6HarLP3zPjwwt6Mg/sHdMDewdyz83ncrermDKP9vrlzsJBco/WIZlWIZlyD+Z802oAl3HPxXgZdojtMg/QnsJ7SW0xz8ZTw9mqGLMP+6c4c4Z7sw/NkrZKGWjxD9bf8haf8jKP6ZXba5xu8g/Np/EOLMIyD/zCu4xXCvKP3nPqvh1f8s/AAAAAAAAzD8=", + "dtype": "f8" + }, + "yaxis": "y" + }, + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ] + ], + "error_y": { + "array": { + "bdata": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", + "legendgroup": "Conversion-Test", + "marker": { + "color": "#EF553B", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Test", + "offsetgroup": "Conversion-Test", + "orientation": "v", + "showlegend": true, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-11-25", + "2024-11-26", + "2024-11-27", + "2024-11-28", + "2024-11-29", + "2024-11-30", + "2024-12-01", + "2024-12-02", + "2024-12-03", + "2024-12-04", + "2024-12-05", + "2024-12-06", + "2024-12-07", + "2024-12-08", + "2024-12-09", + "2024-12-10", + "2024-12-11", + "2024-12-12", + "2024-12-13", + "2024-12-14", + "2024-12-15", + "2024-12-16" + ], + "xaxis": "x2", + "y": { + "bdata": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "dtype": "f8" + }, + "yaxis": "y2" + }, + { + "alignmentgroup": "True", + "customdata": [ + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ], + [ + "Conversion-Test" + ] + ], + "error_y": { + "array": { + "bdata": "QEExHVDcpD9t4SRIA9uXP3SU0Z65FZc/Gc6Zkphklz/Eu0JPfGyXP9002MhZmZc/RQi/reZclz9k9nY8cmiXP0N7zUKimJc/mQG/1ZOzlz+PUBgKX+2WP5xxNGcWBJg/EzVwP2gPlz9iwRciBK+WP0wSXhSIxpY/HptRVciSlz+4CndGCWKWPyEFuuNKwJY/fsbiKaXJlz/Dq6R9zNWXPy6FgMN8KZc/Snq6UOO+mz8=", + "dtype": "f8" + } + }, + "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", + "legendgroup": "Conversion-Test", + "marker": { + "color": "#EF553B", + "pattern": { + "shape": "" + } + }, + "name": "Conversion-Test", + "offsetgroup": "Conversion-Test", + "orientation": "v", + "showlegend": false, + "textposition": "auto", + "type": "bar", + "x": [ + "2024-11-25", + "2024-11-26", + "2024-11-27", + "2024-11-28", + "2024-11-29", + "2024-11-30", + "2024-12-01", + "2024-12-02", + "2024-12-03", + "2024-12-04", + "2024-12-05", + "2024-12-06", + "2024-12-07", + "2024-12-08", + "2024-12-09", + "2024-12-10", + "2024-12-11", + "2024-12-12", + "2024-12-13", + "2024-12-14", + "2024-12-15", + "2024-12-16" + ], + "xaxis": "x", + "y": { + "bdata": "FrKQhSxkyT8KfdokKKfKPwCD61bsYso/QUXb7akAyT/OeVWIp0jKP9mAbEA2IMs/dL87BVxnyT8Obyd3IVPKP9WOzqTa0ck/O7ETO7ETyz92G8FmDLLHPzpm79bCTcs/cF5S0+qDyT+XdmmXdmnHP0WD+iFLzMg/CCL/04Y6yj+mShGJJbnHPzIHu/7epcg/dNY/hxHxyj8RharZ6m/KP+qUP+qUP8o/ZWxk96g6yD8=", + "dtype": "f8" + }, + "yaxis": "y" + } + ], + "layout": { + "annotations": [ + { + "font": {}, + "showarrow": false, + "text": "Web", + "textangle": 90, + "x": 0.98, + "xanchor": "left", + "xref": "paper", + "y": 0.2425, + "yanchor": "middle", + "yref": "paper" + }, + { + "font": {}, + "showarrow": false, + "text": "Email", + "textangle": 90, + "x": 0.98, + "xanchor": "left", + "xref": "paper", + "y": 0.7575000000000001, + "yanchor": "middle", + "yref": "paper" + } + ], + "barmode": "group", + "height": 640, + "hovermode": "x unified", + "legend": { + "title": { + "text": "ExperimentGroup" + }, + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermap": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermap" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "[ENG] Daily Click-through Rate with 95% confidence interval" + }, + "updatemenus": [ + { + "buttons": [ + { + "args": [ + "type", + "bar" + ], + "label": "Bar", + "method": "restyle" + }, + { + "args": [ + "type", + "line" + ], + "label": "Line", + "method": "restyle" + } + ], + "direction": "down", + "showactive": true + } + ], + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 0.98 + ], + "tickfont": { + "size": 10 + }, + "title": { + "text": "Day" + } + }, + "xaxis2": { + "anchor": "y2", + "domain": [ + 0, + 0.98 + ], + "matches": "x", + "showticklabels": false, + "tickfont": { + "size": 10 + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 0.485 + ], + "tickformat": ",.2%", + "title": { + "text": "CTR" + } + }, + "yaxis2": { + "anchor": "x2", + "domain": [ + 0.515, + 1 + ], + "matches": "y", + "tickformat": ",.2%", + "title": { + "text": "CTR" + } + } + } + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (88, 8)
DayChannelExperimentGroupNegativesPositivesCountCTRCI
datestrstru32i32u32f64f64
2024-11-25"Email""Conversion-Control"38603860.00.0
2024-11-25"Email""Conversion-Test"35703570.00.0
2024-11-25"Web""Conversion-Control"269714110.2088240.043206
2024-11-25"Web""Conversion-Test"295734410.198370.040743
2024-11-26"Email""Conversion-Control"1162011620.00.0
……………………
2024-12-15"Web""Conversion-Test"97325114750.2050650.022619
2024-12-16"Email""Conversion-Control"82308230.00.0
2024-12-16"Email""Conversion-Test"85208520.00.0
2024-12-16"Web""Conversion-Control"65018210140.218750.028091
2024-12-16"Web""Conversion-Test"6511529550.189290.027095
" + ], + "text/plain": [ + "shape: (88, 8)\n", + "┌────────────┬─────────┬────────────────────┬───────────┬───────────┬───────┬──────────┬──────────┐\n", + "│ Day ┆ Channel ┆ ExperimentGroup ┆ Negatives ┆ Positives ┆ Count ┆ CTR ┆ CI │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ date ┆ str ┆ str ┆ u32 ┆ i32 ┆ u32 ┆ f64 ┆ f64 │\n", + "╞════════════╪═════════╪════════════════════╪═══════════╪═══════════╪═══════╪══════════╪══════════╡\n", + "│ 2024-11-25 ┆ Email ┆ Conversion-Control ┆ 386 ┆ 0 ┆ 386 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-11-25 ┆ Email ┆ Conversion-Test ┆ 357 ┆ 0 ┆ 357 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-11-25 ┆ Web ┆ Conversion-Control ┆ 269 ┆ 71 ┆ 411 ┆ 0.208824 ┆ 0.043206 │\n", + "│ 2024-11-25 ┆ Web ┆ Conversion-Test ┆ 295 ┆ 73 ┆ 441 ┆ 0.19837 ┆ 0.040743 │\n", + "│ 2024-11-26 ┆ Email ┆ Conversion-Control ┆ 1162 ┆ 0 ┆ 1162 ┆ 0.0 ┆ 0.0 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2024-12-15 ┆ Web ┆ Conversion-Test ┆ 973 ┆ 251 ┆ 1475 ┆ 0.205065 ┆ 0.022619 │\n", + "│ 2024-12-16 ┆ Email ┆ Conversion-Control ┆ 823 ┆ 0 ┆ 823 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-12-16 ┆ Email ┆ Conversion-Test ┆ 852 ┆ 0 ┆ 852 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-12-16 ┆ Web ┆ Conversion-Control ┆ 650 ┆ 182 ┆ 1014 ┆ 0.21875 ┆ 0.028091 │\n", + "│ 2024-12-16 ┆ Web ┆ Conversion-Test ┆ 651 ┆ 152 ┆ 955 ┆ 0.18929 ┆ 0.027095 │\n", + "└────────────┴─────────┴────────────────────┴───────────┴───────────┴───────┴──────────┴──────────┘" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", "\n", diff --git a/examples/ih/ih_helper.py b/examples/ih/ih_helper.py index 38e8c5c5..c9918856 100644 --- a/examples/ih/ih_helper.py +++ b/examples/ih/ih_helper.py @@ -5,7 +5,7 @@ # Some day will move into a proper IH class -class ih_generator: +class interaction_history: interactions_period_days = 21 accept_rate = 0.2 accept_avg_duration_minutes = 10 @@ -17,8 +17,8 @@ def generate(self, n): now = datetime.datetime.now() - def _interpolate(min, max, i, n): - return min + (max - min) * i / (n - 1) + # def _interpolate(min, max, i, n): + # return min + (max - min) * i / (n - 1) def to_prpc_time_str(timestamp): @@ -28,6 +28,11 @@ def to_prpc_time_str(timestamp): ih_fake_impressions = pl.DataFrame( { "InteractionID": [str(int(1e9 + i)) for i in range(n)], + "pyChannel": random.choices(["Web", "Email"], k=n), + "pyIssue": "Acquisition", + "pyGroup": "Phones", + "pyName": "AppleIPhone1564GB", + "ExperimentGroup": ["Conversion-Test", "Conversion-Control"] * int(n / 2), "TimeStamp": [ (now - datetime.timedelta(days=i * self.interactions_period_days / n)) for i in range(n) @@ -38,12 +43,6 @@ def to_prpc_time_str(timestamp): "ConvertDurationDays": [ random.uniform(0, 2 * self.convert_avg_duration_days) for i in range(n) ], - "pyChannel": random.choices(["Web"], k=n), # random.choices(["Web", "Email"], k=n), - "pyIssue": "Acquisition", - "pyGroup": "Phones", - "pyName": "AppleIPhone1564GB", - "ExperimentGroup": ["Conversion-Test", "Conversion-Control"] * int(n / 2), - "pyOutcome": None, } ).with_columns( pyOutcome=pl.when(pl.col.pyChannel == "Web") From c4fff59749abf3d2d258e333739d7c2c542e2722 Mon Sep 17 00:00:00 2001 From: "Uyanik, Yusuf" Date: Mon, 16 Dec 2024 18:37:47 +0100 Subject: [PATCH 11/21] fix experimental charts in DA. --- .../pages/10_Business_Value_Analysis.py | 105 ++++++++++-------- .../pages/11_Business_Lever_Analysis.py | 5 +- .../pages/4_Action_Funnel.py | 1 - .../pages/8_Offer_Quality_Analysis.py | 6 +- .../pages/9_Thresholding_Analysis.py | 11 +- .../decision_analyzer/decision_data.py | 2 +- python/pdstools/decision_analyzer/plots.py | 4 +- .../decision_analyzer/table_definition.py | 4 - 8 files changed, 72 insertions(+), 66 deletions(-) diff --git a/python/pdstools/app/decision_analyzer/pages/10_Business_Value_Analysis.py b/python/pdstools/app/decision_analyzer/pages/10_Business_Value_Analysis.py index 35a85c72..d5f56ecd 100644 --- a/python/pdstools/app/decision_analyzer/pages/10_Business_Value_Analysis.py +++ b/python/pdstools/app/decision_analyzer/pages/10_Business_Value_Analysis.py @@ -1,52 +1,61 @@ -import polars as pl import streamlit as st -from da_streamlit_utils import get_current_scope_index, st_value_distribution -from utils import NBADScope_Mapping, ensure_data - -# TODO Finish up to show effect on proposition distribution (side to side) - -"# Business Value Analysis" - -""" -A closer look at the values associated with actions. - -* Is my value distribution very skewed? Are there actions with significantly different values than the others? -* What's the range of the values? - -""" -ensure_data() st.warning( - "Current sample data action values are artificial so the analysis is just an example." -) - -st.session_state["sidebar"] = st.sidebar - -scope_options = st.session_state.decision_data.getPossibleScopeValues() -if "scope" not in st.session_state: - st.session_state.scope = scope_options[0] - -valueData = st.session_state.decision_data.getValueDistributionData() - -with st.container(border=True): - st.plotly_chart( - st_value_distribution(valueData, st.session_state.scope), - use_container_width=True, - ) - - scope_index = get_current_scope_index(scope_options) - st.selectbox( - "Granularity:", - options=scope_options, - format_func=lambda option: NBADScope_Mapping[option], - index=scope_index, - key="scope", - ) - -"Actions having different values:" - -st.dataframe( - valueData.filter(pl.col("Value_min") != pl.col("Value_max")).collect(), - hide_index=True, - column_config=NBADScope_Mapping, + "In maintenance!!, please see: https://streamlit-dev.dsmcloud.io/Business%20Value%20Analysis for the older version. If the link doesn't work, contact Yusuf Uyanik." ) +# import polars as pl +# import streamlit as st + +# from da_streamlit_utils import ( +# get_current_scope_index, +# st_value_distribution, +# ensure_data, +# ) +# from pdstools.decision_analyzer.utils import NBADScope_Mapping + +# # TODO Finish up to show effect on proposition distribution (side to side) + +# "# Business Value Analysis" + +# """ +# A closer look at the values associated with actions. + +# * Is my value distribution very skewed? Are there actions with significantly different values than the others? +# * What's the range of the values? + +# """ +# ensure_data() +# st.warning( +# "Current sample data action values are artificial so the analysis is just an example." +# ) + +# st.session_state["sidebar"] = st.sidebar + +# scope_options = st.session_state.decision_data.getPossibleScopeValues() +# if "scope" not in st.session_state: +# st.session_state.scope = scope_options[0] + +# valueData = st.session_state.decision_data.getValueDistributionData() + +# with st.container(border=True): +# st.plotly_chart( +# st_value_distribution(valueData, st.session_state.scope), +# use_container_width=True, +# ) + +# scope_index = get_current_scope_index(scope_options) +# st.selectbox( +# "Granularity:", +# options=scope_options, +# format_func=lambda option: NBADScope_Mapping[option], +# index=scope_index, +# key="scope", +# ) + +# "Actions having different values:" + +# st.dataframe( +# valueData.filter(pl.col("Value_min") != pl.col("Value_max")).collect(), +# hide_index=True, +# column_config=NBADScope_Mapping, +# ) diff --git a/python/pdstools/app/decision_analyzer/pages/11_Business_Lever_Analysis.py b/python/pdstools/app/decision_analyzer/pages/11_Business_Lever_Analysis.py index b9523beb..3020e087 100644 --- a/python/pdstools/app/decision_analyzer/pages/11_Business_Lever_Analysis.py +++ b/python/pdstools/app/decision_analyzer/pages/11_Business_Lever_Analysis.py @@ -3,7 +3,10 @@ import polars as pl import streamlit as st -from utils import ensure_data, find_lever_value +from da_streamlit_utils import ( + ensure_data, +) +from pdstools.decision_analyzer.utils import find_lever_value # TODO not so sure what to do with this tool - maybe generalize to work across a selection not just a single action and figure out a multiplier # TODO but do show the effect of levering right away (distributions side to side) just like we should do in the thresholding analysis (share code) diff --git a/python/pdstools/app/decision_analyzer/pages/4_Action_Funnel.py b/python/pdstools/app/decision_analyzer/pages/4_Action_Funnel.py index 93fde563..92730742 100644 --- a/python/pdstools/app/decision_analyzer/pages/4_Action_Funnel.py +++ b/python/pdstools/app/decision_analyzer/pages/4_Action_Funnel.py @@ -36,7 +36,6 @@ st.session_state["sidebar"] = st.sidebar if "local_filters" in st.session_state: del st.session_state["local_filters"] - with st.session_state["sidebar"]: scope_options = st.session_state.decision_data.getPossibleScopeValues() stage_options = st.session_state.decision_data.getPossibleStageValues() diff --git a/python/pdstools/app/decision_analyzer/pages/8_Offer_Quality_Analysis.py b/python/pdstools/app/decision_analyzer/pages/8_Offer_Quality_Analysis.py index 07db5069..5d37cb50 100644 --- a/python/pdstools/app/decision_analyzer/pages/8_Offer_Quality_Analysis.py +++ b/python/pdstools/app/decision_analyzer/pages/8_Offer_Quality_Analysis.py @@ -1,13 +1,13 @@ import streamlit as st -from plots import getTrendChart, offer_quality_piecharts +from pdstools.decision_analyzer.plots import getTrendChart, offer_quality_piecharts from da_streamlit_utils import ( get_current_scope_index, get_current_stage_index, + ensure_data, ) -from utils import ( +from pdstools.decision_analyzer.utils import ( NBADScope_Mapping, - ensure_data, filtered_action_counts, ) diff --git a/python/pdstools/app/decision_analyzer/pages/9_Thresholding_Analysis.py b/python/pdstools/app/decision_analyzer/pages/9_Thresholding_Analysis.py index 6bcd938b..edf87779 100644 --- a/python/pdstools/app/decision_analyzer/pages/9_Thresholding_Analysis.py +++ b/python/pdstools/app/decision_analyzer/pages/9_Thresholding_Analysis.py @@ -2,8 +2,7 @@ import polars as pl import streamlit as st -from plots import distribution, threshold_deciles -from utils import ensure_data +from da_streamlit_utils import ensure_data # TODO Interactive Thresholding isn't working properly yet. Also show the total numbers. # TODO Instead of priority/propensity side to side have a drop-down to select which property to show @@ -80,7 +79,9 @@ # st.dataframe(plotData) st.plotly_chart( - threshold_deciles(threshold_deciles_data, thresholding_mapping[thresholding_on]), + st.session_state.decision_data.plot.threshold_deciles( + thresholding_on, thresholding_mapping[thresholding_on] + ), use_container_width=True, ) @@ -98,13 +99,11 @@ ), # Hmm, probalby not the right way # additional_filters=((pl.col(thresholding_on).list.eval(pl.element() > current_threshold)).list.any()), ) -# st.write(xxx.head().collect()) st.write( - distribution( + st.session_state.decision_data.plot.distribution( xxx, scope="pyIssue", breakdown="pyGroup", - title="Effect of Thresholding", horizontal=True, ) ) diff --git a/python/pdstools/decision_analyzer/decision_data.py b/python/pdstools/decision_analyzer/decision_data.py index 71b1a6b9..1664728a 100644 --- a/python/pdstools/decision_analyzer/decision_data.py +++ b/python/pdstools/decision_analyzer/decision_data.py @@ -81,7 +81,7 @@ def __init__(self, raw_data: pl.LazyFrame): "pyIssue", "pyGroup", "pyName", - # "pyTreatment", # should be in there dependent on what's in the data + "pyTreatment", # should be in there dependent on what's in the data "pyChannel", "pyDirection", "pxComponentName", diff --git a/python/pdstools/decision_analyzer/plots.py b/python/pdstools/decision_analyzer/plots.py index 0ba0d41f..d4ed8638 100644 --- a/python/pdstools/decision_analyzer/plots.py +++ b/python/pdstools/decision_analyzer/plots.py @@ -13,8 +13,8 @@ class Plot: def __init__(self, decision_data): self._decision_data = decision_data - def threshold_deciles(self, thresholding_name, return_df=False): - df = self._decision_data.whatever_preprocessing + def threshold_deciles(self, thresholding_on, thresholding_name, return_df=False): + df = self._decision_data.getThresholdingData(thresholding_on) if return_df: return df diff --git a/python/pdstools/decision_analyzer/table_definition.py b/python/pdstools/decision_analyzer/table_definition.py index 57ae6bbb..bd5f30a3 100644 --- a/python/pdstools/decision_analyzer/table_definition.py +++ b/python/pdstools/decision_analyzer/table_definition.py @@ -345,10 +345,6 @@ class TableConfig(TypedDict): "Arbitration.TopSelection", "TreatmentPlacements", "Channels.ExtensionPoint", - "Channels.ExtensionPoint", - "Channels.ExtensionPoint", - "Channels.ExtensionPoint", - "Channels.ExtensionPoint", "ContactPolicies.ChannelLimits", "ContactPolicies.ExtensionPoint", "FinalLimitsAndBundlingPostExtensionPoint", From 6279aedf342d9c6c60c4eb87226ae1a58ca10cd8 Mon Sep 17 00:00:00 2001 From: "Uyanik, Yusuf" Date: Mon, 16 Dec 2024 19:03:05 +0100 Subject: [PATCH 12/21] temp solution for offer quality --- python/pdstools/decision_analyzer/plots.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/pdstools/decision_analyzer/plots.py b/python/pdstools/decision_analyzer/plots.py index d4ed8638..50e5c856 100644 --- a/python/pdstools/decision_analyzer/plots.py +++ b/python/pdstools/decision_analyzer/plots.py @@ -590,12 +590,17 @@ def offer_quality_piecharts( "only_irrelevant_actions", "has_no_offers", ] - df = ( + all_frames = ( df.group_by("pxEngagementStage") .agg(pl.sum(value_finder_names)) .collect() .partition_by("pxEngagementStage", as_dict=True) ) + # TODO Temporary solution to fit the pie charts into the screen, pick only first 5 stages + df = {} + NBADStages_FilterView = NBADStages_FilterView[:5] + for stage in NBADStages_FilterView[:5]: + df[(stage,)] = all_frames[(stage,)] if return_df: return df @@ -608,7 +613,7 @@ def offer_quality_piecharts( ) for i, stage in enumerate(NBADStages_FilterView): - plotdf = df[stage].drop("pxEngagementStage") + plotdf = df[(stage,)].drop("pxEngagementStage") fig.add_trace( go.Pie( values=list(plotdf.to_numpy())[0], From 6f89c5dca85e9e207be3a2f5ac3101cedb0817d5 Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Tue, 17 Dec 2024 09:37:34 +0100 Subject: [PATCH 13/21] Fixing up --- .../ih/Conversion_Modeling_Reporting.ipynb | 927 ++++++++++-------- examples/ih/ih_helper.py | 6 + python/pdstools/utils/cdh_utils.py | 131 +-- 3 files changed, 574 insertions(+), 490 deletions(-) diff --git a/examples/ih/Conversion_Modeling_Reporting.ipynb b/examples/ih/Conversion_Modeling_Reporting.ipynb index 644c6a04..0f61ef7f 100644 --- a/examples/ih/Conversion_Modeling_Reporting.ipynb +++ b/examples/ih/Conversion_Modeling_Reporting.ipynb @@ -3911,7 +3911,7 @@ "from plotly.subplots import make_subplots\n", "\n", "plotly.offline.init_notebook_mode()\n", - "pio.renderers.default = \"vscode\"\n" + "pio.renderers.default = \"vscode\"" ] }, { @@ -3929,20 +3929,20 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (4, 7)
ChannelExperimentGroupClickedPendingConversionImpressionAccepted
strstru32u32u32u32u32
"Web""Conversion-Test"5013null230824968null
"Email""Conversion-Control"null249571370null4937
"Email""Conversion-Test"null250312303null4966
"Web""Conversion-Control"5075null140525043null
" + "shape: (4, 7)
ChannelExperimentGroupConversionPendingAcceptedClickedImpression
strstru32u32u32u32u32
"Web""Conversion-Test"2310nullnull494824929
"Web""Conversion-Control"1384nullnull503225129
"Email""Conversion-Test"2358250705066nullnull
"Email""Conversion-Control"1379248714950nullnull
" ], "text/plain": [ "shape: (4, 7)\n", - "┌─────────┬────────────────────┬─────────┬─────────┬────────────┬────────────┬──────────┐\n", - "│ Channel ┆ ExperimentGroup ┆ Clicked ┆ Pending ┆ Conversion ┆ Impression ┆ Accepted │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ u32 ┆ u32 ┆ u32 ┆ u32 ┆ u32 │\n", - "╞═════════╪════════════════════╪═════════╪═════════╪════════════╪════════════╪══════════╡\n", - "│ Web ┆ Conversion-Test ┆ 5013 ┆ null ┆ 2308 ┆ 24968 ┆ null │\n", - "│ Email ┆ Conversion-Control ┆ null ┆ 24957 ┆ 1370 ┆ null ┆ 4937 │\n", - "│ Email ┆ Conversion-Test ┆ null ┆ 25031 ┆ 2303 ┆ null ┆ 4966 │\n", - "│ Web ┆ Conversion-Control ┆ 5075 ┆ null ┆ 1405 ┆ 25043 ┆ null │\n", - "└─────────┴────────────────────┴─────────┴─────────┴────────────┴────────────┴──────────┘" + "┌─────────┬────────────────────┬────────────┬─────────┬──────────┬─────────┬────────────┐\n", + "│ Channel ┆ ExperimentGroup ┆ Conversion ┆ Pending ┆ Accepted ┆ Clicked ┆ Impression │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ u32 ┆ u32 ┆ u32 ┆ u32 ┆ u32 │\n", + "╞═════════╪════════════════════╪════════════╪═════════╪══════════╪═════════╪════════════╡\n", + "│ Web ┆ Conversion-Test ┆ 2310 ┆ null ┆ null ┆ 4948 ┆ 24929 │\n", + "│ Web ┆ Conversion-Control ┆ 1384 ┆ null ┆ null ┆ 5032 ┆ 25129 │\n", + "│ Email ┆ Conversion-Test ┆ 2358 ┆ 25070 ┆ 5066 ┆ null ┆ null │\n", + "│ Email ┆ Conversion-Control ┆ 1379 ┆ 24871 ┆ 4950 ┆ null ┆ null │\n", + "└─────────┴────────────────────┴────────────┴─────────┴──────────┴─────────┴────────────┘" ] }, "execution_count": 2, @@ -3955,16 +3955,28 @@ "\n", "# we really only need a few columns\n", "# Outcome outcomes: Conversionm, Impression, Pending\n", - "ih_cols = [\"pyOutcome\", \"pxOutcomeTime\", \"pyChannel\", \"pyIssue\", \"pyGroup\", \"pyName\", \"ExperimentGroup\"]\n", + "ih_cols = [\n", + " \"pyOutcome\",\n", + " \"pxOutcomeTime\",\n", + " \"pyChannel\",\n", + " \"pyIssue\",\n", + " \"pyGroup\",\n", + " \"pyName\",\n", + " \"ExperimentGroup\",\n", + "]\n", "\n", - "ih_export_file = Path(\"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip \")\n", + "ih_export_file = Path(\n", + " \"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip \"\n", + ")\n", "if not ih_export_file.exists():\n", " ih = interaction_history().generate(100000).select(ih_cols)\n", "else:\n", " ih = read_ds_export(ih_export_file).select(ih_cols)\n", "ih = cdh_utils._polars_capitalize(ih)\n", - "ih = ih.filter(pl.col('ExperimentGroup').is_not_null())\n", - "ih.collect().group_by([\"Channel\", \"ExperimentGroup\", \"Outcome\"]).agg(pl.len()).pivot(on=\"Outcome\",index=[\"Channel\",\"ExperimentGroup\"])" + "ih = ih.filter(pl.col(\"ExperimentGroup\").is_not_null())\n", + "ih.collect().group_by([\"Channel\", \"ExperimentGroup\", \"Outcome\"]).agg(pl.len()).pivot(\n", + " on=\"Outcome\", index=[\"Channel\", \"ExperimentGroup\"]\n", + ")" ] }, { @@ -3989,7 +4001,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (9, 13)
statisticOutcomeOutcomeTimeChannelIssueGroupNameExperimentGroupOutcomeDateTimeDayMonthYearQuarter
strstrstrstrstrstrstrstrstrstrstrstrstr
"count""127376""127376""127376""127376""127376""127376""127376""127376""127376""127376""127376""127376"
"null_count""0""0""0""0""0""0""0""0""0""0""0""0"
"mean"nullnullnullnullnullnullnull"2024-12-06 05:42:04.989000""2024-12-05 17:42:58.470000"nullnullnull
"std"nullnullnullnullnullnullnullnullnullnullnullnull
"min""Accepted""20241125T164124""Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control""2024-11-25 16:41:24""2024-11-25""2024-11""2024""2024_Q4"
"25%"nullnullnullnullnullnullnull"2024-12-01 00:13:56""2024-12-01"nullnullnull
"50%"nullnullnullnullnullnullnull"2024-12-06 05:43:42""2024-12-06"nullnullnull
"75%"nullnullnullnullnullnullnull"2024-12-11 11:42:22""2024-12-11"nullnullnull
"max""Pending""20241216T164048""Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test""2024-12-16 16:40:48""2024-12-16""2024-12""2024""2024_Q4"
" + "shape: (9, 13)
statisticOutcomeOutcomeTimeChannelIssueGroupNameExperimentGroupOutcomeDateTimeDayMonthYearQuarter
strstrstrstrstrstrstrstrstrstrstrstrstr
"count""127426""127426""127426""127426""127426""127426""127426""127426""127426""127426""127426""127426"
"null_count""0""0""0""0""0""0""0""0""0""0""0""0"
"mean"nullnullnullnullnullnullnull"2024-12-06 05:34:26.383000""2024-12-05 17:35:55.434000"nullnullnull
"std"nullnullnullnullnullnullnullnullnullnullnullnull
"min""Accepted""20241125T164722""Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control""2024-11-25 16:47:22""2024-11-25""2024-11""2024""2024_Q4"
"25%"nullnullnullnullnullnullnull"2024-12-01 00:15:40""2024-12-01"nullnullnull
"50%"nullnullnullnullnullnullnull"2024-12-06 05:10:39""2024-12-06"nullnullnull
"75%"nullnullnullnullnullnullnull"2024-12-11 11:11:34""2024-12-11"nullnullnull
"max""Pending""20241216T164646""Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test""2024-12-16 16:46:46""2024-12-16""2024-12""2024""2024_Q4"
" ], "text/plain": [ "shape: (9, 13)\n", @@ -3998,19 +4010,19 @@ "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ str ┆ str ┆ str ┆ str ┆ ┆ str ┆ str ┆ str ┆ str │\n", "╞════════════╪══════════╪═══════════════╪═════════╪═══╪═══════════════╪═════════╪════════╪═════════╡\n", - "│ count ┆ 127376 ┆ 127376 ┆ 127376 ┆ … ┆ 127376 ┆ 127376 ┆ 127376 ┆ 127376 │\n", + "│ count ┆ 127426 ┆ 127426 ┆ 127426 ┆ … ┆ 127426 ┆ 127426 ┆ 127426 ┆ 127426 │\n", "│ null_count ┆ 0 ┆ 0 ┆ 0 ┆ … ┆ 0 ┆ 0 ┆ 0 ┆ 0 │\n", "│ mean ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-05 ┆ null ┆ null ┆ null │\n", - "│ ┆ ┆ ┆ ┆ ┆ 17:42:58.4700 ┆ ┆ ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ 17:35:55.4340 ┆ ┆ ┆ │\n", "│ ┆ ┆ ┆ ┆ ┆ 00 ┆ ┆ ┆ │\n", "│ std ┆ null ┆ null ┆ null ┆ … ┆ null ┆ null ┆ null ┆ null │\n", - "│ min ┆ Accepted ┆ 20241125T1641 ┆ Email ┆ … ┆ 2024-11-25 ┆ 2024-11 ┆ 2024 ┆ 2024_Q4 │\n", - "│ ┆ ┆ 24 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ min ┆ Accepted ┆ 20241125T1647 ┆ Email ┆ … ┆ 2024-11-25 ┆ 2024-11 ┆ 2024 ┆ 2024_Q4 │\n", + "│ ┆ ┆ 22 ┆ ┆ ┆ ┆ ┆ ┆ │\n", "│ 25% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-01 ┆ null ┆ null ┆ null │\n", "│ 50% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-06 ┆ null ┆ null ┆ null │\n", "│ 75% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-11 ┆ null ┆ null ┆ null │\n", - "│ max ┆ Pending ┆ 20241216T1640 ┆ Web ┆ … ┆ 2024-12-16 ┆ 2024-12 ┆ 2024 ┆ 2024_Q4 │\n", - "│ ┆ ┆ 48 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ max ┆ Pending ┆ 20241216T1646 ┆ Web ┆ … ┆ 2024-12-16 ┆ 2024-12 ┆ 2024 ┆ 2024_Q4 │\n", + "│ ┆ ┆ 46 ┆ ┆ ┆ ┆ ┆ ┆ │\n", "└────────────┴──────────┴───────────────┴─────────┴───┴───────────────┴─────────┴────────┴─────────┘" ] }, @@ -4054,8 +4066,17 @@ "source": [ "positive_model_response = [\"Conversion\"]\n", "all_model_response = [\"Impression\", \"Pending\"]\n", - "group_by = [\"Day\", \"Month\", \"Year\", \"Quarter\", \"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", - "\n" + "group_by = [\n", + " \"Day\",\n", + " \"Month\",\n", + " \"Year\",\n", + " \"Quarter\",\n", + " \"Channel\",\n", + " \"Issue\",\n", + " \"Group\",\n", + " \"Name\",\n", + " \"ExperimentGroup\",\n", + "]" ] }, { @@ -4102,7 +4123,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (4, 7)
ChannelExperimentGroupNegativesPositivesCountConversionRateStdErr
strstru32i32u32f64f64
"Email""Conversion-Control"235871370263270.0548940.001442
"Email""Conversion-Test"227282303273340.0920060.001827
"Web""Conversion-Control"236381405264480.0561040.001454
"Web""Conversion-Test"226602308272760.0924380.001833
" + "shape: (4, 7)
ChannelExperimentGroupNegativesPositivesCountConversionRateStdErr
strstru32i32u32f64f64
"Email""Conversion-Control"234921379262500.0554460.001451
"Email""Conversion-Test"227122358274280.0940570.001844
"Web""Conversion-Control"237451384265130.0550760.001439
"Web""Conversion-Test"226192310272390.0926630.001836
" ], "text/plain": [ "shape: (4, 7)\n", @@ -4111,10 +4132,10 @@ "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ str ┆ str ┆ u32 ┆ i32 ┆ u32 ┆ f64 ┆ f64 │\n", "╞═════════╪════════════════════╪═══════════╪═══════════╪═══════╪════════════════╪══════════╡\n", - "│ Email ┆ Conversion-Control ┆ 23587 ┆ 1370 ┆ 26327 ┆ 0.054894 ┆ 0.001442 │\n", - "│ Email ┆ Conversion-Test ┆ 22728 ┆ 2303 ┆ 27334 ┆ 0.092006 ┆ 0.001827 │\n", - "│ Web ┆ Conversion-Control ┆ 23638 ┆ 1405 ┆ 26448 ┆ 0.056104 ┆ 0.001454 │\n", - "│ Web ┆ Conversion-Test ┆ 22660 ┆ 2308 ┆ 27276 ┆ 0.092438 ┆ 0.001833 │\n", + "│ Email ┆ Conversion-Control ┆ 23492 ┆ 1379 ┆ 26250 ┆ 0.055446 ┆ 0.001451 │\n", + "│ Email ┆ Conversion-Test ┆ 22712 ┆ 2358 ┆ 27428 ┆ 0.094057 ┆ 0.001844 │\n", + "│ Web ┆ Conversion-Control ┆ 23745 ┆ 1384 ┆ 26513 ┆ 0.055076 ┆ 0.001439 │\n", + "│ Web ┆ Conversion-Test ┆ 22619 ┆ 2310 ┆ 27239 ┆ 0.092663 ┆ 0.001836 │\n", "└─────────┴────────────────────┴───────────┴───────────┴───────┴────────────────┴──────────┘" ] }, @@ -4125,30 +4146,32 @@ ], "source": [ "gauge_group_by = [\"Channel\", \"ExperimentGroup\"]\n", - "reference = {'WebConversion-Test' : 0.055, 'WebConversion-Control' : 0.055}\n", + "reference = {\"WebConversion-Test\": 0.055, \"WebConversion-Control\": 0.055, \"EmailConversion-Test\": 0.09, \"EmailConversion-Control\": 0.09}\n", "gauge_data = (\n", - " ih_analysis.group_by(gauge_group_by)\n", - " .agg(\n", - " pl.sum([\"Negatives\", \"Positives\", \"Count\"])\n", - " )\n", - " .with_columns(\n", - " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"ConversionRate\")]\n", - " )\n", - " .with_columns(\n", - " [\n", + " ih_analysis.group_by(gauge_group_by)\n", + " .agg(pl.sum([\"Negatives\", \"Positives\", \"Count\"]))\n", + " .with_columns(\n", + " [\n", + " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", + " \"ConversionRate\"\n", + " )\n", + " ]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", " (\n", " (\n", - " (\n", - " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", - " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " ).sqrt()\n", - " )\n", - " ).alias(\"StdErr\")\n", - " ]\n", - " )\n", - " .sort(gauge_group_by, descending=False)\n", - " .collect()\n", + " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", + " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", + " ).sqrt()\n", + " )\n", + " ).alias(\"StdErr\")\n", + " ]\n", " )\n", + " .sort(gauge_group_by, descending=False)\n", + " .collect()\n", + ")\n", "gauge_data" ] }, @@ -4166,6 +4189,7 @@ "data": [ { "delta": { + "reference": 0.09, "valueformat": ",.2%" }, "domain": { @@ -4182,12 +4206,16 @@ "axis": { "tickformat": ",.2%" }, + "bar": { + "color": "#EC5300" + }, "threshold": { "line": { "color": "red", "width": 2 }, - "thickness": 0.75 + "thickness": 0.75, + "value": 0.09 } }, "mode": "gauge+number+delta", @@ -4198,10 +4226,11 @@ "text": "EmailConversion-Control" }, "type": "indicator", - "value": 0.05489441839964739 + "value": 0.05544610188573037 }, { "delta": { + "reference": 0.09, "valueformat": ",.2%" }, "domain": { @@ -4223,7 +4252,8 @@ "color": "red", "width": 2 }, - "thickness": 0.75 + "thickness": 0.75, + "value": 0.09 } }, "mode": "gauge+number+delta", @@ -4234,7 +4264,7 @@ "text": "EmailConversion-Test" }, "type": "indicator", - "value": 0.09200591266829132 + "value": 0.09405664140406861 }, { "delta": { @@ -4272,7 +4302,7 @@ "text": "WebConversion-Control" }, "type": "indicator", - "value": 0.056103501976600245 + "value": 0.05507580882645549 }, { "delta": { @@ -4310,7 +4340,7 @@ "text": "WebConversion-Test" }, "type": "indicator", - "value": 0.09243832105094521 + "value": 0.0926631633840106 } ], "layout": { @@ -5113,55 +5143,61 @@ "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", "\n", "gauge_data = gauge_data.with_columns(\n", - " pl.concat_str(gauge_group_by).alias('Name'),\n", + " pl.concat_str(gauge_group_by).alias(\"Name\"),\n", " # pl.concat_str(gauge_group_by, separator = \"_\").alias('CName'),\n", - " )\n", + ")\n", "\n", - "fig = make_subplots(rows=rows,\n", - " cols=cols,\n", - " specs=[[{\"type\": \"indicator\"} for c in range(cols)] for t in range(rows)]\n", - " )\n", + "fig = make_subplots(\n", + " rows=rows,\n", + " cols=cols,\n", + " specs=[[{\"type\": \"indicator\"} for c in range(cols)] for t in range(rows)],\n", + ")\n", "fig.update_layout(\n", - " height=270 * rows,\n", - " autosize=True,\n", - " title='[CONV] Conversion (Channel/Model Type)',\n", - " margin=dict(b=10, t=120, l=10, r=10))\n", + " height=270 * rows,\n", + " autosize=True,\n", + " title=\"[CONV] Conversion (Channel/Model Type)\",\n", + " margin=dict(b=10, t=120, l=10, r=10),\n", + ")\n", "index = 0\n", "for row in gauge_data.iter_rows(named=True):\n", - " ref_value = reference.get(row['Name'], None)\n", - " gauge = {\n", - " 'axis': {'tickformat': ',.2%'},\n", - " 'threshold': {\n", - " 'line': {'color': \"red\", 'width': 2},\n", - " 'thickness': 0.75,\n", - " 'value': ref_value\n", + " ref_value = reference.get(row[\"Name\"], None)\n", + " gauge = {\n", + " \"axis\": {\"tickformat\": \",.2%\"},\n", + " \"threshold\": {\n", + " \"line\": {\"color\": \"red\", \"width\": 2},\n", + " \"thickness\": 0.75,\n", + " \"value\": ref_value,\n", + " },\n", + " }\n", + " if ref_value:\n", + " if row[\"ConversionRate\"] < ref_value:\n", + " gauge = {\n", + " \"axis\": {\"tickformat\": \",.2%\"},\n", + " \"bar\": {\n", + " \"color\": (\n", + " \"#EC5300\"\n", + " if row[\"ConversionRate\"] < (0.75 * ref_value)\n", + " else \"#EC9B00\"\n", + " )\n", + " },\n", + " \"threshold\": {\n", + " \"line\": {\"color\": \"red\", \"width\": 2},\n", + " \"thickness\": 0.75,\n", + " \"value\": ref_value,\n", + " },\n", " }\n", - " }\n", - " if ref_value:\n", - " if row['ConversionRate'] < ref_value:\n", - " gauge = {\n", - " 'axis': {'tickformat': ',.2%'},\n", - " 'bar': {'color': '#EC5300' if row['ConversionRate'] < (0.75 * ref_value) else '#EC9B00'},\n", - " 'threshold': {\n", - " 'line': {'color': \"red\", 'width': 2},\n", - " 'thickness': 0.75,\n", - " 'value': ref_value\n", - " }\n", - " }\n", "\n", - " trace1 = go.Indicator(mode=\"gauge+number+delta\",\n", - " number={'valueformat': \",.2%\"},\n", - " value=row['ConversionRate'],\n", - " delta={'reference': ref_value, 'valueformat': \",.2%\"},\n", - " title={'text': row['Name']},\n", - " gauge=gauge,\n", - " )\n", - " r, c = divmod(index, cols)\n", - " fig.add_trace(\n", - " trace1,\n", - " row=(r + 1), col=(c + 1)\n", - " )\n", - " index = index + 1\n", + " trace1 = go.Indicator(\n", + " mode=\"gauge+number+delta\",\n", + " number={\"valueformat\": \",.2%\"},\n", + " value=row[\"ConversionRate\"],\n", + " delta={\"reference\": ref_value, \"valueformat\": \",.2%\"},\n", + " title={\"text\": row[\"Name\"]},\n", + " gauge=gauge,\n", + " )\n", + " r, c = divmod(index, cols)\n", + " fig.add_trace(trace1, row=(r + 1), col=(c + 1))\n", + " index = index + 1\n", "\n", "fig.show()" ] @@ -5189,82 +5225,82 @@ "branchvalues": "total", "customdata": [ [ - "0.0018330411793641235", - "2308", - "22660", - 0.09243832105094521 + "0.0014391011575603573", + "1384", + "23745", + 0.05507580882645549 ], [ - "0.0014541660330504179", - "1405", - "23638", - 0.05610350197660025 + "0.0018436058995561235", + "2358", + "22712", + 0.09405664140406861 ], [ - "0.001826881083957284", - "2303", - "22728", - 0.09200591266829132 + "0.0018364766203625416", + "2310", + "22619", + 0.0926631633840106 ], [ - "0.0014418101169569632", - "1370", - "23587", - 0.05489441839964739 + "0.0014511164251962663", + "1379", + "23492", + 0.05544610188573037 ], [ "(?)", "(?)", "(?)", - 0.07379838188037106 + 0.07517503884144745 ], [ "(?)", "(?)", "(?)", - 0.07455090956114035 + 0.07412332242210297 ], [ "(?)", "(?)", "(?)", - 0.07379838188037106 + 0.07412332242210297 ], [ "(?)", "(?)", "(?)", - 0.07455090956114035 + 0.07517503884144745 ], [ "(?)", "(?)", "(?)", - 0.07455090956114035 + 0.07517503884144745 ], [ "(?)", "(?)", "(?)", - 0.07379838188037106 + 0.07412332242210297 ], [ "(?)", "(?)", "(?)", - 0.07455090956114035 + 0.07517503884144745 ], [ "(?)", "(?)", "(?)", - 0.07379838188037106 + 0.07412332242210297 ], [ "(?)", "(?)", "(?)", - 0.07417486646501184 + 0.07464881840979332 ] ], "domain": { @@ -5279,24 +5315,24 @@ }, "hovertemplate": "labels=%{label}
Count=%{value}
parent=%{parent}
id=%{id}
StdErr=%{customdata[0]}
Positives=%{customdata[1]}
Negatives=%{customdata[2]}
ConversionRate=%{color}", "ids": [ - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Control", "ALL/Email/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", "ALL/Email/Acquisition/Phones/AppleIPhone1564GB/Conversion-Control", "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Email/Acquisition/Phones", "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition", + "ALL/Email/Acquisition/Phones", "ALL/Email/Acquisition", - "ALL/Web", + "ALL/Web/Acquisition", "ALL/Email", + "ALL/Web", "ALL" ], "labels": [ - "Conversion-Test", "Conversion-Control", "Conversion-Test", + "Conversion-Test", "Conversion-Control", "AppleIPhone1564GB", "AppleIPhone1564GB", @@ -5304,29 +5340,29 @@ "Phones", "Acquisition", "Acquisition", - "Web", "Email", + "Web", "ALL" ], "marker": { "coloraxis": "coloraxis", "colors": { - "bdata": "8prPrQmqtz+Zc1wkmbmsP/Pt8hGzjbc/g9OEBx8brD+Lhaxkc+SyPzTAc7bEFbM/i4WsZHPksj80wHO2xBWzPzTAc7bEFbM/i4WsZHPksj80wHO2xBWzP4uFrGRz5LI/OgOnwR/9sj8=", + "bdata": "NLZre+UyrD+8VM2WGBS4Pxg6R+jFuLc/O22UdG5jrD8Vr0zdqz6zP+F9rP2++bI/4X2s/b75sj8Vr0zdqz6zPxWvTN2rPrM/4X2s/b75sj8Vr0zdqz6zP+F9rP2++bI/kE7BWS8csz8=", "dtype": "f8" } }, "name": "", "parents": [ - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", + "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", "ALL/Email/Acquisition/Phones", "ALL/Web/Acquisition/Phones", - "ALL/Email/Acquisition", "ALL/Web/Acquisition", - "ALL/Web", + "ALL/Email/Acquisition", "ALL/Email", + "ALL/Web", "ALL", "ALL", "" @@ -5334,7 +5370,7 @@ "textinfo": "label+value+percent parent+percent root", "type": "treemap", "values": { - "bdata": "AAAAAACj2kAAAAAAANTZQAAAAACAsdpAAAAAAMC12UAAAAAAoDPqQAAAAACAO+pAAAAAAKAz6kAAAAAAgDvqQAAAAACAO+pAAAAAAKAz6kAAAAAAgDvqQAAAAACgM+pAAAAAAJA3+kA=", + "bdata": "AAAAAEDk2UAAAAAAAMnaQAAAAADAmdpAAAAAAICi2UAAAAAAwDXqQAAAAAAAP+pAAAAAAAA/6kAAAAAAwDXqQAAAAADANepAAAAAAAA/6kAAAAAAwDXqQAAAAAAAP+pAAAAAAGA6+kA=", "dtype": "f8" } } @@ -6198,7 +6234,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (4, 10)
ChannelIssueGroupNameExperimentGroupNegativesPositivesCountConversionRateStdErr
strstrstrstrstru32i32u32f64f64
"Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control"235871370263270.0548940.001442
"Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test"227282303273340.0920060.001827
"Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control"236381405264480.0561040.001454
"Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test"226602308272760.0924380.001833
" + "shape: (4, 10)
ChannelIssueGroupNameExperimentGroupNegativesPositivesCountConversionRateStdErr
strstrstrstrstru32i32u32f64f64
"Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control"234921379262500.0554460.001451
"Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test"227122358274280.0940570.001844
"Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control"237451384265130.0550760.001439
"Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test"226192310272390.0926630.001836
" ], "text/plain": [ "shape: (4, 10)\n", @@ -6208,13 +6244,13 @@ "│ str ┆ str ┆ str ┆ str ┆ ┆ i32 ┆ u32 ┆ --- ┆ f64 │\n", "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ f64 ┆ │\n", "╞═════════╪═════════════╪════════╪═══════════════╪═══╪═══════════╪═══════╪══════════════╪══════════╡\n", - "│ Email ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 1370 ┆ 26327 ┆ 0.054894 ┆ 0.001442 │\n", + "│ Email ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 1379 ┆ 26250 ┆ 0.055446 ┆ 0.001451 │\n", "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", - "│ Email ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 2303 ┆ 27334 ┆ 0.092006 ┆ 0.001827 │\n", + "│ Email ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 2358 ┆ 27428 ┆ 0.094057 ┆ 0.001844 │\n", "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", - "│ Web ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 1405 ┆ 26448 ┆ 0.056104 ┆ 0.001454 │\n", + "│ Web ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 1384 ┆ 26513 ┆ 0.055076 ┆ 0.001439 │\n", "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", - "│ Web ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 2308 ┆ 27276 ┆ 0.092438 ┆ 0.001833 │\n", + "│ Web ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 2310 ┆ 27239 ┆ 0.092663 ┆ 0.001836 │\n", "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", "└─────────┴─────────────┴────────┴───────────────┴───┴───────────┴───────┴──────────────┴──────────┘" ] @@ -6228,41 +6264,48 @@ "treemap_group_by = [\"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", "\n", "treemap_data = (\n", - " ih_analysis.group_by(treemap_group_by)\n", - " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\")\n", - " )\n", - " .with_columns(\n", - " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"ConversionRate\")]\n", - " )\n", - " .with_columns(\n", - " [\n", + " ih_analysis.group_by(treemap_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\"),\n", + " )\n", + " .with_columns(\n", + " [\n", + " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", + " \"ConversionRate\"\n", + " )\n", + " ]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", " (\n", " (\n", - " (\n", - " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", - " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " )\n", - " ** 0.5\n", + " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", + " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", " )\n", - " ).alias(\"StdErr\")\n", - " ]\n", - " )\n", - " .sort(treemap_group_by, descending=False)\n", - " .collect()\n", + " ** 0.5\n", + " )\n", + " ).alias(\"StdErr\")\n", + " ]\n", " )\n", + " .sort(treemap_group_by, descending=False)\n", + " .collect()\n", + ")\n", "\n", "treemap_data = treemap_data\n", "\n", - "fig = px.treemap(treemap_data, path=[px.Constant(\"ALL\")] + treemap_group_by, values='Count',\n", - " color=\"ConversionRate\",\n", - " color_continuous_scale=px.colors.sequential.RdBu_r,\n", - " title=\"[BIZ] Conversion rate treemap\",\n", - " hover_data=['StdErr', 'Positives', 'Negatives'],\n", - " height=640,\n", - " )\n", + "fig = px.treemap(\n", + " treemap_data,\n", + " path=[px.Constant(\"ALL\")] + treemap_group_by,\n", + " values=\"Count\",\n", + " color=\"ConversionRate\",\n", + " color_continuous_scale=px.colors.sequential.RdBu_r,\n", + " title=\"[BIZ] Conversion rate treemap\",\n", + " hover_data=[\"StdErr\", \"Positives\", \"Negatives\"],\n", + " height=640,\n", + ")\n", "fig.update_traces(textinfo=\"label+value+percent parent+percent root\")\n", "fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))\n", "\n", @@ -6361,7 +6404,7 @@ ], "error_y": { "array": { - "bdata": "DWL/DCcahz/F8zxNjF9/P3Tu/vQ4OYU/+UB9PBWRiT+uJCCKJ5GLP6qc0gMkLos/sOqXzYhbiz8ZE5WuVMqNP56dBfW5eIs/F5Um+TLRjD8Fw6EaqcCIPxLWvK84AY0/Z97fcm1dij8UsGe1gmaMP8PEyHdeFow/74I3ik+NiD/vxqe8smuKP1LwvmDI0ok/QhYikYr1iz8Jb1zA7KCMP4LbYtJ1+Ik/ofVLZUkMkj8=", + "bdata": "3phnesMBhD9/yffWnl6AP4+VVEEZ/YY/Vmtn/o5jiz+a85LlufOJPypcmAaZSow/G49V5xO4jD9CBMwykc+LP8s/5UOm/os/wCzeBH1biz99Rtc/UNWKP8+ZI8MaxYo/ZW6984wsiz8R0dNIiB2KP2QJ2QPNYoo/dX6udzFbij+tYpsxqDeLP5DU/iMePo0/AdDtQW0Sjj8siHchC0GMP63SUkxte4k/J/LD1LIlkD8=", "dtype": "f8" } }, @@ -6405,7 +6448,7 @@ ], "xaxis": "x2", "y": { - "bdata": "5RMxm0uHij+/9pDLioGSPxLtoxHto6E/l0aBCI5dqj8W01lMZzGtP/7GZFg+5K0/kgUzd932rD8hfqm6EJyxPw4y9WgEjK0/YPS0eH8fsT+rbDhi2hOpPxP6ZVtIWbE/ZNmNB6eJrD/+8Iw4XBiwP//5ZMZ8sa4/inzWDabIpz+/IeU6YvGrP3oEW+WM26o/+Se1t8jkrz+mtSf/zuWwP4ex2vrbA6s/k5E04TRasj8=", + "bdata": "krGohtnBgT+OWDJgXMeTP1hPYT2F9aQ/24G5dmCurT8UHD3B0ROsPxAEQRAEQbA/ugE85d2esD99FOsqcw6vPx2vBUImC68/+NSN25RrrT+Gu/omeX+tP6T7C5/KUqw/zGEOc5jDrD8or6G8hvKqP+bTxUalj6w/4SQCD2nfqz/RfndxXbatP1EMZgRJ77A/wqYARzs9sj8CMYS/8x6wP34rfv461qk/HczBHMzBrD8=", "dtype": "f8" }, "yaxis": "y2" @@ -6482,7 +6525,7 @@ ], "error_y": { "array": { - "bdata": "BhvaZK6lgD9g9NQqhHt8P/YRPcBHs4Q/sJTFsjZTiT+a85LlufOJPzWSWY6BKI0/8hDzKlyBjD9fSRI8cv2LP033NblDn4k/DpFd3Yb1jT/dzOvMjjCKP88YkQCLqIk/qRi32GTaiz/5QH08FZGJP/TAqvbfuYs/Tw9FoNZ2jD/IGXKbok6NPxJNgzS0NI0/eDXPHhYKjD9ZfQYKmTmMP08JQRaHhYw/C+4jlOOwkD8=", + "bdata": "oJbfRua2gj83Rdq5q159P4tq0gQIlIU/x0Yrn4EXiT+LDk89aaGNPxDyNFyGg4s/CY/ItNlUiD/yflX8xiiIP7Quw1jQgYw/v+K828Vdiz8VxQfoyeyLPz5xXOQKeoo/Oelfc6Jmjj+iqs43VP6IPyFBJXEgHY0/siqq/cGziD/waO/gLTuMP73ZBIP5U40/ufMPio15iz+6i6ZbMr6LP8ZZjYY1bY0/bC8BeNakjj8=", "dtype": "f8" } }, @@ -6526,7 +6569,7 @@ ], "xaxis": "x", "y": { - "bdata": "GBgYGBgYeD/F3aDW1uuPPwzh77wHxKA/QlBnbfE6qT8UHD3B0ROsP7Ot4dknHLE/5SfEWfkJsT9K8QKZFC+wP16QlH/o26o/Eh/xER/xsT/tdPyDC5OqP9DOOiH4y6k/m4eRDM9trj+XRoEIjl2qPweQagCpBrA/fAfxHcR3sD+R2myR2myxP5AWqmSUI7E/xY+jkP9Rrz+DmZN+3oivP14uwx0SZLA/J3ZiJ3Zirz8=", + "bdata": "sAlB2fuagD9qNuzvYlCRP5gin3WDKaI/y7hl3DJuqT93mYTNPA2xP1Pks24wRa4/RyIgPpqQpz8gWvwo2SynP87tJvTLtrA/a3oFmhvDrj95tPMD0byuPxzHcRzHcaw/c3Nzc3Nzsz/T2611TRypP1hlUgFZvrA/fvN96K4nqD802j1sfRuwP241gtVJ4bE/CJ5VDzyrrj8faPoeaPquPwXJn1dZnLE/Fdt6w/ugqj8=", "dtype": "f8" }, "yaxis": "y" @@ -6603,7 +6646,7 @@ ], "error_y": { "array": { - "bdata": "4mc+p4tchj9Cj+n8AbyCP8nNFnqwD4w/fY7rs6vYjj+JF9Do/p2RP+xmu3n90pE/FKn+7DDZkT+DPChQUwuSP2TR+IhRKpE/wNa4/iQxkT8FbpkqvNOPPy1lVHUFPZI/hcF8vFVfkT8ThJ0vaF6QP0VkZ+LqN5I/QTq09Wd0kD+XDKATrDuSP4c9NmO4uJE/a3oK2u2pkT+LPTfBflCRP9Y+P6IhypE/KNT1ORgBkz8=", + "bdata": "tAFsSta9hj8L8ji12BeHP2uErbQh1oo/WeTK8kqSjz/cNVUG14uRPyoka/XEdpE/IGEv33dCkj/F/mBbQm6RP+C9HoUBdJM//oNbPgVAkj9f66GfiTmSP9knedqpyZA/Qj3P/QVXjj/GfFPemDiRP3ey5JJP6Y8/4j+Uj3+VkT+3ZWJ0IPaQP98l8yGZ1pA/kJwqn0xokT92psU/Bz2RPzxlzWNo0ZI/yr9T4oIglj8=", "dtype": "f8" } }, @@ -6647,7 +6690,7 @@ ], "xaxis": "x2", "y": { - "bdata": "F2DyFmDyhj8mGn5L09WbP6rPtDX+bq4/FBQUFBQUtD+kFeTexve5P+PtRg9SaLo/HwK6BUNcuz+BHfJv6VW7PzaU11BeQ7k/28o1p7qguD9XbnC2X4a1P/SSmlYfzLw/ENZvgKnmuD/EBqDxi562P925c+fOnbs/wPXPbtehtj9DUmfH3aO7PyiYmGLvA7o/3i/w4lt/uj/h2+MBvdS5P57xjGc847k/VVVVVVVVtT8=", + "bdata": "bXUBwspWhz8yrbajyX2lP8GnQP5ZZaw/JyDN6DTGtT/exqhXRYO5P4GEn1yIurk/iYakAPd9vD/OE96eciC6P/4XsF7SfsA/HMIhHMIhvD9q3mvviDS8P8MjSkI4Hbc/av1KgVq/sj+TRQ4QUty3P+QYrK6CB7U/mr4Hmr4Huj8a7R62vg24P3rhwfbcpLc/hqgBKGWIuj+HWRzrASe5P7QMoYS4W74/fKSVyRMQvT8=", "dtype": "f8" }, "yaxis": "y2" @@ -6724,7 +6767,7 @@ ], "error_y": { "array": { - "bdata": "TKR2mfHQkT+RLdJBoUqCP7NgpqHuEYw/M2KzEJVAkj9regra7amRPxYh4UTMA44/uLl9++sAkj8wW7FCpE2RPweqaj+7vZE/p2Oz92GQkD85Qk+VPgGSP1AJwmjc4pA/CxY4ce4xkT80L7vWISCSP9X1Qg+XNJE/3qBgEugtkT+k0wSEcauQP7DcEYv6HZE/4S8o21V+kj8qDgjB8g2QP95H2xEmQpI/gXtejFBylT8=", + "bdata": "wej/AubJij/OsFC07yKFP/0wrogfBYw/8I7SDM8OkT/22KH4RkmRPxagz5rYGJE/olrup9fnkD8PIAGvbM6SP6Ja7qfX55A/a3oK2u2pkT+vJEZFX46RP1XkTE0hmpE/taOSqEs3kT8DJ4xENp+QPx5kr9eeJpE/vVqJ6B7TkT8jP186UkWRP0XfdaXHAZA/ulrXTR0nkj+24dNTD5iRP8C93aVev5E/cczWlVvYlD8=", "dtype": "f8" } }, @@ -6768,7 +6811,7 @@ ], "xaxis": "x", "y": { - "bdata": "6k1vetObnj/EzUolSnKZP36daxhijrA/Qd+DvLxvuz/eL/DiW3+6P5WYSkwlprI/qi6f2QcKuz/lLmRKk125P3MJaSr77bk/+Jphqivntj/tdPyDC5O6P1J6ZEk8X7c/lUDHa5syuT+btVmbtVm7P0dwXBJdU7k/16BiTZd+uD/yHeekJ563P1SNWjId97g/f206zln/vD+BLF8IibW0P53yR53yR70/oxZz/Vckuj8=", + "bdata": "EQ7hEA7hkD+UWSwNDiChP7Xo3Si3F7A/7QViBszLtj+CPRKml3O5P4ZckQrlXrg/5d2ekGG+tz/jJPSMyaO9P+XdnpBhvrc/3i/w4lt/uj+6RHhSkPy5P72q/UwTg7o/6ifWilHkuD9WJIHycOC3PztDYt7OkLg/rlHc6UDeuj/6GJyPwfm4P6fFohyfGbU/CIZki2jOuj8XqmSUI9G5P+9h69uAobo/XMRm8BaxuT8=", "dtype": "f8" }, "yaxis": "y" @@ -7677,7 +7720,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (88, 8)
DayChannelExperimentGroupNegativesPositivesCountConversionRateCI
datestrstru32i32u32f64f64
2024-11-25"Email""Conversion-Control"38153910.0129530.01128
2024-11-25"Email""Conversion-Test"35343610.0112040.010919
2024-11-25"Web""Conversion-Control"33823420.0058820.008129
2024-11-25"Web""Conversion-Test"357113790.0298910.017399
2024-11-26"Email""Conversion-Control"11412111830.0180720.007659
……………………
2024-12-15"Web""Conversion-Test"108414013640.1143790.01783
2024-12-16"Email""Conversion-Control"764598820.0716890.017625
2024-12-16"Email""Conversion-Test"781719230.0833330.018559
2024-12-16"Web""Conversion-Control"781518830.0612980.0163
2024-12-16"Web""Conversion-Test"721828850.1021170.020944
" + "shape: (88, 8)
DayChannelExperimentGroupNegativesPositivesCountConversionRateCI
datestrstru32i32u32f64f64
2024-11-25"Email""Conversion-Control"34333490.0086710.009769
2024-11-25"Email""Conversion-Test"34743550.0113960.011104
2024-11-25"Web""Conversion-Control"36733730.0081080.009138
2024-11-25"Web""Conversion-Test"35863700.0164840.01308
2024-11-26"Email""Conversion-Control"11172211610.0193150.007993
……………………
2024-12-15"Web""Conversion-Test"106812413160.1040270.017332
2024-12-16"Email""Conversion-Control"773468650.0561660.015769
2024-12-16"Email""Conversion-Test"734949220.1135270.021608
2024-12-16"Web""Conversion-Control"802448900.0520090.014963
2024-12-16"Web""Conversion-Test"753849210.1003580.020357
" ], "text/plain": [ "shape: (88, 8)\n", @@ -7687,26 +7730,26 @@ "│ date ┆ str ┆ --- ┆ u32 ┆ i32 ┆ u32 ┆ --- ┆ f64 │\n", "│ ┆ ┆ str ┆ ┆ ┆ ┆ f64 ┆ │\n", "╞════════════╪═════════╪════════════════╪═══════════╪═══════════╪═══════╪═══════════════╪══════════╡\n", - "│ 2024-11-25 ┆ Email ┆ Conversion-Con ┆ 381 ┆ 5 ┆ 391 ┆ 0.012953 ┆ 0.01128 │\n", + "│ 2024-11-25 ┆ Email ┆ Conversion-Con ┆ 343 ┆ 3 ┆ 349 ┆ 0.008671 ┆ 0.009769 │\n", "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-11-25 ┆ Email ┆ Conversion-Tes ┆ 353 ┆ 4 ┆ 361 ┆ 0.011204 ┆ 0.010919 │\n", + "│ 2024-11-25 ┆ Email ┆ Conversion-Tes ┆ 347 ┆ 4 ┆ 355 ┆ 0.011396 ┆ 0.011104 │\n", "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-11-25 ┆ Web ┆ Conversion-Con ┆ 338 ┆ 2 ┆ 342 ┆ 0.005882 ┆ 0.008129 │\n", + "│ 2024-11-25 ┆ Web ┆ Conversion-Con ┆ 367 ┆ 3 ┆ 373 ┆ 0.008108 ┆ 0.009138 │\n", "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-11-25 ┆ Web ┆ Conversion-Tes ┆ 357 ┆ 11 ┆ 379 ┆ 0.029891 ┆ 0.017399 │\n", + "│ 2024-11-25 ┆ Web ┆ Conversion-Tes ┆ 358 ┆ 6 ┆ 370 ┆ 0.016484 ┆ 0.01308 │\n", "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-11-26 ┆ Email ┆ Conversion-Con ┆ 1141 ┆ 21 ┆ 1183 ┆ 0.018072 ┆ 0.007659 │\n", + "│ 2024-11-26 ┆ Email ┆ Conversion-Con ┆ 1117 ┆ 22 ┆ 1161 ┆ 0.019315 ┆ 0.007993 │\n", "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2024-12-15 ┆ Web ┆ Conversion-Tes ┆ 1084 ┆ 140 ┆ 1364 ┆ 0.114379 ┆ 0.01783 │\n", + "│ 2024-12-15 ┆ Web ┆ Conversion-Tes ┆ 1068 ┆ 124 ┆ 1316 ┆ 0.104027 ┆ 0.017332 │\n", "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-12-16 ┆ Email ┆ Conversion-Con ┆ 764 ┆ 59 ┆ 882 ┆ 0.071689 ┆ 0.017625 │\n", + "│ 2024-12-16 ┆ Email ┆ Conversion-Con ┆ 773 ┆ 46 ┆ 865 ┆ 0.056166 ┆ 0.015769 │\n", "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-12-16 ┆ Email ┆ Conversion-Tes ┆ 781 ┆ 71 ┆ 923 ┆ 0.083333 ┆ 0.018559 │\n", + "│ 2024-12-16 ┆ Email ┆ Conversion-Tes ┆ 734 ┆ 94 ┆ 922 ┆ 0.113527 ┆ 0.021608 │\n", "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-12-16 ┆ Web ┆ Conversion-Con ┆ 781 ┆ 51 ┆ 883 ┆ 0.061298 ┆ 0.0163 │\n", + "│ 2024-12-16 ┆ Web ┆ Conversion-Con ┆ 802 ┆ 44 ┆ 890 ┆ 0.052009 ┆ 0.014963 │\n", "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-12-16 ┆ Web ┆ Conversion-Tes ┆ 721 ┆ 82 ┆ 885 ┆ 0.102117 ┆ 0.020944 │\n", + "│ 2024-12-16 ┆ Web ┆ Conversion-Tes ┆ 753 ┆ 84 ┆ 921 ┆ 0.100358 ┆ 0.020357 │\n", "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", "└────────────┴─────────┴────────────────┴───────────┴───────────┴───────┴───────────────┴──────────┘" ] @@ -7720,92 +7763,104 @@ "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", "\n", "line_data = (\n", - " ih_analysis.group_by(line_group_by)\n", - " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\")\n", - " )\n", - " .with_columns(\n", - " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"ConversionRate\")]\n", - " )\n", - " .with_columns(\n", - " [\n", + " ih_analysis.group_by(line_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\"),\n", + " )\n", + " .with_columns(\n", + " [\n", + " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", + " \"ConversionRate\"\n", + " )\n", + " ]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", " (\n", " (\n", - " ((\n", + " (\n", " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", " )\n", - " ** 0.5) * 1.96\n", + " ** 0.5\n", " )\n", - " ).alias(\"CI\")\n", - " ]\n", - " )\n", - " .sort(line_group_by, descending=False)\n", - " .collect()\n", + " * 1.96\n", + " )\n", + " ).alias(\"CI\")\n", + " ]\n", " )\n", + " .sort(line_group_by, descending=False)\n", + " .collect()\n", + ")\n", "\n", "line_data = line_data\n", "\n", "if len(line_data[\"Day\"].unique()) < 30:\n", - " fig = px.bar(line_data,\n", - " x=\"Day\",\n", - " y=\"ConversionRate\",\n", - " color=\"ExperimentGroup\",\n", - " error_y='CI',\n", - " facet_row=\"Channel\",\n", - " barmode=\"group\",\n", - " title=\"[CONV] Daily Conversion Rate with 95% confidence interval\",\n", - " custom_data=[\"ExperimentGroup\"]\n", - " )\n", - " fig.update_layout(\n", - " updatemenus=[\n", - " dict(\n", - " buttons=list([\n", - " dict(\n", - " args=[\"type\", \"bar\"],\n", - " label=\"Bar\",\n", - " method=\"restyle\"\n", - " ),\n", - " dict(\n", - " args=[\"type\", \"line\"],\n", - " label=\"Line\",\n", - " method=\"restyle\"\n", - " )\n", - " ]),\n", - " direction=\"down\",\n", - " showactive=True,\n", + " fig = px.bar(\n", + " line_data,\n", + " x=\"Day\",\n", + " y=\"ConversionRate\",\n", + " color=\"ExperimentGroup\",\n", + " error_y=\"CI\",\n", + " facet_row=\"Channel\",\n", + " barmode=\"group\",\n", + " title=\"[CONV] Daily Conversion Rate with 95% confidence interval\",\n", + " custom_data=[\"ExperimentGroup\"],\n", + " )\n", + " fig.update_layout(\n", + " updatemenus=[\n", + " dict(\n", + " buttons=list(\n", + " [\n", + " dict(args=[\"type\", \"bar\"], label=\"Bar\", method=\"restyle\"),\n", + " dict(args=[\"type\", \"line\"], label=\"Line\", method=\"restyle\"),\n", + " ]\n", " ),\n", - " ]\n", - " )\n", + " direction=\"down\",\n", + " showactive=True,\n", + " ),\n", + " ]\n", + " )\n", "else:\n", - " fig = px.line(\n", - " line_data,\n", - " x=\"Day\",\n", - " y=\"ConversionRate\",\n", - " color=\"ExperimentGroup\",\n", - " title=\"[CONV] Daily Conversion Rate\",\n", - " acet_row=\"Channel\",\n", - " custom_data=[\"ExperimentGroup\"]\n", - " )\n", + " fig = px.line(\n", + " line_data,\n", + " x=\"Day\",\n", + " y=\"ConversionRate\",\n", + " color=\"ExperimentGroup\",\n", + " title=\"[CONV] Daily Conversion Rate\",\n", + " acet_row=\"Channel\",\n", + " custom_data=[\"ExperimentGroup\"],\n", + " )\n", "\n", "fig.update_xaxes(tickfont=dict(size=10))\n", - "fig.update_yaxes(tickformat=',.2%')\n", - "yaxis_names = ['yaxis'] + [axis_name for axis_name in fig.layout._subplotid_props if 'yaxis' in axis_name]\n", - "yaxis_layout_dict = {yaxis_name + \"_tickformat\": ',.2%' for yaxis_name in yaxis_names}\n", + "fig.update_yaxes(tickformat=\",.2%\")\n", + "yaxis_names = [\"yaxis\"] + [\n", + " axis_name for axis_name in fig.layout._subplotid_props if \"yaxis\" in axis_name\n", + "]\n", + "yaxis_layout_dict = {yaxis_name + \"_tickformat\": \",.2%\" for yaxis_name in yaxis_names}\n", "fig.update_layout(yaxis_layout_dict)\n", "height = max(640, 300 * len(line_data[\"Channel\"].unique()))\n", "fig.update_layout(\n", - " xaxis_title=\"Day\",\n", - " yaxis_title=\"Conversion Rate\",\n", - " hovermode=\"x unified\",\n", - " height=height\n", - " )\n", + " xaxis_title=\"Day\",\n", + " yaxis_title=\"Conversion Rate\",\n", + " hovermode=\"x unified\",\n", + " height=height,\n", + ")\n", "fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[1]))\n", - "fig = fig.update_traces(hovertemplate=\"Day\" + ' : %{x}' + '
' +\n", - " \"Experiment Group\" + ' : %{customdata[0]}' + '
' +\n", - " \"Conversion Rate\" + ' : %{y:.2%}' + '')\n", + "fig = fig.update_traces(\n", + " hovertemplate=\"Day\"\n", + " + \" : %{x}\"\n", + " + \"
\"\n", + " + \"Experiment Group\"\n", + " + \" : %{customdata[0]}\"\n", + " + \"
\"\n", + " + \"Conversion Rate\"\n", + " + \" : %{y:.2%}\"\n", + " + \"\"\n", + ")\n", "\n", "fig.show()\n", "line_data" @@ -7941,7 +7996,7 @@ "text": "WebConversion-Control" }, "type": "indicator", - "value": 0.20265143952401868 + "value": 0.20024672688925146 }, { "delta": { @@ -7982,7 +8037,7 @@ "text": "WebConversion-Test" }, "type": "indicator", - "value": 0.2007769945530279 + "value": 0.19848369369007982 } ], "layout": { @@ -8789,7 +8844,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (4, 9)
ChannelExperimentGroupNegativesPositivesCountCTRStdErrNameCName
strstru32i32u32f64f64strstr
"Email""Conversion-Control"249570249570.00.0"EmailConversion-Control""Email_Conversion-Control"
"Email""Conversion-Test"250310250310.00.0"EmailConversion-Test""Email_Conversion-Test"
"Web""Conversion-Control"199685075301180.2026510.00254"WebConversion-Control""Web_Conversion-Control"
"Web""Conversion-Test"199555013299810.2007770.002535"WebConversion-Test""Web_Conversion-Test"
" + "shape: (4, 9)
ChannelExperimentGroupNegativesPositivesCountCTRStdErrNameCName
strstru32i32u32f64f64strstr
"Email""Conversion-Control"248710248710.00.0"EmailConversion-Control""Email_Conversion-Control"
"Email""Conversion-Test"250700250700.00.0"EmailConversion-Test""Email_Conversion-Test"
"Web""Conversion-Control"200975032301610.2002470.002524"WebConversion-Control""Web_Conversion-Control"
"Web""Conversion-Test"199814948298770.1984840.002526"WebConversion-Test""Web_Conversion-Test"
" ], "text/plain": [ "shape: (4, 9)\n", @@ -8799,16 +8854,16 @@ "│ str ┆ --- ┆ u32 ┆ i32 ┆ ┆ f64 ┆ f64 ┆ str ┆ str │\n", "│ ┆ str ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", "╞═════════╪════════════╪═══════════╪═══════════╪═══╪══════════╪══════════╪════════════╪════════════╡\n", - "│ Email ┆ Conversion ┆ 24957 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ EmailConve ┆ Email_Conv │\n", + "│ Email ┆ Conversion ┆ 24871 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ EmailConve ┆ Email_Conv │\n", "│ ┆ -Control ┆ ┆ ┆ ┆ ┆ ┆ rsion-Cont ┆ ersion-Con │\n", "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ rol ┆ trol │\n", - "│ Email ┆ Conversion ┆ 25031 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ EmailConve ┆ Email_Conv │\n", + "│ Email ┆ Conversion ┆ 25070 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ EmailConve ┆ Email_Conv │\n", "│ ┆ -Test ┆ ┆ ┆ ┆ ┆ ┆ rsion-Test ┆ ersion-Tes │\n", "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ t │\n", - "│ Web ┆ Conversion ┆ 19968 ┆ 5075 ┆ … ┆ 0.202651 ┆ 0.00254 ┆ WebConvers ┆ Web_Conver │\n", + "│ Web ┆ Conversion ┆ 20097 ┆ 5032 ┆ … ┆ 0.200247 ┆ 0.002524 ┆ WebConvers ┆ Web_Conver │\n", "│ ┆ -Control ┆ ┆ ┆ ┆ ┆ ┆ ion-Contro ┆ sion-Contr │\n", "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ l ┆ ol │\n", - "│ Web ┆ Conversion ┆ 19955 ┆ 5013 ┆ … ┆ 0.200777 ┆ 0.002535 ┆ WebConvers ┆ Web_Conver │\n", + "│ Web ┆ Conversion ┆ 19981 ┆ 4948 ┆ … ┆ 0.198484 ┆ 0.002526 ┆ WebConvers ┆ Web_Conver │\n", "│ ┆ -Test ┆ ┆ ┆ ┆ ┆ ┆ ion-Test ┆ sion-Test │\n", "└─────────┴────────────┴───────────┴───────────┴───┴──────────┴──────────┴────────────┴────────────┘" ] @@ -8821,53 +8876,64 @@ "source": [ "positive_model_response = [\"Clicked\"]\n", "all_model_response = [\"Impression\", \"Pending\"]\n", - "group_by = [\"Day\", \"Month\", \"Year\", \"Quarter\", \"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", + "group_by = [\n", + " \"Day\",\n", + " \"Month\",\n", + " \"Year\",\n", + " \"Quarter\",\n", + " \"Channel\",\n", + " \"Issue\",\n", + " \"Group\",\n", + " \"Name\",\n", + " \"ExperimentGroup\",\n", + "]\n", "\n", "ih_analysis = (\n", - " ih.filter(\n", - " (pl.col(\"Outcome\").is_in(all_model_response + positive_model_response))\n", - " )\n", - " .with_columns([\n", - " pl.when(pl.col('Outcome').is_in(positive_model_response)).\n", - " then(1).otherwise(0).alias('Outcome_Binary')\n", - " ])\n", - " .group_by(group_by)\n", - " .agg([\n", - " pl.len().alias('Count'),\n", - " pl.sum(\"Outcome_Binary\").alias(\"Positives\")\n", - " ])\n", - " .with_columns([\n", - " (pl.col(\"Count\") - (2 * pl.col(\"Positives\"))).alias(\"Negatives\")\n", - " ])\n", - " )\n", + " ih.filter((pl.col(\"Outcome\").is_in(all_model_response + positive_model_response)))\n", + " .with_columns(\n", + " [\n", + " pl.when(pl.col(\"Outcome\").is_in(positive_model_response))\n", + " .then(1)\n", + " .otherwise(0)\n", + " .alias(\"Outcome_Binary\")\n", + " ]\n", + " )\n", + " .group_by(group_by)\n", + " .agg([pl.len().alias(\"Count\"), pl.sum(\"Outcome_Binary\").alias(\"Positives\")])\n", + " .with_columns([(pl.col(\"Count\") - (2 * pl.col(\"Positives\"))).alias(\"Negatives\")])\n", + ")\n", "gauge_group_by = [\"Channel\", \"ExperimentGroup\"]\n", - "reference = {'Web_Conversion-Test' : 0.25, 'Web_Conversion-Control' : 0.25}\n", + "reference = {\"Web_Conversion-Test\": 0.25, \"Web_Conversion-Control\": 0.25}\n", "gauge_data = (\n", - " ih_analysis.group_by(gauge_group_by)\n", - " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\")\n", - " )\n", - " .with_columns(\n", - " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"CTR\")]\n", - " )\n", - " .with_columns(\n", - " [\n", + " ih_analysis.group_by(gauge_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\"),\n", + " )\n", + " .with_columns(\n", + " [\n", + " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", + " \"CTR\"\n", + " )\n", + " ]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", " (\n", " (\n", - " (\n", - " (pl.col(\"CTR\") * (1 - pl.col(\"CTR\")))\n", - " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " )\n", - " ** 0.5\n", + " (pl.col(\"CTR\") * (1 - pl.col(\"CTR\")))\n", + " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", " )\n", - " ).alias(\"StdErr\")\n", - " ]\n", - " )\n", - " .sort(gauge_group_by, descending=False)\n", - " .collect()\n", + " ** 0.5\n", + " )\n", + " ).alias(\"StdErr\")\n", + " ]\n", " )\n", + " .sort(gauge_group_by, descending=False)\n", + " .collect()\n", + ")\n", "\n", "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", @@ -8876,56 +8942,58 @@ "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", "\n", "gauge_data = gauge_data.with_columns(\n", - " pl.concat_str(gauge_group_by).alias('Name'),\n", - " pl.concat_str(gauge_group_by, separator = \"_\").alias('CName'),\n", - " )\n", + " pl.concat_str(gauge_group_by).alias(\"Name\"),\n", + " pl.concat_str(gauge_group_by, separator=\"_\").alias(\"CName\"),\n", + ")\n", "\n", - "fig = make_subplots(rows=rows,\n", - " cols=cols,\n", - " specs=[[{\"type\": \"indicator\"} for c in range(cols)] for t in range(rows)]\n", - " )\n", + "fig = make_subplots(\n", + " rows=rows,\n", + " cols=cols,\n", + " specs=[[{\"type\": \"indicator\"} for c in range(cols)] for t in range(rows)],\n", + ")\n", "fig.update_layout(\n", - " height=270 * rows,\n", - " autosize=True,\n", - " title='[CONV] Conversion (Channel/Model Type)',\n", - " margin=dict(b=10, t=120, l=10, r=10))\n", + " height=270 * rows,\n", + " autosize=True,\n", + " title=\"[CONV] Conversion (Channel/Model Type)\",\n", + " margin=dict(b=10, t=120, l=10, r=10),\n", + ")\n", "index = 0\n", "for row in gauge_data.iter_rows(named=True):\n", - " ref_value = reference.get(row['CName'], None)\n", - " gauge = {\n", - " 'axis': {'tickformat': ',.2%'},\n", - " 'threshold': {\n", - " 'line': {'color': \"red\", 'width': 2},\n", - " 'thickness': 0.75,\n", - " 'value': ref_value\n", + " ref_value = reference.get(row[\"CName\"], None)\n", + " gauge = {\n", + " \"axis\": {\"tickformat\": \",.2%\"},\n", + " \"threshold\": {\n", + " \"line\": {\"color\": \"red\", \"width\": 2},\n", + " \"thickness\": 0.75,\n", + " \"value\": ref_value,\n", + " },\n", + " }\n", + " if ref_value:\n", + " if row[\"CTR\"] < ref_value:\n", + " gauge = {\n", + " \"axis\": {\"tickformat\": \",.2%\"},\n", + " \"bar\": {\n", + " \"color\": \"#EC5300\" if row[\"CTR\"] < (0.75 * ref_value) else \"#EC9B00\"\n", + " },\n", + " \"threshold\": {\n", + " \"line\": {\"color\": \"red\", \"width\": 2},\n", + " \"thickness\": 0.75,\n", + " \"value\": ref_value,\n", + " },\n", " }\n", - " }\n", - " if ref_value:\n", - " if row['CTR'] < ref_value:\n", - " gauge = {\n", - " 'axis': {'tickformat': ',.2%'},\n", - " 'bar': {'color': '#EC5300' if row['CTR'] < (0.75 * ref_value) else '#EC9B00'},\n", - " 'threshold': {\n", - " 'line': {'color': \"red\", 'width': 2},\n", - " 'thickness': 0.75,\n", - " 'value': ref_value\n", - " }\n", - " }\n", "\n", - " trace1 = go.Indicator(mode=\"gauge+number+delta\",\n", - " number={'valueformat': \",.2%\"},\n", - " value=row['CTR'],\n", - " delta={'reference': ref_value, 'valueformat': \",.2%\"},\n", - " title={'text': row['Name']},\n", - " gauge=gauge,\n", - " )\n", - " r, c = divmod(index, cols)\n", - " fig.add_trace(\n", - " trace1,\n", - " row=(r + 1), col=(c + 1)\n", - " )\n", - " index = index + 1\n", - " \n", + " trace1 = go.Indicator(\n", + " mode=\"gauge+number+delta\",\n", + " number={\"valueformat\": \",.2%\"},\n", + " value=row[\"CTR\"],\n", + " delta={\"reference\": ref_value, \"valueformat\": \",.2%\"},\n", + " title={\"text\": row[\"Name\"]},\n", + " gauge=gauge,\n", + " )\n", + " r, c = divmod(index, cols)\n", + " fig.add_trace(trace1, row=(r + 1), col=(c + 1))\n", + " index = index + 1\n", + "\n", "fig.show()\n", "gauge_data" ] @@ -9135,7 +9203,7 @@ ], "error_y": { "array": { - "bdata": "EnayRxUfpj+/qpINNPeWP8dIXiOeyZc/7qkrsYdemD836iN6tHWXP61dFbdTHJc//F8A635dlz8hTrDO6EKXP2dAV13pH5c/7QiiTOYLlz+I3ZTNW82WP15RQCmUJZc/0pJ7kYDhlj9C0FFlggeYPyDl8Rxx75c/DCQFd1pVlT/GXPuVkaqXPzoVEK6HEZc/ZxO5+6jYlj+N2VEilqyXP1XBEqTi7Jc/Fm4m19LDnD8=", + "bdata": "37Loir80pD++yRLUEBKXP9sydnq4Vpc/OQ05D94clz/CVkyFkQaXP8ytEHkCoZc/o4+H2I6Ulz8sw5nLPQmXPxjunZ/rAZc/jHbdNFSdlz9fRlPM6EWXPyKpPv3CCZc/tdPjG+IXlz98sAG5NIyXPzvqeeNAppc/0GcXlpOKlz/8ftPL9s2XP3CSnFlAmZY/LziJrM52lj9du5ax+WqXP7dsXEOTtZY/Q8FDrvy1mz8=", "dtype": "f8" } }, @@ -9179,7 +9247,7 @@ ], "xaxis": "x", "y": { - "bdata": "u7q6urq6yj/0UmdcGITJP7MpkujKMMs/2M4x0/2ZzD+q26G6HarLP3zPjwwt6Mg/sHdMDewdyz83ncrermDKP9vrlzsJBco/WIZlWIZlyD+Z802oAl3HPxXgZdojtMg/QnsJ7SW0xz8ZTw9mqGLMP+6c4c4Z7sw/NkrZKGWjxD9bf8haf8jKP6ZXba5xu8g/Np/EOLMIyD/zCu4xXCvKP3nPqvh1f8s/AAAAAAAAzD8=", + "bdata": "ZHhxHjqGxz9v3yr4fnzKP/JZN5gin8k/zZHmSHOkyT8G1DOV647HP8ln3WCKfMo/nWwPKM+ryj/GxPMQyf/IP9jq2SFwY8k/s9LoERVkyz+obtymXOvIPxdRk//SXck/CgoKCgoKyj+GTrWkHtDKP6H3dkrr4sk/zg1APcxPyj8kf15lcUbLP+RCQgWnZsg/Jv/RS/6jxz+avgeavgfKPxrtHra+Dcg/Z6O+s1HfyT8=", "dtype": "f8" }, "yaxis": "y" @@ -9377,7 +9445,7 @@ ], "error_y": { "array": { - "bdata": "QEExHVDcpD9t4SRIA9uXP3SU0Z65FZc/Gc6Zkphklz/Eu0JPfGyXP9002MhZmZc/RQi/reZclz9k9nY8cmiXP0N7zUKimJc/mQG/1ZOzlz+PUBgKX+2WP5xxNGcWBJg/EzVwP2gPlz9iwRciBK+WP0wSXhSIxpY/HptRVciSlz+4CndGCWKWPyEFuuNKwJY/fsbiKaXJlz/Dq6R9zNWXPy6FgMN8KZc/Snq6UOO+mz8=", + "bdata": "irwrdIbWoj/EKhfRr52XPwMQKZQ5t5c/kRJZUkfqmD+h1EQm+BeXP/yIJxZnBpc/sQYm5X0Ulz9pZa2s0YCXP3erHmDGCpc/cl7Lgz4slz8qQpOxpvCXP17Lyv+ARZc/QIpw3Ughlj+JGmjJH3OWPy8t2DQJH5c/lG1DwNillj/Bc5M84IqXP7pXXyzWXJc/F4rkaI/5lz9fpW22PJeXP6N2ovTe75U/JWf8bH74mz8=", "dtype": "f8" } }, @@ -9421,7 +9489,7 @@ ], "xaxis": "x", "y": { - "bdata": "FrKQhSxkyT8KfdokKKfKPwCD61bsYso/QUXb7akAyT/OeVWIp0jKP9mAbEA2IMs/dL87BVxnyT8Obyd3IVPKP9WOzqTa0ck/O7ETO7ETyz92G8FmDLLHPzpm79bCTcs/cF5S0+qDyT+XdmmXdmnHP0WD+iFLzMg/CCL/04Y6yj+mShGJJbnHPzIHu/7epcg/dNY/hxHxyj8RharZ6m/KP+qUP+qUP8o/ZWxk96g6yD8=", + "bdata": "czVXczVXwz9J1ijFSejJP4cbbrjhhss/aYf6B79+zD9uyCmdvY7JP5YuuE5Rzcg/lwLa10zuyD9pXg+7+pPJPxNFYNGr0sg/lfw3SzCJyT/y51UWZ7TLPwx/n3B7Fso/9axKBAyIxj+PH+gCUmXIPzfAOXW4Gsk/fnGQoOPbxz8Rd8vjBmDKP34rfv461sk/dy547whcyj8Qy/qJN0DKP0a7h61vA8Y/J5poookmyj8=", "dtype": "f8" }, "yaxis": "y" @@ -10330,7 +10398,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (88, 8)
DayChannelExperimentGroupNegativesPositivesCountCTRCI
datestrstru32i32u32f64f64
2024-11-25"Email""Conversion-Control"38603860.00.0
2024-11-25"Email""Conversion-Test"35703570.00.0
2024-11-25"Web""Conversion-Control"269714110.2088240.043206
2024-11-25"Web""Conversion-Test"295734410.198370.040743
2024-11-26"Email""Conversion-Control"1162011620.00.0
……………………
2024-12-15"Web""Conversion-Test"97325114750.2050650.022619
2024-12-16"Email""Conversion-Control"82308230.00.0
2024-12-16"Email""Conversion-Test"85208520.00.0
2024-12-16"Web""Conversion-Control"65018210140.218750.028091
2024-12-16"Web""Conversion-Test"6511529550.189290.027095
" + "shape: (88, 8)
DayChannelExperimentGroupNegativesPositivesCountCTRCI
datestrstru32i32u32f64f64
2024-11-25"Email""Conversion-Control"34603460.00.0
2024-11-25"Email""Conversion-Test"35103510.00.0
2024-11-25"Web""Conversion-Control"302684380.1837840.039465
2024-11-25"Web""Conversion-Test"309554190.1510990.036793
2024-11-26"Email""Conversion-Control"1139011390.00.0
……………………
2024-12-15"Web""Conversion-Test"98720513970.171980.021423
2024-12-16"Email""Conversion-Control"81908190.00.0
2024-12-16"Email""Conversion-Test"82808280.00.0
2024-12-16"Web""Conversion-Control"67517110170.2021280.027061
2024-12-16"Web""Conversion-Test"66617110080.2043010.027315
" ], "text/plain": [ "shape: (88, 8)\n", @@ -10339,17 +10407,17 @@ "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ date ┆ str ┆ str ┆ u32 ┆ i32 ┆ u32 ┆ f64 ┆ f64 │\n", "╞════════════╪═════════╪════════════════════╪═══════════╪═══════════╪═══════╪══════════╪══════════╡\n", - "│ 2024-11-25 ┆ Email ┆ Conversion-Control ┆ 386 ┆ 0 ┆ 386 ┆ 0.0 ┆ 0.0 │\n", - "│ 2024-11-25 ┆ Email ┆ Conversion-Test ┆ 357 ┆ 0 ┆ 357 ┆ 0.0 ┆ 0.0 │\n", - "│ 2024-11-25 ┆ Web ┆ Conversion-Control ┆ 269 ┆ 71 ┆ 411 ┆ 0.208824 ┆ 0.043206 │\n", - "│ 2024-11-25 ┆ Web ┆ Conversion-Test ┆ 295 ┆ 73 ┆ 441 ┆ 0.19837 ┆ 0.040743 │\n", - "│ 2024-11-26 ┆ Email ┆ Conversion-Control ┆ 1162 ┆ 0 ┆ 1162 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-11-25 ┆ Email ┆ Conversion-Control ┆ 346 ┆ 0 ┆ 346 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-11-25 ┆ Email ┆ Conversion-Test ┆ 351 ┆ 0 ┆ 351 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-11-25 ┆ Web ┆ Conversion-Control ┆ 302 ┆ 68 ┆ 438 ┆ 0.183784 ┆ 0.039465 │\n", + "│ 2024-11-25 ┆ Web ┆ Conversion-Test ┆ 309 ┆ 55 ┆ 419 ┆ 0.151099 ┆ 0.036793 │\n", + "│ 2024-11-26 ┆ Email ┆ Conversion-Control ┆ 1139 ┆ 0 ┆ 1139 ┆ 0.0 ┆ 0.0 │\n", "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2024-12-15 ┆ Web ┆ Conversion-Test ┆ 973 ┆ 251 ┆ 1475 ┆ 0.205065 ┆ 0.022619 │\n", - "│ 2024-12-16 ┆ Email ┆ Conversion-Control ┆ 823 ┆ 0 ┆ 823 ┆ 0.0 ┆ 0.0 │\n", - "│ 2024-12-16 ┆ Email ┆ Conversion-Test ┆ 852 ┆ 0 ┆ 852 ┆ 0.0 ┆ 0.0 │\n", - "│ 2024-12-16 ┆ Web ┆ Conversion-Control ┆ 650 ┆ 182 ┆ 1014 ┆ 0.21875 ┆ 0.028091 │\n", - "│ 2024-12-16 ┆ Web ┆ Conversion-Test ┆ 651 ┆ 152 ┆ 955 ┆ 0.18929 ┆ 0.027095 │\n", + "│ 2024-12-15 ┆ Web ┆ Conversion-Test ┆ 987 ┆ 205 ┆ 1397 ┆ 0.17198 ┆ 0.021423 │\n", + "│ 2024-12-16 ┆ Email ┆ Conversion-Control ┆ 819 ┆ 0 ┆ 819 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-12-16 ┆ Email ┆ Conversion-Test ┆ 828 ┆ 0 ┆ 828 ┆ 0.0 ┆ 0.0 │\n", + "│ 2024-12-16 ┆ Web ┆ Conversion-Control ┆ 675 ┆ 171 ┆ 1017 ┆ 0.202128 ┆ 0.027061 │\n", + "│ 2024-12-16 ┆ Web ┆ Conversion-Test ┆ 666 ┆ 171 ┆ 1008 ┆ 0.204301 ┆ 0.027315 │\n", "└────────────┴─────────┴────────────────────┴───────────┴───────────┴───────┴──────────┴──────────┘" ] }, @@ -10362,92 +10430,101 @@ "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", "\n", "line_data = (\n", - " ih_analysis.group_by(line_group_by)\n", - " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\")\n", - " )\n", - " .with_columns(\n", - " [(pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\"CTR\")]\n", - " )\n", - " .with_columns(\n", - " [\n", + " ih_analysis.group_by(line_group_by)\n", + " .agg(\n", + " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", + " pl.sum(\"Positives\").alias(\"Positives\"),\n", + " pl.sum(\"Count\").alias(\"Count\"),\n", + " )\n", + " .with_columns(\n", + " [\n", + " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", + " \"CTR\"\n", + " )\n", + " ]\n", + " )\n", + " .with_columns(\n", + " [\n", + " (\n", " (\n", " (\n", - " ((\n", + " (\n", " (pl.col(\"CTR\") * (1 - pl.col(\"CTR\")))\n", " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", " )\n", - " ** 0.5) * 1.96\n", + " ** 0.5\n", " )\n", - " ).alias(\"CI\")\n", - " ]\n", - " )\n", - " .sort(line_group_by, descending=False)\n", - " .collect()\n", + " * 1.96\n", + " )\n", + " ).alias(\"CI\")\n", + " ]\n", " )\n", + " .sort(line_group_by, descending=False)\n", + " .collect()\n", + ")\n", "\n", "line_data = line_data\n", "\n", "if len(line_data[\"Day\"].unique()) < 30:\n", - " fig = px.bar(line_data,\n", - " x=\"Day\",\n", - " y=\"CTR\",\n", - " color=\"ExperimentGroup\",\n", - " error_y='CI',\n", - " facet_row=\"Channel\",\n", - " barmode=\"group\",\n", - " title=\"[ENG] Daily Click-through Rate with 95% confidence interval\",\n", - " custom_data=[\"ExperimentGroup\"]\n", - " )\n", - " fig.update_layout(\n", - " updatemenus=[\n", - " dict(\n", - " buttons=list([\n", - " dict(\n", - " args=[\"type\", \"bar\"],\n", - " label=\"Bar\",\n", - " method=\"restyle\"\n", - " ),\n", - " dict(\n", - " args=[\"type\", \"line\"],\n", - " label=\"Line\",\n", - " method=\"restyle\"\n", - " )\n", - " ]),\n", - " direction=\"down\",\n", - " showactive=True,\n", + " fig = px.bar(\n", + " line_data,\n", + " x=\"Day\",\n", + " y=\"CTR\",\n", + " color=\"ExperimentGroup\",\n", + " error_y=\"CI\",\n", + " facet_row=\"Channel\",\n", + " barmode=\"group\",\n", + " title=\"[ENG] Daily Click-through Rate with 95% confidence interval\",\n", + " custom_data=[\"ExperimentGroup\"],\n", + " )\n", + " fig.update_layout(\n", + " updatemenus=[\n", + " dict(\n", + " buttons=list(\n", + " [\n", + " dict(args=[\"type\", \"bar\"], label=\"Bar\", method=\"restyle\"),\n", + " dict(args=[\"type\", \"line\"], label=\"Line\", method=\"restyle\"),\n", + " ]\n", " ),\n", - " ]\n", - " )\n", + " direction=\"down\",\n", + " showactive=True,\n", + " ),\n", + " ]\n", + " )\n", "else:\n", - " fig = px.line(\n", - " line_data,\n", - " x=\"Day\",\n", - " y=\"CTR\",\n", - " color=\"ExperimentGroup\",\n", - " title=\"[ENG] Daily Click-through Rate\",\n", - " acet_row=\"Channel\",\n", - " custom_data=[\"ExperimentGroup\"]\n", - " )\n", + " fig = px.line(\n", + " line_data,\n", + " x=\"Day\",\n", + " y=\"CTR\",\n", + " color=\"ExperimentGroup\",\n", + " title=\"[ENG] Daily Click-through Rate\",\n", + " acet_row=\"Channel\",\n", + " custom_data=[\"ExperimentGroup\"],\n", + " )\n", "\n", "fig.update_xaxes(tickfont=dict(size=10))\n", - "fig.update_yaxes(tickformat=',.2%')\n", - "yaxis_names = ['yaxis'] + [axis_name for axis_name in fig.layout._subplotid_props if 'yaxis' in axis_name]\n", - "yaxis_layout_dict = {yaxis_name + \"_tickformat\": ',.2%' for yaxis_name in yaxis_names}\n", + "fig.update_yaxes(tickformat=\",.2%\")\n", + "yaxis_names = [\"yaxis\"] + [\n", + " axis_name for axis_name in fig.layout._subplotid_props if \"yaxis\" in axis_name\n", + "]\n", + "yaxis_layout_dict = {yaxis_name + \"_tickformat\": \",.2%\" for yaxis_name in yaxis_names}\n", "fig.update_layout(yaxis_layout_dict)\n", "height = max(640, 300 * len(line_data[\"Channel\"].unique()))\n", "fig.update_layout(\n", - " xaxis_title=\"Day\",\n", - " yaxis_title=\"CTR\",\n", - " hovermode=\"x unified\",\n", - " height=height\n", - " )\n", + " xaxis_title=\"Day\", yaxis_title=\"CTR\", hovermode=\"x unified\", height=height\n", + ")\n", "fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[1]))\n", - "fig = fig.update_traces(hovertemplate=\"Day\" + ' : %{x}' + '
' +\n", - " \"Experiment Group\" + ' : %{customdata[0]}' + '
' +\n", - " \"CTR\" + ' : %{y:.2%}' + '')\n", + "fig = fig.update_traces(\n", + " hovertemplate=\"Day\"\n", + " + \" : %{x}\"\n", + " + \"
\"\n", + " + \"Experiment Group\"\n", + " + \" : %{customdata[0]}\"\n", + " + \"
\"\n", + " + \"CTR\"\n", + " + \" : %{y:.2%}\"\n", + " + \"\"\n", + ")\n", "\n", "fig.show()\n", "line_data" diff --git a/examples/ih/ih_helper.py b/examples/ih/ih_helper.py index c9918856..35a39661 100644 --- a/examples/ih/ih_helper.py +++ b/examples/ih/ih_helper.py @@ -5,6 +5,9 @@ # Some day will move into a proper IH class +# ih.plots.gauge(conversion/engagement) etc +# constructor define objective (conversion and engagement) labels (positives/negatives) + class interaction_history: interactions_period_days = 21 accept_rate = 0.2 @@ -13,6 +16,9 @@ class interaction_history: convert_over_accept_rate_control = 0.3 convert_avg_duration_days = 2 + def __init__(self, outcome_definitions): + pass + def generate(self, n): now = datetime.datetime.now() diff --git a/python/pdstools/utils/cdh_utils.py b/python/pdstools/utils/cdh_utils.py index f26d728b..0258efdb 100644 --- a/python/pdstools/utils/cdh_utils.py +++ b/python/pdstools/utils/cdh_utils.py @@ -473,7 +473,7 @@ def auc_to_gini(auc: float) -> float: return 2 * safe_range_auc(auc) - 1 -def _capitalize(fields: Union[str, Iterable[str]]) -> List[str]: +def _capitalize(fields: Union[str, Iterable[str]], extra:Optional[List[str]]=[]) -> List[str]: """Applies automatic capitalization, aligned with the R couterpart. Parameters @@ -487,82 +487,83 @@ def _capitalize(fields: Union[str, Iterable[str]]) -> List[str]: The input list, but each value properly capitalized """ capitalize_endwords = [ - "Active", - "BinNegatives", - "BinPositives", - "BinResponseCount", - "BinSymbol", - "Bins", - "Cap", + "ID", + "Key", + "Name", + "Treatment", + "Count", "Category", "Class", - "Component", - "Configuration", - "ConfigurationName", - "Context", - "Control", - "Count", - "Date", + "Time", "DateTime", - "Description", - "Email", - "Enabled", + "UpdateTime", + "ToClass", + "Version", + "Predictor", + "Predictors", + "Rate", + "Ratio", + "Negatives", + "Positives", + "Threshold", "Error", - "Evidence", - "Execution", - "Group", - "GroupIndex", - "Hash", - "ID", - "Identifier", "Importance", + "Type", + "Percentage", "Index", - "Issue", - "Key", - "Limit", + "Symbol", "LowerBound", - "Message", - "ModelTechnique", - "Name", - "Negatives", + "UpperBound", + "Bins", + "GroupIndex", + "ResponseCount", "NegativesPercentage", - "Offline", - "Omni", - "Outcome", - "Paid", - "Percentage", - "Performance", - "Positives", "PositivesPercentage", - "Prediction", - "Predictor", - "Predictors", - "Propensity", - "Proposition", - "Rate", - "Ratio", - "Reference", - "Relevant", - "ResponseCount", + "BinPositives", + "BinNegatives", + "BinResponseCount", + "BinSymbol", "ResponseCountPercentage", + "ConfigurationName", + "Configuration", "SMS", - "Stage", - "Strategy", - "Subject", - "Symbol", - "Template", - "Threshold", - "Time", - "ToClass", - "Treatment", - "Type", + "Relevant", + "Proposition", + "Active", + "Description", + "Reference", + "Date", + "Performance", + "Identifier", + "Component", + "Prediction", + "Outcome", + "Hash", "URL", - "Update", - "UpdateTime", - "UpperBound", - "Version", + "Cap", + "Template", + "Issue", + "Group", + "Control", + "Evidence", + "Propensity", + "Paid", + "Subject", + "Email", "Web", + "Context", + "Limit", + "Stage", + "Omni", + "Execution", + "Enabled", + "Message", + "Offline", + "Update", + "Strategy", + "ModelTechnique" ] + if not isinstance(fields, list): fields = [fields] fields = [re.sub("^p(x|y|z)", "", field.lower()) for field in fields] @@ -575,9 +576,9 @@ def _capitalize(fields: Union[str, Iterable[str]]) -> List[str]: return fields -def _polars_capitalize(df: F) -> F: +def _polars_capitalize(df: F, extra:Optional[List[str]]=[]) -> F: cols = df.collect_schema().names() - renamed_cols = _capitalize(cols) + renamed_cols = _capitalize(cols, extra) def deduplicate(columns: List[str]): seen: Dict[str, int] = {} From 5cd3106765501fc5f0d549e0a3704696a9c3b86f Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Wed, 18 Dec 2024 16:34:17 +0100 Subject: [PATCH 14/21] First cut at implementing an IH class --- .../ih/Conversion_Modeling_Reporting.ipynb | 10515 +--------------- python/pdstools/__init__.py | 2 + python/pdstools/ih/Aggregates.py | 71 +- python/pdstools/ih/IH.py | 106 +- python/pdstools/ih/Plots.py | 134 +- python/pdstools/ih/__init__.py | 3 + python/pdstools/reports/HealthCheck.qmd | 2 - 7 files changed, 374 insertions(+), 10459 deletions(-) diff --git a/examples/ih/Conversion_Modeling_Reporting.ipynb b/examples/ih/Conversion_Modeling_Reporting.ipynb index 0f61ef7f..f8c86aec 100644 --- a/examples/ih/Conversion_Modeling_Reporting.ipynb +++ b/examples/ih/Conversion_Modeling_Reporting.ipynb @@ -2,3905 +2,12 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - " \n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import polars as pl\n", - "from pdstools import read_ds_export\n", + "from pdstools import read_ds_export, IH\n", "from pdstools.utils import cdh_utils\n", "from ih_helper import interaction_history\n", "\n", @@ -3914,190 +21,51 @@ "pio.renderers.default = \"vscode\"" ] }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (4, 7)
ChannelExperimentGroupConversionPendingAcceptedClickedImpression
strstru32u32u32u32u32
"Web""Conversion-Test"2310nullnull494824929
"Web""Conversion-Control"1384nullnull503225129
"Email""Conversion-Test"2358250705066nullnull
"Email""Conversion-Control"1379248714950nullnull
" - ], - "text/plain": [ - "shape: (4, 7)\n", - "┌─────────┬────────────────────┬────────────┬─────────┬──────────┬─────────┬────────────┐\n", - "│ Channel ┆ ExperimentGroup ┆ Conversion ┆ Pending ┆ Accepted ┆ Clicked ┆ Impression │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ u32 ┆ u32 ┆ u32 ┆ u32 ┆ u32 │\n", - "╞═════════╪════════════════════╪════════════╪═════════╪══════════╪═════════╪════════════╡\n", - "│ Web ┆ Conversion-Test ┆ 2310 ┆ null ┆ null ┆ 4948 ┆ 24929 │\n", - "│ Web ┆ Conversion-Control ┆ 1384 ┆ null ┆ null ┆ 5032 ┆ 25129 │\n", - "│ Email ┆ Conversion-Test ┆ 2358 ┆ 25070 ┆ 5066 ┆ null ┆ null │\n", - "│ Email ┆ Conversion-Control ┆ 1379 ┆ 24871 ┆ 4950 ┆ null ┆ null │\n", - "└─────────┴────────────────────┴────────────┴─────────┴──────────┴─────────┴────────────┘" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from pathlib import Path\n", - "\n", - "# we really only need a few columns\n", - "# Outcome outcomes: Conversionm, Impression, Pending\n", - "ih_cols = [\n", - " \"pyOutcome\",\n", - " \"pxOutcomeTime\",\n", - " \"pyChannel\",\n", - " \"pyIssue\",\n", - " \"pyGroup\",\n", - " \"pyName\",\n", - " \"ExperimentGroup\",\n", - "]\n", - "\n", - "ih_export_file = Path(\n", - " \"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip \"\n", - ")\n", - "if not ih_export_file.exists():\n", - " ih = interaction_history().generate(100000).select(ih_cols)\n", - "else:\n", - " ih = read_ds_export(ih_export_file).select(ih_cols)\n", - "ih = cdh_utils._polars_capitalize(ih)\n", - "ih = ih.filter(pl.col(\"ExperimentGroup\").is_not_null())\n", - "ih.collect().group_by([\"Channel\", \"ExperimentGroup\", \"Outcome\"]).agg(pl.len()).pivot(\n", - " on=\"Outcome\", index=[\"Channel\", \"ExperimentGroup\"]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "At first, take a look into the IH dataframe, explore the columns, outcome types and business structure" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (9, 13)
statisticOutcomeOutcomeTimeChannelIssueGroupNameExperimentGroupOutcomeDateTimeDayMonthYearQuarter
strstrstrstrstrstrstrstrstrstrstrstrstr
"count""127426""127426""127426""127426""127426""127426""127426""127426""127426""127426""127426""127426"
"null_count""0""0""0""0""0""0""0""0""0""0""0""0"
"mean"nullnullnullnullnullnullnull"2024-12-06 05:34:26.383000""2024-12-05 17:35:55.434000"nullnullnull
"std"nullnullnullnullnullnullnullnullnullnullnullnull
"min""Accepted""20241125T164722""Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control""2024-11-25 16:47:22""2024-11-25""2024-11""2024""2024_Q4"
"25%"nullnullnullnullnullnullnull"2024-12-01 00:15:40""2024-12-01"nullnullnull
"50%"nullnullnullnullnullnullnull"2024-12-06 05:10:39""2024-12-06"nullnullnull
"75%"nullnullnullnullnullnullnull"2024-12-11 11:11:34""2024-12-11"nullnullnull
"max""Pending""20241216T164646""Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test""2024-12-16 16:46:46""2024-12-16""2024-12""2024""2024_Q4"
" - ], - "text/plain": [ - "shape: (9, 13)\n", - "┌────────────┬──────────┬───────────────┬─────────┬───┬───────────────┬─────────┬────────┬─────────┐\n", - "│ statistic ┆ Outcome ┆ OutcomeTime ┆ Channel ┆ … ┆ Day ┆ Month ┆ Year ┆ Quarter │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ str ┆ str ┆ ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════╪═══════════════╪═════════╪═══╪═══════════════╪═════════╪════════╪═════════╡\n", - "│ count ┆ 127426 ┆ 127426 ┆ 127426 ┆ … ┆ 127426 ┆ 127426 ┆ 127426 ┆ 127426 │\n", - "│ null_count ┆ 0 ┆ 0 ┆ 0 ┆ … ┆ 0 ┆ 0 ┆ 0 ┆ 0 │\n", - "│ mean ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-05 ┆ null ┆ null ┆ null │\n", - "│ ┆ ┆ ┆ ┆ ┆ 17:35:55.4340 ┆ ┆ ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ 00 ┆ ┆ ┆ │\n", - "│ std ┆ null ┆ null ┆ null ┆ … ┆ null ┆ null ┆ null ┆ null │\n", - "│ min ┆ Accepted ┆ 20241125T1647 ┆ Email ┆ … ┆ 2024-11-25 ┆ 2024-11 ┆ 2024 ┆ 2024_Q4 │\n", - "│ ┆ ┆ 22 ┆ ┆ ┆ ┆ ┆ ┆ │\n", - "│ 25% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-01 ┆ null ┆ null ┆ null │\n", - "│ 50% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-06 ┆ null ┆ null ┆ null │\n", - "│ 75% ┆ null ┆ null ┆ null ┆ … ┆ 2024-12-11 ┆ null ┆ null ┆ null │\n", - "│ max ┆ Pending ┆ 20241216T1646 ┆ Web ┆ … ┆ 2024-12-16 ┆ 2024-12 ┆ 2024 ┆ 2024_Q4 │\n", - "│ ┆ ┆ 46 ┆ ┆ ┆ ┆ ┆ ┆ │\n", - "└────────────┴──────────┴───────────────┴─────────┴───┴───────────────┴─────────┴────────┴─────────┘" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ih = ih.with_columns(\n", - " pl.col(\"OutcomeTime\")\n", - " .str.strptime(pl.Datetime, \"%Y%m%dT%H%M%S%.3f %Z\")\n", - " .alias(\"OutcomeDateTime\")\n", - ").with_columns(\n", - " [\n", - " pl.col(\"OutcomeDateTime\").dt.date().alias(\"Day\"),\n", - " (pl.col(\"OutcomeDateTime\").dt.strftime(\"%Y-%m\")).alias(\"Month\"),\n", - " pl.col(\"OutcomeDateTime\").dt.year().cast(str).alias(\"Year\"),\n", - " (\n", - " pl.col(\"OutcomeDateTime\").dt.year().cast(str)\n", - " + \"_Q\"\n", - " + pl.col(\"OutcomeDateTime\").dt.quarter().cast(str)\n", - " ).alias(\"Quarter\"),\n", - " ]\n", - ")\n", - "ih.describe()" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Assuming conversion modelling setup folllows OOTB approach, so that IH contains Conversion outcome as positive result and Impression (for inbound channels) and Pending (for outbound channels) are treated as negative outcome. Each Conversion has corresponding Impression/Pending record, so to calculate correct Conversion Rate is count(Conversion) / (count(Impression/Pending) - 2 * count(Conversion))" + "# Conversion Results" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "positive_model_response = [\"Conversion\"]\n", - "all_model_response = [\"Impression\", \"Pending\"]\n", - "group_by = [\n", - " \"Day\",\n", - " \"Month\",\n", - " \"Year\",\n", - " \"Quarter\",\n", - " \"Channel\",\n", - " \"Issue\",\n", - " \"Group\",\n", - " \"Name\",\n", - " \"ExperimentGroup\",\n", - "]" + "from pathlib import Path\n", + "\n", + "ih_export_file = Path(\n", + " \"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\"\n", + ")\n", + "\n", + "if not ih_export_file.exists():\n", + " ih = IH.from_mock_data()\n", + "else:\n", + " ih = IH.from_ds_export(ih_export_file)\n", + "\n", + "ih.aggregates.summary_by_experiment(\n", + " experiment_field=\"ExperimentGroup\",\n", + " by=[\"Channel\"],\n", + " positive_labels=[\"Conversion\"],\n", + " negative_labels=[\"Impression\", \"Pending\"],\n", + ").collect()" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "ih_analysis = (\n", - " ih.filter((pl.col(\"Outcome\").is_in(all_model_response + positive_model_response)))\n", - " .with_columns(\n", - " [\n", - " pl.when(pl.col(\"Outcome\").is_in(positive_model_response))\n", - " .then(1)\n", - " .otherwise(0)\n", - " .alias(\"Outcome_Binary\")\n", - " ]\n", - " )\n", - " .group_by(group_by)\n", - " .agg([pl.len().alias(\"Count\"), pl.sum(\"Outcome_Binary\").alias(\"Positives\")])\n", - " .with_columns([(pl.col(\"Count\") - (2 * pl.col(\"Positives\"))).alias(\"Negatives\")])\n", + "ih.plots.experiment_gauges(\n", + " experiment_field=\"ExperimentGroup\",\n", + " by=\"Channel\",\n", + " positive_labels=[\"Conversion\"],\n", + " negative_labels=[\"Impression\", \"Pending\"],\n", + " reference_values={\"Web\": 0.055, \"Email\": 0.09},\n", + " title = \"Overall Conversion\",\n", ")" ] }, @@ -4105,6429 +73,68 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Showing results as a gauge plot across channel dimension to compare conversion rates inside specific channel between conversion and Engagement models. Set relevant reference data (baseline conversion rate). Delta from baseline is shown inside Gauge." + "## Detailed View " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (4, 7)
ChannelExperimentGroupNegativesPositivesCountConversionRateStdErr
strstru32i32u32f64f64
"Email""Conversion-Control"234921379262500.0554460.001451
"Email""Conversion-Test"227122358274280.0940570.001844
"Web""Conversion-Control"237451384265130.0550760.001439
"Web""Conversion-Test"226192310272390.0926630.001836
" - ], - "text/plain": [ - "shape: (4, 7)\n", - "┌─────────┬────────────────────┬───────────┬───────────┬───────┬────────────────┬──────────┐\n", - "│ Channel ┆ ExperimentGroup ┆ Negatives ┆ Positives ┆ Count ┆ ConversionRate ┆ StdErr │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ u32 ┆ i32 ┆ u32 ┆ f64 ┆ f64 │\n", - "╞═════════╪════════════════════╪═══════════╪═══════════╪═══════╪════════════════╪══════════╡\n", - "│ Email ┆ Conversion-Control ┆ 23492 ┆ 1379 ┆ 26250 ┆ 0.055446 ┆ 0.001451 │\n", - "│ Email ┆ Conversion-Test ┆ 22712 ┆ 2358 ┆ 27428 ┆ 0.094057 ┆ 0.001844 │\n", - "│ Web ┆ Conversion-Control ┆ 23745 ┆ 1384 ┆ 26513 ┆ 0.055076 ┆ 0.001439 │\n", - "│ Web ┆ Conversion-Test ┆ 22619 ┆ 2310 ┆ 27239 ┆ 0.092663 ┆ 0.001836 │\n", - "└─────────┴────────────────────┴───────────┴───────────┴───────┴────────────────┴──────────┘" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "gauge_group_by = [\"Channel\", \"ExperimentGroup\"]\n", - "reference = {\"WebConversion-Test\": 0.055, \"WebConversion-Control\": 0.055, \"EmailConversion-Test\": 0.09, \"EmailConversion-Control\": 0.09}\n", - "gauge_data = (\n", - " ih_analysis.group_by(gauge_group_by)\n", - " .agg(pl.sum([\"Negatives\", \"Positives\", \"Count\"]))\n", - " .with_columns(\n", - " [\n", - " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", - " \"ConversionRate\"\n", - " )\n", - " ]\n", - " )\n", - " .with_columns(\n", - " [\n", - " (\n", - " (\n", - " (\n", - " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", - " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " ).sqrt()\n", - " )\n", - " ).alias(\"StdErr\")\n", - " ]\n", - " )\n", - " .sort(gauge_group_by, descending=False)\n", - " .collect()\n", - ")\n", - "gauge_data" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "delta": { - "reference": 0.09, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0, - 0.45 - ], - "y": [ - 0.575, - 1 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "bar": { - "color": "#EC5300" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.09 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "EmailConversion-Control" - }, - "type": "indicator", - "value": 0.05544610188573037 - }, - { - "delta": { - "reference": 0.09, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0.55, - 1 - ], - "y": [ - 0.575, - 1 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.09 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "EmailConversion-Test" - }, - "type": "indicator", - "value": 0.09405664140406861 - }, - { - "delta": { - "reference": 0.055, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0, - 0.45 - ], - "y": [ - 0, - 0.425 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.055 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "WebConversion-Control" - }, - "type": "indicator", - "value": 0.05507580882645549 - }, - { - "delta": { - "reference": 0.055, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0.55, - 1 - ], - "y": [ - 0, - 0.425 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.055 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "WebConversion-Test" - }, - "type": "indicator", - "value": 0.0926631633840106 - } - ], - "layout": { - "autosize": true, - "height": 540, - "margin": { - "b": 10, - "l": 10, - "r": 10, - "t": 120 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[CONV] Conversion (Channel/Model Type)" - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", - "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", - "\n", - "gauge_data = gauge_data.with_columns(\n", - " pl.concat_str(gauge_group_by).alias(\"Name\"),\n", - " # pl.concat_str(gauge_group_by, separator = \"_\").alias('CName'),\n", - ")\n", - "\n", - "fig = make_subplots(\n", - " rows=rows,\n", - " cols=cols,\n", - " specs=[[{\"type\": \"indicator\"} for c in range(cols)] for t in range(rows)],\n", - ")\n", - "fig.update_layout(\n", - " height=270 * rows,\n", - " autosize=True,\n", - " title=\"[CONV] Conversion (Channel/Model Type)\",\n", - " margin=dict(b=10, t=120, l=10, r=10),\n", - ")\n", - "index = 0\n", - "for row in gauge_data.iter_rows(named=True):\n", - " ref_value = reference.get(row[\"Name\"], None)\n", - " gauge = {\n", - " \"axis\": {\"tickformat\": \",.2%\"},\n", - " \"threshold\": {\n", - " \"line\": {\"color\": \"red\", \"width\": 2},\n", - " \"thickness\": 0.75,\n", - " \"value\": ref_value,\n", - " },\n", - " }\n", - " if ref_value:\n", - " if row[\"ConversionRate\"] < ref_value:\n", - " gauge = {\n", - " \"axis\": {\"tickformat\": \",.2%\"},\n", - " \"bar\": {\n", - " \"color\": (\n", - " \"#EC5300\"\n", - " if row[\"ConversionRate\"] < (0.75 * ref_value)\n", - " else \"#EC9B00\"\n", - " )\n", - " },\n", - " \"threshold\": {\n", - " \"line\": {\"color\": \"red\", \"width\": 2},\n", - " \"thickness\": 0.75,\n", - " \"value\": ref_value,\n", - " },\n", - " }\n", - "\n", - " trace1 = go.Indicator(\n", - " mode=\"gauge+number+delta\",\n", - " number={\"valueformat\": \",.2%\"},\n", - " value=row[\"ConversionRate\"],\n", - " delta={\"reference\": ref_value, \"valueformat\": \",.2%\"},\n", - " title={\"text\": row[\"Name\"]},\n", - " gauge=gauge,\n", - " )\n", - " r, c = divmod(index, cols)\n", - " fig.add_trace(trace1, row=(r + 1), col=(c + 1))\n", - " index = index + 1\n", - "\n", - "fig.show()" + "# TODO maybe this would only be useful for the test group\n", + "# then in the hovers show both test and control values\n", + "ih.plots.tree_map(\n", + " experiment_field=\"ExperimentGroup\", # should be optional, can also give query to select only the Test group\n", + " positive_labels=[\"Conversion\"],\n", + " negative_labels=[\"Impression\", \"Pending\"],\n", + " title=\"Conversion rates for all Actions\",\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "This plot provides detailed view on individual actions conversion rates and model type used." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "branchvalues": "total", - "customdata": [ - [ - "0.0014391011575603573", - "1384", - "23745", - 0.05507580882645549 - ], - [ - "0.0018436058995561235", - "2358", - "22712", - 0.09405664140406861 - ], - [ - "0.0018364766203625416", - "2310", - "22619", - 0.0926631633840106 - ], - [ - "0.0014511164251962663", - "1379", - "23492", - 0.05544610188573037 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07517503884144745 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07412332242210297 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07412332242210297 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07517503884144745 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07517503884144745 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07412332242210297 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07517503884144745 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07412332242210297 - ], - [ - "(?)", - "(?)", - "(?)", - 0.07464881840979332 - ] - ], - "domain": { - "x": [ - 0, - 1 - ], - "y": [ - 0, - 1 - ] - }, - "hovertemplate": "labels=%{label}
Count=%{value}
parent=%{parent}
id=%{id}
StdErr=%{customdata[0]}
Positives=%{customdata[1]}
Negatives=%{customdata[2]}
ConversionRate=%{color}", - "ids": [ - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Control", - "ALL/Email/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB/Conversion-Test", - "ALL/Email/Acquisition/Phones/AppleIPhone1564GB/Conversion-Control", - "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Web/Acquisition/Phones", - "ALL/Email/Acquisition/Phones", - "ALL/Email/Acquisition", - "ALL/Web/Acquisition", - "ALL/Email", - "ALL/Web", - "ALL" - ], - "labels": [ - "Conversion-Control", - "Conversion-Test", - "Conversion-Test", - "Conversion-Control", - "AppleIPhone1564GB", - "AppleIPhone1564GB", - "Phones", - "Phones", - "Acquisition", - "Acquisition", - "Email", - "Web", - "ALL" - ], - "marker": { - "coloraxis": "coloraxis", - "colors": { - "bdata": "NLZre+UyrD+8VM2WGBS4Pxg6R+jFuLc/O22UdG5jrD8Vr0zdqz6zP+F9rP2++bI/4X2s/b75sj8Vr0zdqz6zPxWvTN2rPrM/4X2s/b75sj8Vr0zdqz6zP+F9rP2++bI/kE7BWS8csz8=", - "dtype": "f8" - } - }, - "name": "", - "parents": [ - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Web/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Email/Acquisition/Phones/AppleIPhone1564GB", - "ALL/Email/Acquisition/Phones", - "ALL/Web/Acquisition/Phones", - "ALL/Web/Acquisition", - "ALL/Email/Acquisition", - "ALL/Email", - "ALL/Web", - "ALL", - "ALL", - "" - ], - "textinfo": "label+value+percent parent+percent root", - "type": "treemap", - "values": { - "bdata": "AAAAAEDk2UAAAAAAAMnaQAAAAADAmdpAAAAAAICi2UAAAAAAwDXqQAAAAAAAP+pAAAAAAAA/6kAAAAAAwDXqQAAAAADANepAAAAAAAA/6kAAAAAAwDXqQAAAAAAAP+pAAAAAAGA6+kA=", - "dtype": "f8" - } - } - ], - "layout": { - "coloraxis": { - "colorbar": { - "title": { - "text": "ConversionRate" - } - }, - "colorscale": [ - [ - 0, - "rgb(5,48,97)" - ], - [ - 0.1, - "rgb(33,102,172)" - ], - [ - 0.2, - "rgb(67,147,195)" - ], - [ - 0.3, - "rgb(146,197,222)" - ], - [ - 0.4, - "rgb(209,229,240)" - ], - [ - 0.5, - "rgb(247,247,247)" - ], - [ - 0.6, - "rgb(253,219,199)" - ], - [ - 0.7, - "rgb(244,165,130)" - ], - [ - 0.8, - "rgb(214,96,77)" - ], - [ - 0.9, - "rgb(178,24,43)" - ], - [ - 1, - "rgb(103,0,31)" - ] - ] - }, - "height": 640, - "legend": { - "tracegroupgap": 0 - }, - "margin": { - "b": 25, - "l": 25, - "r": 25, - "t": 50 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[BIZ] Conversion rate treemap" - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "shape: (4, 10)
ChannelIssueGroupNameExperimentGroupNegativesPositivesCountConversionRateStdErr
strstrstrstrstru32i32u32f64f64
"Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control"234921379262500.0554460.001451
"Email""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test"227122358274280.0940570.001844
"Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Control"237451384265130.0550760.001439
"Web""Acquisition""Phones""AppleIPhone1564GB""Conversion-Test"226192310272390.0926630.001836
" - ], - "text/plain": [ - "shape: (4, 10)\n", - "┌─────────┬─────────────┬────────┬───────────────┬───┬───────────┬───────┬──────────────┬──────────┐\n", - "│ Channel ┆ Issue ┆ Group ┆ Name ┆ … ┆ Positives ┆ Count ┆ ConversionRa ┆ StdErr │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ te ┆ --- │\n", - "│ str ┆ str ┆ str ┆ str ┆ ┆ i32 ┆ u32 ┆ --- ┆ f64 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ f64 ┆ │\n", - "╞═════════╪═════════════╪════════╪═══════════════╪═══╪═══════════╪═══════╪══════════════╪══════════╡\n", - "│ Email ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 1379 ┆ 26250 ┆ 0.055446 ┆ 0.001451 │\n", - "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", - "│ Email ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 2358 ┆ 27428 ┆ 0.094057 ┆ 0.001844 │\n", - "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", - "│ Web ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 1384 ┆ 26513 ┆ 0.055076 ┆ 0.001439 │\n", - "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", - "│ Web ┆ Acquisition ┆ Phones ┆ AppleIPhone15 ┆ … ┆ 2310 ┆ 27239 ┆ 0.092663 ┆ 0.001836 │\n", - "│ ┆ ┆ ┆ 64GB ┆ ┆ ┆ ┆ ┆ │\n", - "└─────────┴─────────────┴────────┴───────────────┴───┴───────────┴───────┴──────────────┴──────────┘" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "treemap_group_by = [\"Channel\", \"Issue\", \"Group\", \"Name\", \"ExperimentGroup\"]\n", - "\n", - "treemap_data = (\n", - " ih_analysis.group_by(treemap_group_by)\n", - " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\"),\n", - " )\n", - " .with_columns(\n", - " [\n", - " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", - " \"ConversionRate\"\n", - " )\n", - " ]\n", - " )\n", - " .with_columns(\n", - " [\n", - " (\n", - " (\n", - " (\n", - " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", - " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " )\n", - " ** 0.5\n", - " )\n", - " ).alias(\"StdErr\")\n", - " ]\n", - " )\n", - " .sort(treemap_group_by, descending=False)\n", - " .collect()\n", - ")\n", - "\n", - "treemap_data = treemap_data\n", + "## Trends\n", "\n", - "fig = px.treemap(\n", - " treemap_data,\n", - " path=[px.Constant(\"ALL\")] + treemap_group_by,\n", - " values=\"Count\",\n", - " color=\"ConversionRate\",\n", - " color_continuous_scale=px.colors.sequential.RdBu_r,\n", - " title=\"[BIZ] Conversion rate treemap\",\n", - " hover_data=[\"StdErr\", \"Positives\", \"Negatives\"],\n", - " height=640,\n", - ")\n", - "fig.update_traces(textinfo=\"label+value+percent parent+percent root\")\n", - "fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))\n", - "\n", - "fig.show()\n", - "treemap_data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Detailed line/bar plot." + "side-by-side bars and lines (separate methods) with error bars" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ] - ], - "error_y": { - "array": { - "bdata": "3phnesMBhD9/yffWnl6AP4+VVEEZ/YY/Vmtn/o5jiz+a85LlufOJPypcmAaZSow/G49V5xO4jD9CBMwykc+LP8s/5UOm/os/wCzeBH1biz99Rtc/UNWKP8+ZI8MaxYo/ZW6984wsiz8R0dNIiB2KP2QJ2QPNYoo/dX6udzFbij+tYpsxqDeLP5DU/iMePo0/AdDtQW0Sjj8siHchC0GMP63SUkxte4k/J/LD1LIlkD8=", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", - "legendgroup": "Conversion-Control", - "marker": { - "color": "#636efa", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Control", - "offsetgroup": "Conversion-Control", - "orientation": "v", - "showlegend": true, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-11-25", - "2024-11-26", - "2024-11-27", - "2024-11-28", - "2024-11-29", - "2024-11-30", - "2024-12-01", - "2024-12-02", - "2024-12-03", - "2024-12-04", - "2024-12-05", - "2024-12-06", - "2024-12-07", - "2024-12-08", - "2024-12-09", - "2024-12-10", - "2024-12-11", - "2024-12-12", - "2024-12-13", - "2024-12-14", - "2024-12-15", - "2024-12-16" - ], - "xaxis": "x2", - "y": { - "bdata": "krGohtnBgT+OWDJgXMeTP1hPYT2F9aQ/24G5dmCurT8UHD3B0ROsPxAEQRAEQbA/ugE85d2esD99FOsqcw6vPx2vBUImC68/+NSN25RrrT+Gu/omeX+tP6T7C5/KUqw/zGEOc5jDrD8or6G8hvKqP+bTxUalj6w/4SQCD2nfqz/RfndxXbatP1EMZgRJ77A/wqYARzs9sj8CMYS/8x6wP34rfv461qk/HczBHMzBrD8=", - "dtype": "f8" - }, - "yaxis": "y2" - }, - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ] - ], - "error_y": { - "array": { - "bdata": "oJbfRua2gj83Rdq5q159P4tq0gQIlIU/x0Yrn4EXiT+LDk89aaGNPxDyNFyGg4s/CY/ItNlUiD/yflX8xiiIP7Quw1jQgYw/v+K828Vdiz8VxQfoyeyLPz5xXOQKeoo/Oelfc6Jmjj+iqs43VP6IPyFBJXEgHY0/siqq/cGziD/waO/gLTuMP73ZBIP5U40/ufMPio15iz+6i6ZbMr6LP8ZZjYY1bY0/bC8BeNakjj8=", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", - "legendgroup": "Conversion-Control", - "marker": { - "color": "#636efa", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Control", - "offsetgroup": "Conversion-Control", - "orientation": "v", - "showlegend": false, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-11-25", - "2024-11-26", - "2024-11-27", - "2024-11-28", - "2024-11-29", - "2024-11-30", - "2024-12-01", - "2024-12-02", - "2024-12-03", - "2024-12-04", - "2024-12-05", - "2024-12-06", - "2024-12-07", - "2024-12-08", - "2024-12-09", - "2024-12-10", - "2024-12-11", - "2024-12-12", - "2024-12-13", - "2024-12-14", - "2024-12-15", - "2024-12-16" - ], - "xaxis": "x", - "y": { - "bdata": "sAlB2fuagD9qNuzvYlCRP5gin3WDKaI/y7hl3DJuqT93mYTNPA2xP1Pks24wRa4/RyIgPpqQpz8gWvwo2SynP87tJvTLtrA/a3oFmhvDrj95tPMD0byuPxzHcRzHcaw/c3Nzc3Nzsz/T2611TRypP1hlUgFZvrA/fvN96K4nqD802j1sfRuwP241gtVJ4bE/CJ5VDzyrrj8faPoeaPquPwXJn1dZnLE/Fdt6w/ugqj8=", - "dtype": "f8" - }, - "yaxis": "y" - }, - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ] - ], - "error_y": { - "array": { - "bdata": "tAFsSta9hj8L8ji12BeHP2uErbQh1oo/WeTK8kqSjz/cNVUG14uRPyoka/XEdpE/IGEv33dCkj/F/mBbQm6RP+C9HoUBdJM//oNbPgVAkj9f66GfiTmSP9knedqpyZA/Qj3P/QVXjj/GfFPemDiRP3ey5JJP6Y8/4j+Uj3+VkT+3ZWJ0IPaQP98l8yGZ1pA/kJwqn0xokT92psU/Bz2RPzxlzWNo0ZI/yr9T4oIglj8=", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", - "legendgroup": "Conversion-Test", - "marker": { - "color": "#EF553B", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Test", - "offsetgroup": "Conversion-Test", - "orientation": "v", - "showlegend": true, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-11-25", - "2024-11-26", - "2024-11-27", - "2024-11-28", - "2024-11-29", - "2024-11-30", - "2024-12-01", - "2024-12-02", - "2024-12-03", - "2024-12-04", - "2024-12-05", - "2024-12-06", - "2024-12-07", - "2024-12-08", - "2024-12-09", - "2024-12-10", - "2024-12-11", - "2024-12-12", - "2024-12-13", - "2024-12-14", - "2024-12-15", - "2024-12-16" - ], - "xaxis": "x2", - "y": { - "bdata": "bXUBwspWhz8yrbajyX2lP8GnQP5ZZaw/JyDN6DTGtT/exqhXRYO5P4GEn1yIurk/iYakAPd9vD/OE96eciC6P/4XsF7SfsA/HMIhHMIhvD9q3mvviDS8P8MjSkI4Hbc/av1KgVq/sj+TRQ4QUty3P+QYrK6CB7U/mr4Hmr4Huj8a7R62vg24P3rhwfbcpLc/hqgBKGWIuj+HWRzrASe5P7QMoYS4W74/fKSVyRMQvT8=", - "dtype": "f8" - }, - "yaxis": "y2" - }, - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ] - ], - "error_y": { - "array": { - "bdata": "wej/AubJij/OsFC07yKFP/0wrogfBYw/8I7SDM8OkT/22KH4RkmRPxagz5rYGJE/olrup9fnkD8PIAGvbM6SP6Ja7qfX55A/a3oK2u2pkT+vJEZFX46RP1XkTE0hmpE/taOSqEs3kT8DJ4xENp+QPx5kr9eeJpE/vVqJ6B7TkT8jP186UkWRP0XfdaXHAZA/ulrXTR0nkj+24dNTD5iRP8C93aVev5E/cczWlVvYlD8=", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
Conversion Rate : %{y:.2%}", - "legendgroup": "Conversion-Test", - "marker": { - "color": "#EF553B", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Test", - "offsetgroup": "Conversion-Test", - "orientation": "v", - "showlegend": false, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-11-25", - "2024-11-26", - "2024-11-27", - "2024-11-28", - "2024-11-29", - "2024-11-30", - "2024-12-01", - "2024-12-02", - "2024-12-03", - "2024-12-04", - "2024-12-05", - "2024-12-06", - "2024-12-07", - "2024-12-08", - "2024-12-09", - "2024-12-10", - "2024-12-11", - "2024-12-12", - "2024-12-13", - "2024-12-14", - "2024-12-15", - "2024-12-16" - ], - "xaxis": "x", - "y": { - "bdata": "EQ7hEA7hkD+UWSwNDiChP7Xo3Si3F7A/7QViBszLtj+CPRKml3O5P4ZckQrlXrg/5d2ekGG+tz/jJPSMyaO9P+XdnpBhvrc/3i/w4lt/uj+6RHhSkPy5P72q/UwTg7o/6ifWilHkuD9WJIHycOC3PztDYt7OkLg/rlHc6UDeuj/6GJyPwfm4P6fFohyfGbU/CIZki2jOuj8XqmSUI9G5P+9h69uAobo/XMRm8BaxuT8=", - "dtype": "f8" - }, - "yaxis": "y" - } - ], - "layout": { - "annotations": [ - { - "font": {}, - "showarrow": false, - "text": "Web", - "textangle": 90, - "x": 0.98, - "xanchor": "left", - "xref": "paper", - "y": 0.2425, - "yanchor": "middle", - "yref": "paper" - }, - { - "font": {}, - "showarrow": false, - "text": "Email", - "textangle": 90, - "x": 0.98, - "xanchor": "left", - "xref": "paper", - "y": 0.7575000000000001, - "yanchor": "middle", - "yref": "paper" - } - ], - "barmode": "group", - "height": 640, - "hovermode": "x unified", - "legend": { - "title": { - "text": "ExperimentGroup" - }, - "tracegroupgap": 0 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[CONV] Daily Conversion Rate with 95% confidence interval" - }, - "updatemenus": [ - { - "buttons": [ - { - "args": [ - "type", - "bar" - ], - "label": "Bar", - "method": "restyle" - }, - { - "args": [ - "type", - "line" - ], - "label": "Line", - "method": "restyle" - } - ], - "direction": "down", - "showactive": true - } - ], - "xaxis": { - "anchor": "y", - "domain": [ - 0, - 0.98 - ], - "tickfont": { - "size": 10 - }, - "title": { - "text": "Day" - } - }, - "xaxis2": { - "anchor": "y2", - "domain": [ - 0, - 0.98 - ], - "matches": "x", - "showticklabels": false, - "tickfont": { - "size": 10 - } - }, - "yaxis": { - "anchor": "x", - "domain": [ - 0, - 0.485 - ], - "tickformat": ",.2%", - "title": { - "text": "Conversion Rate" - } - }, - "yaxis2": { - "anchor": "x2", - "domain": [ - 0.515, - 1 - ], - "matches": "y", - "tickformat": ",.2%", - "title": { - "text": "ConversionRate" - } - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "shape: (88, 8)
DayChannelExperimentGroupNegativesPositivesCountConversionRateCI
datestrstru32i32u32f64f64
2024-11-25"Email""Conversion-Control"34333490.0086710.009769
2024-11-25"Email""Conversion-Test"34743550.0113960.011104
2024-11-25"Web""Conversion-Control"36733730.0081080.009138
2024-11-25"Web""Conversion-Test"35863700.0164840.01308
2024-11-26"Email""Conversion-Control"11172211610.0193150.007993
……………………
2024-12-15"Web""Conversion-Test"106812413160.1040270.017332
2024-12-16"Email""Conversion-Control"773468650.0561660.015769
2024-12-16"Email""Conversion-Test"734949220.1135270.021608
2024-12-16"Web""Conversion-Control"802448900.0520090.014963
2024-12-16"Web""Conversion-Test"753849210.1003580.020357
" - ], - "text/plain": [ - "shape: (88, 8)\n", - "┌────────────┬─────────┬────────────────┬───────────┬───────────┬───────┬───────────────┬──────────┐\n", - "│ Day ┆ Channel ┆ ExperimentGrou ┆ Negatives ┆ Positives ┆ Count ┆ ConversionRat ┆ CI │\n", - "│ --- ┆ --- ┆ p ┆ --- ┆ --- ┆ --- ┆ e ┆ --- │\n", - "│ date ┆ str ┆ --- ┆ u32 ┆ i32 ┆ u32 ┆ --- ┆ f64 │\n", - "│ ┆ ┆ str ┆ ┆ ┆ ┆ f64 ┆ │\n", - "╞════════════╪═════════╪════════════════╪═══════════╪═══════════╪═══════╪═══════════════╪══════════╡\n", - "│ 2024-11-25 ┆ Email ┆ Conversion-Con ┆ 343 ┆ 3 ┆ 349 ┆ 0.008671 ┆ 0.009769 │\n", - "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-11-25 ┆ Email ┆ Conversion-Tes ┆ 347 ┆ 4 ┆ 355 ┆ 0.011396 ┆ 0.011104 │\n", - "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-11-25 ┆ Web ┆ Conversion-Con ┆ 367 ┆ 3 ┆ 373 ┆ 0.008108 ┆ 0.009138 │\n", - "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-11-25 ┆ Web ┆ Conversion-Tes ┆ 358 ┆ 6 ┆ 370 ┆ 0.016484 ┆ 0.01308 │\n", - "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-11-26 ┆ Email ┆ Conversion-Con ┆ 1117 ┆ 22 ┆ 1161 ┆ 0.019315 ┆ 0.007993 │\n", - "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2024-12-15 ┆ Web ┆ Conversion-Tes ┆ 1068 ┆ 124 ┆ 1316 ┆ 0.104027 ┆ 0.017332 │\n", - "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-12-16 ┆ Email ┆ Conversion-Con ┆ 773 ┆ 46 ┆ 865 ┆ 0.056166 ┆ 0.015769 │\n", - "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-12-16 ┆ Email ┆ Conversion-Tes ┆ 734 ┆ 94 ┆ 922 ┆ 0.113527 ┆ 0.021608 │\n", - "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-12-16 ┆ Web ┆ Conversion-Con ┆ 802 ┆ 44 ┆ 890 ┆ 0.052009 ┆ 0.014963 │\n", - "│ ┆ ┆ trol ┆ ┆ ┆ ┆ ┆ │\n", - "│ 2024-12-16 ┆ Web ┆ Conversion-Tes ┆ 753 ┆ 84 ┆ 921 ┆ 0.100358 ┆ 0.020357 │\n", - "│ ┆ ┆ t ┆ ┆ ┆ ┆ ┆ │\n", - "└────────────┴─────────┴────────────────┴───────────┴───────────┴───────┴───────────────┴──────────┘" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", - "\n", - "line_data = (\n", - " ih_analysis.group_by(line_group_by)\n", - " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\"),\n", - " )\n", - " .with_columns(\n", - " [\n", - " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", - " \"ConversionRate\"\n", - " )\n", - " ]\n", - " )\n", - " .with_columns(\n", - " [\n", - " (\n", - " (\n", - " (\n", - " (\n", - " (pl.col(\"ConversionRate\") * (1 - pl.col(\"ConversionRate\")))\n", - " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " )\n", - " ** 0.5\n", - " )\n", - " * 1.96\n", - " )\n", - " ).alias(\"CI\")\n", - " ]\n", - " )\n", - " .sort(line_group_by, descending=False)\n", - " .collect()\n", - ")\n", - "\n", - "line_data = line_data\n", - "\n", - "if len(line_data[\"Day\"].unique()) < 30:\n", - " fig = px.bar(\n", - " line_data,\n", - " x=\"Day\",\n", - " y=\"ConversionRate\",\n", - " color=\"ExperimentGroup\",\n", - " error_y=\"CI\",\n", - " facet_row=\"Channel\",\n", - " barmode=\"group\",\n", - " title=\"[CONV] Daily Conversion Rate with 95% confidence interval\",\n", - " custom_data=[\"ExperimentGroup\"],\n", - " )\n", - " fig.update_layout(\n", - " updatemenus=[\n", - " dict(\n", - " buttons=list(\n", - " [\n", - " dict(args=[\"type\", \"bar\"], label=\"Bar\", method=\"restyle\"),\n", - " dict(args=[\"type\", \"line\"], label=\"Line\", method=\"restyle\"),\n", - " ]\n", - " ),\n", - " direction=\"down\",\n", - " showactive=True,\n", - " ),\n", - " ]\n", - " )\n", - "else:\n", - " fig = px.line(\n", - " line_data,\n", - " x=\"Day\",\n", - " y=\"ConversionRate\",\n", - " color=\"ExperimentGroup\",\n", - " title=\"[CONV] Daily Conversion Rate\",\n", - " acet_row=\"Channel\",\n", - " custom_data=[\"ExperimentGroup\"],\n", - " )\n", - "\n", - "fig.update_xaxes(tickfont=dict(size=10))\n", - "fig.update_yaxes(tickformat=\",.2%\")\n", - "yaxis_names = [\"yaxis\"] + [\n", - " axis_name for axis_name in fig.layout._subplotid_props if \"yaxis\" in axis_name\n", - "]\n", - "yaxis_layout_dict = {yaxis_name + \"_tickformat\": \",.2%\" for yaxis_name in yaxis_names}\n", - "fig.update_layout(yaxis_layout_dict)\n", - "height = max(640, 300 * len(line_data[\"Channel\"].unique()))\n", - "fig.update_layout(\n", - " xaxis_title=\"Day\",\n", - " yaxis_title=\"Conversion Rate\",\n", - " hovermode=\"x unified\",\n", - " height=height,\n", - ")\n", - "fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[1]))\n", - "fig = fig.update_traces(\n", - " hovertemplate=\"Day\"\n", - " + \" : %{x}\"\n", - " + \"
\"\n", - " + \"Experiment Group\"\n", - " + \" : %{customdata[0]}\"\n", - " + \"
\"\n", - " + \"Conversion Rate\"\n", - " + \" : %{y:.2%}\"\n", - " + \"\"\n", - ")\n", - "\n", - "fig.show()\n", - "line_data" + "# ih.plots.trend_line(\n", + "# experiment_field=\"ExperimentGroup\", # should be optional, can also give query to select only the Test group\n", + "# granularity=\"1d\", # string language polars\n", + "# positive_labels=[\"Conversion\"],\n", + "# negative_labels=[\"Impression\", \"Pending\"],\n", + "# title=\"Conversion Rate trends\",\n", + "# )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Engagement rates (CTR)" + "# Engagement" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "delta": { - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0, - 0.45 - ], - "y": [ - 0.575, - 1 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "EmailConversion-Control" - }, - "type": "indicator", - "value": 0 - }, - { - "delta": { - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0.55, - 1 - ], - "y": [ - 0.575, - 1 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "EmailConversion-Test" - }, - "type": "indicator", - "value": 0 - }, - { - "delta": { - "reference": 0.25, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0, - 0.45 - ], - "y": [ - 0, - 0.425 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "bar": { - "color": "#EC9B00" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.25 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "WebConversion-Control" - }, - "type": "indicator", - "value": 0.20024672688925146 - }, - { - "delta": { - "reference": 0.25, - "valueformat": ",.2%" - }, - "domain": { - "x": [ - 0.55, - 1 - ], - "y": [ - 0, - 0.425 - ] - }, - "gauge": { - "axis": { - "tickformat": ",.2%" - }, - "bar": { - "color": "#EC9B00" - }, - "threshold": { - "line": { - "color": "red", - "width": 2 - }, - "thickness": 0.75, - "value": 0.25 - } - }, - "mode": "gauge+number+delta", - "number": { - "valueformat": ",.2%" - }, - "title": { - "text": "WebConversion-Test" - }, - "type": "indicator", - "value": 0.19848369369007982 - } - ], - "layout": { - "autosize": true, - "height": 540, - "margin": { - "b": 10, - "l": 10, - "r": 10, - "t": 120 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[CONV] Conversion (Channel/Model Type)" - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "shape: (4, 9)
ChannelExperimentGroupNegativesPositivesCountCTRStdErrNameCName
strstru32i32u32f64f64strstr
"Email""Conversion-Control"248710248710.00.0"EmailConversion-Control""Email_Conversion-Control"
"Email""Conversion-Test"250700250700.00.0"EmailConversion-Test""Email_Conversion-Test"
"Web""Conversion-Control"200975032301610.2002470.002524"WebConversion-Control""Web_Conversion-Control"
"Web""Conversion-Test"199814948298770.1984840.002526"WebConversion-Test""Web_Conversion-Test"
" - ], - "text/plain": [ - "shape: (4, 9)\n", - "┌─────────┬────────────┬───────────┬───────────┬───┬──────────┬──────────┬────────────┬────────────┐\n", - "│ Channel ┆ Experiment ┆ Negatives ┆ Positives ┆ … ┆ CTR ┆ StdErr ┆ Name ┆ CName │\n", - "│ --- ┆ Group ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ --- ┆ u32 ┆ i32 ┆ ┆ f64 ┆ f64 ┆ str ┆ str │\n", - "│ ┆ str ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", - "╞═════════╪════════════╪═══════════╪═══════════╪═══╪══════════╪══════════╪════════════╪════════════╡\n", - "│ Email ┆ Conversion ┆ 24871 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ EmailConve ┆ Email_Conv │\n", - "│ ┆ -Control ┆ ┆ ┆ ┆ ┆ ┆ rsion-Cont ┆ ersion-Con │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ rol ┆ trol │\n", - "│ Email ┆ Conversion ┆ 25070 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ EmailConve ┆ Email_Conv │\n", - "│ ┆ -Test ┆ ┆ ┆ ┆ ┆ ┆ rsion-Test ┆ ersion-Tes │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ t │\n", - "│ Web ┆ Conversion ┆ 20097 ┆ 5032 ┆ … ┆ 0.200247 ┆ 0.002524 ┆ WebConvers ┆ Web_Conver │\n", - "│ ┆ -Control ┆ ┆ ┆ ┆ ┆ ┆ ion-Contro ┆ sion-Contr │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ l ┆ ol │\n", - "│ Web ┆ Conversion ┆ 19981 ┆ 4948 ┆ … ┆ 0.198484 ┆ 0.002526 ┆ WebConvers ┆ Web_Conver │\n", - "│ ┆ -Test ┆ ┆ ┆ ┆ ┆ ┆ ion-Test ┆ sion-Test │\n", - "└─────────┴────────────┴───────────┴───────────┴───┴──────────┴──────────┴────────────┴────────────┘" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "positive_model_response = [\"Clicked\"]\n", - "all_model_response = [\"Impression\", \"Pending\"]\n", - "group_by = [\n", - " \"Day\",\n", - " \"Month\",\n", - " \"Year\",\n", - " \"Quarter\",\n", - " \"Channel\",\n", - " \"Issue\",\n", - " \"Group\",\n", - " \"Name\",\n", - " \"ExperimentGroup\",\n", - "]\n", - "\n", - "ih_analysis = (\n", - " ih.filter((pl.col(\"Outcome\").is_in(all_model_response + positive_model_response)))\n", - " .with_columns(\n", - " [\n", - " pl.when(pl.col(\"Outcome\").is_in(positive_model_response))\n", - " .then(1)\n", - " .otherwise(0)\n", - " .alias(\"Outcome_Binary\")\n", - " ]\n", - " )\n", - " .group_by(group_by)\n", - " .agg([pl.len().alias(\"Count\"), pl.sum(\"Outcome_Binary\").alias(\"Positives\")])\n", - " .with_columns([(pl.col(\"Count\") - (2 * pl.col(\"Positives\"))).alias(\"Negatives\")])\n", - ")\n", - "gauge_group_by = [\"Channel\", \"ExperimentGroup\"]\n", - "reference = {\"Web_Conversion-Test\": 0.25, \"Web_Conversion-Control\": 0.25}\n", - "gauge_data = (\n", - " ih_analysis.group_by(gauge_group_by)\n", - " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\"),\n", - " )\n", - " .with_columns(\n", - " [\n", - " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", - " \"CTR\"\n", - " )\n", - " ]\n", - " )\n", - " .with_columns(\n", - " [\n", - " (\n", - " (\n", - " (\n", - " (pl.col(\"CTR\") * (1 - pl.col(\"CTR\")))\n", - " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " )\n", - " ** 0.5\n", - " )\n", - " ).alias(\"StdErr\")\n", - " ]\n", - " )\n", - " .sort(gauge_group_by, descending=False)\n", - " .collect()\n", - ")\n", - "\n", - "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", - "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", - "\n", - "cols = gauge_data[gauge_group_by[0]].unique().shape[0]\n", - "rows = gauge_data[gauge_group_by[1]].unique().shape[0]\n", - "\n", - "gauge_data = gauge_data.with_columns(\n", - " pl.concat_str(gauge_group_by).alias(\"Name\"),\n", - " pl.concat_str(gauge_group_by, separator=\"_\").alias(\"CName\"),\n", - ")\n", - "\n", - "fig = make_subplots(\n", - " rows=rows,\n", - " cols=cols,\n", - " specs=[[{\"type\": \"indicator\"} for c in range(cols)] for t in range(rows)],\n", - ")\n", - "fig.update_layout(\n", - " height=270 * rows,\n", - " autosize=True,\n", - " title=\"[CONV] Conversion (Channel/Model Type)\",\n", - " margin=dict(b=10, t=120, l=10, r=10),\n", - ")\n", - "index = 0\n", - "for row in gauge_data.iter_rows(named=True):\n", - " ref_value = reference.get(row[\"CName\"], None)\n", - " gauge = {\n", - " \"axis\": {\"tickformat\": \",.2%\"},\n", - " \"threshold\": {\n", - " \"line\": {\"color\": \"red\", \"width\": 2},\n", - " \"thickness\": 0.75,\n", - " \"value\": ref_value,\n", - " },\n", - " }\n", - " if ref_value:\n", - " if row[\"CTR\"] < ref_value:\n", - " gauge = {\n", - " \"axis\": {\"tickformat\": \",.2%\"},\n", - " \"bar\": {\n", - " \"color\": \"#EC5300\" if row[\"CTR\"] < (0.75 * ref_value) else \"#EC9B00\"\n", - " },\n", - " \"threshold\": {\n", - " \"line\": {\"color\": \"red\", \"width\": 2},\n", - " \"thickness\": 0.75,\n", - " \"value\": ref_value,\n", - " },\n", - " }\n", - "\n", - " trace1 = go.Indicator(\n", - " mode=\"gauge+number+delta\",\n", - " number={\"valueformat\": \",.2%\"},\n", - " value=row[\"CTR\"],\n", - " delta={\"reference\": ref_value, \"valueformat\": \",.2%\"},\n", - " title={\"text\": row[\"Name\"]},\n", - " gauge=gauge,\n", - " )\n", - " r, c = divmod(index, cols)\n", - " fig.add_trace(trace1, row=(r + 1), col=(c + 1))\n", - " index = index + 1\n", - "\n", - "fig.show()\n", - "gauge_data" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ] - ], - "error_y": { - "array": { - "bdata": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", - "legendgroup": "Conversion-Control", - "marker": { - "color": "#636efa", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Control", - "offsetgroup": "Conversion-Control", - "orientation": "v", - "showlegend": true, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-11-25", - "2024-11-26", - "2024-11-27", - "2024-11-28", - "2024-11-29", - "2024-11-30", - "2024-12-01", - "2024-12-02", - "2024-12-03", - "2024-12-04", - "2024-12-05", - "2024-12-06", - "2024-12-07", - "2024-12-08", - "2024-12-09", - "2024-12-10", - "2024-12-11", - "2024-12-12", - "2024-12-13", - "2024-12-14", - "2024-12-15", - "2024-12-16" - ], - "xaxis": "x2", - "y": { - "bdata": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "dtype": "f8" - }, - "yaxis": "y2" - }, - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ], - [ - "Conversion-Control" - ] - ], - "error_y": { - "array": { - "bdata": "37Loir80pD++yRLUEBKXP9sydnq4Vpc/OQ05D94clz/CVkyFkQaXP8ytEHkCoZc/o4+H2I6Ulz8sw5nLPQmXPxjunZ/rAZc/jHbdNFSdlz9fRlPM6EWXPyKpPv3CCZc/tdPjG+IXlz98sAG5NIyXPzvqeeNAppc/0GcXlpOKlz/8ftPL9s2XP3CSnFlAmZY/LziJrM52lj9du5ax+WqXP7dsXEOTtZY/Q8FDrvy1mz8=", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", - "legendgroup": "Conversion-Control", - "marker": { - "color": "#636efa", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Control", - "offsetgroup": "Conversion-Control", - "orientation": "v", - "showlegend": false, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-11-25", - "2024-11-26", - "2024-11-27", - "2024-11-28", - "2024-11-29", - "2024-11-30", - "2024-12-01", - "2024-12-02", - "2024-12-03", - "2024-12-04", - "2024-12-05", - "2024-12-06", - "2024-12-07", - "2024-12-08", - "2024-12-09", - "2024-12-10", - "2024-12-11", - "2024-12-12", - "2024-12-13", - "2024-12-14", - "2024-12-15", - "2024-12-16" - ], - "xaxis": "x", - "y": { - "bdata": "ZHhxHjqGxz9v3yr4fnzKP/JZN5gin8k/zZHmSHOkyT8G1DOV647HP8ln3WCKfMo/nWwPKM+ryj/GxPMQyf/IP9jq2SFwY8k/s9LoERVkyz+obtymXOvIPxdRk//SXck/CgoKCgoKyj+GTrWkHtDKP6H3dkrr4sk/zg1APcxPyj8kf15lcUbLP+RCQgWnZsg/Jv/RS/6jxz+avgeavgfKPxrtHra+Dcg/Z6O+s1HfyT8=", - "dtype": "f8" - }, - "yaxis": "y" - }, - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ] - ], - "error_y": { - "array": { - "bdata": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", - "legendgroup": "Conversion-Test", - "marker": { - "color": "#EF553B", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Test", - "offsetgroup": "Conversion-Test", - "orientation": "v", - "showlegend": true, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-11-25", - "2024-11-26", - "2024-11-27", - "2024-11-28", - "2024-11-29", - "2024-11-30", - "2024-12-01", - "2024-12-02", - "2024-12-03", - "2024-12-04", - "2024-12-05", - "2024-12-06", - "2024-12-07", - "2024-12-08", - "2024-12-09", - "2024-12-10", - "2024-12-11", - "2024-12-12", - "2024-12-13", - "2024-12-14", - "2024-12-15", - "2024-12-16" - ], - "xaxis": "x2", - "y": { - "bdata": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "dtype": "f8" - }, - "yaxis": "y2" - }, - { - "alignmentgroup": "True", - "customdata": [ - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ], - [ - "Conversion-Test" - ] - ], - "error_y": { - "array": { - "bdata": "irwrdIbWoj/EKhfRr52XPwMQKZQ5t5c/kRJZUkfqmD+h1EQm+BeXP/yIJxZnBpc/sQYm5X0Ulz9pZa2s0YCXP3erHmDGCpc/cl7Lgz4slz8qQpOxpvCXP17Lyv+ARZc/QIpw3Ughlj+JGmjJH3OWPy8t2DQJH5c/lG1DwNillj/Bc5M84IqXP7pXXyzWXJc/F4rkaI/5lz9fpW22PJeXP6N2ovTe75U/JWf8bH74mz8=", - "dtype": "f8" - } - }, - "hovertemplate": "Day : %{x}
Experiment Group : %{customdata[0]}
CTR : %{y:.2%}", - "legendgroup": "Conversion-Test", - "marker": { - "color": "#EF553B", - "pattern": { - "shape": "" - } - }, - "name": "Conversion-Test", - "offsetgroup": "Conversion-Test", - "orientation": "v", - "showlegend": false, - "textposition": "auto", - "type": "bar", - "x": [ - "2024-11-25", - "2024-11-26", - "2024-11-27", - "2024-11-28", - "2024-11-29", - "2024-11-30", - "2024-12-01", - "2024-12-02", - "2024-12-03", - "2024-12-04", - "2024-12-05", - "2024-12-06", - "2024-12-07", - "2024-12-08", - "2024-12-09", - "2024-12-10", - "2024-12-11", - "2024-12-12", - "2024-12-13", - "2024-12-14", - "2024-12-15", - "2024-12-16" - ], - "xaxis": "x", - "y": { - "bdata": "czVXczVXwz9J1ijFSejJP4cbbrjhhss/aYf6B79+zD9uyCmdvY7JP5YuuE5Rzcg/lwLa10zuyD9pXg+7+pPJPxNFYNGr0sg/lfw3SzCJyT/y51UWZ7TLPwx/n3B7Fso/9axKBAyIxj+PH+gCUmXIPzfAOXW4Gsk/fnGQoOPbxz8Rd8vjBmDKP34rfv461sk/dy547whcyj8Qy/qJN0DKP0a7h61vA8Y/J5poookmyj8=", - "dtype": "f8" - }, - "yaxis": "y" - } - ], - "layout": { - "annotations": [ - { - "font": {}, - "showarrow": false, - "text": "Web", - "textangle": 90, - "x": 0.98, - "xanchor": "left", - "xref": "paper", - "y": 0.2425, - "yanchor": "middle", - "yref": "paper" - }, - { - "font": {}, - "showarrow": false, - "text": "Email", - "textangle": 90, - "x": 0.98, - "xanchor": "left", - "xref": "paper", - "y": 0.7575000000000001, - "yanchor": "middle", - "yref": "paper" - } - ], - "barmode": "group", - "height": 640, - "hovermode": "x unified", - "legend": { - "title": { - "text": "ExperimentGroup" - }, - "tracegroupgap": 0 - }, - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermap": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermap" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "[ENG] Daily Click-through Rate with 95% confidence interval" - }, - "updatemenus": [ - { - "buttons": [ - { - "args": [ - "type", - "bar" - ], - "label": "Bar", - "method": "restyle" - }, - { - "args": [ - "type", - "line" - ], - "label": "Line", - "method": "restyle" - } - ], - "direction": "down", - "showactive": true - } - ], - "xaxis": { - "anchor": "y", - "domain": [ - 0, - 0.98 - ], - "tickfont": { - "size": 10 - }, - "title": { - "text": "Day" - } - }, - "xaxis2": { - "anchor": "y2", - "domain": [ - 0, - 0.98 - ], - "matches": "x", - "showticklabels": false, - "tickfont": { - "size": 10 - } - }, - "yaxis": { - "anchor": "x", - "domain": [ - 0, - 0.485 - ], - "tickformat": ",.2%", - "title": { - "text": "CTR" - } - }, - "yaxis2": { - "anchor": "x2", - "domain": [ - 0.515, - 1 - ], - "matches": "y", - "tickformat": ",.2%", - "title": { - "text": "CTR" - } - } - } - } - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "shape: (88, 8)
DayChannelExperimentGroupNegativesPositivesCountCTRCI
datestrstru32i32u32f64f64
2024-11-25"Email""Conversion-Control"34603460.00.0
2024-11-25"Email""Conversion-Test"35103510.00.0
2024-11-25"Web""Conversion-Control"302684380.1837840.039465
2024-11-25"Web""Conversion-Test"309554190.1510990.036793
2024-11-26"Email""Conversion-Control"1139011390.00.0
……………………
2024-12-15"Web""Conversion-Test"98720513970.171980.021423
2024-12-16"Email""Conversion-Control"81908190.00.0
2024-12-16"Email""Conversion-Test"82808280.00.0
2024-12-16"Web""Conversion-Control"67517110170.2021280.027061
2024-12-16"Web""Conversion-Test"66617110080.2043010.027315
" - ], - "text/plain": [ - "shape: (88, 8)\n", - "┌────────────┬─────────┬────────────────────┬───────────┬───────────┬───────┬──────────┬──────────┐\n", - "│ Day ┆ Channel ┆ ExperimentGroup ┆ Negatives ┆ Positives ┆ Count ┆ CTR ┆ CI │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ date ┆ str ┆ str ┆ u32 ┆ i32 ┆ u32 ┆ f64 ┆ f64 │\n", - "╞════════════╪═════════╪════════════════════╪═══════════╪═══════════╪═══════╪══════════╪══════════╡\n", - "│ 2024-11-25 ┆ Email ┆ Conversion-Control ┆ 346 ┆ 0 ┆ 346 ┆ 0.0 ┆ 0.0 │\n", - "│ 2024-11-25 ┆ Email ┆ Conversion-Test ┆ 351 ┆ 0 ┆ 351 ┆ 0.0 ┆ 0.0 │\n", - "│ 2024-11-25 ┆ Web ┆ Conversion-Control ┆ 302 ┆ 68 ┆ 438 ┆ 0.183784 ┆ 0.039465 │\n", - "│ 2024-11-25 ┆ Web ┆ Conversion-Test ┆ 309 ┆ 55 ┆ 419 ┆ 0.151099 ┆ 0.036793 │\n", - "│ 2024-11-26 ┆ Email ┆ Conversion-Control ┆ 1139 ┆ 0 ┆ 1139 ┆ 0.0 ┆ 0.0 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2024-12-15 ┆ Web ┆ Conversion-Test ┆ 987 ┆ 205 ┆ 1397 ┆ 0.17198 ┆ 0.021423 │\n", - "│ 2024-12-16 ┆ Email ┆ Conversion-Control ┆ 819 ┆ 0 ┆ 819 ┆ 0.0 ┆ 0.0 │\n", - "│ 2024-12-16 ┆ Email ┆ Conversion-Test ┆ 828 ┆ 0 ┆ 828 ┆ 0.0 ┆ 0.0 │\n", - "│ 2024-12-16 ┆ Web ┆ Conversion-Control ┆ 675 ┆ 171 ┆ 1017 ┆ 0.202128 ┆ 0.027061 │\n", - "│ 2024-12-16 ┆ Web ┆ Conversion-Test ┆ 666 ┆ 171 ┆ 1008 ┆ 0.204301 ┆ 0.027315 │\n", - "└────────────┴─────────┴────────────────────┴───────────┴───────────┴───────┴──────────┴──────────┘" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "line_group_by = [\"Day\", \"Channel\", \"ExperimentGroup\"]\n", - "\n", - "line_data = (\n", - " ih_analysis.group_by(line_group_by)\n", - " .agg(\n", - " pl.sum(\"Negatives\").alias(\"Negatives\"),\n", - " pl.sum(\"Positives\").alias(\"Positives\"),\n", - " pl.sum(\"Count\").alias(\"Count\"),\n", - " )\n", - " .with_columns(\n", - " [\n", - " (pl.col(\"Positives\") / (pl.col(\"Positives\") + pl.col(\"Negatives\"))).alias(\n", - " \"CTR\"\n", - " )\n", - " ]\n", - " )\n", - " .with_columns(\n", - " [\n", - " (\n", - " (\n", - " (\n", - " (\n", - " (pl.col(\"CTR\") * (1 - pl.col(\"CTR\")))\n", - " / (pl.col(\"Positives\") + pl.col(\"Negatives\"))\n", - " )\n", - " ** 0.5\n", - " )\n", - " * 1.96\n", - " )\n", - " ).alias(\"CI\")\n", - " ]\n", - " )\n", - " .sort(line_group_by, descending=False)\n", - " .collect()\n", - ")\n", - "\n", - "line_data = line_data\n", - "\n", - "if len(line_data[\"Day\"].unique()) < 30:\n", - " fig = px.bar(\n", - " line_data,\n", - " x=\"Day\",\n", - " y=\"CTR\",\n", - " color=\"ExperimentGroup\",\n", - " error_y=\"CI\",\n", - " facet_row=\"Channel\",\n", - " barmode=\"group\",\n", - " title=\"[ENG] Daily Click-through Rate with 95% confidence interval\",\n", - " custom_data=[\"ExperimentGroup\"],\n", - " )\n", - " fig.update_layout(\n", - " updatemenus=[\n", - " dict(\n", - " buttons=list(\n", - " [\n", - " dict(args=[\"type\", \"bar\"], label=\"Bar\", method=\"restyle\"),\n", - " dict(args=[\"type\", \"line\"], label=\"Line\", method=\"restyle\"),\n", - " ]\n", - " ),\n", - " direction=\"down\",\n", - " showactive=True,\n", - " ),\n", - " ]\n", - " )\n", - "else:\n", - " fig = px.line(\n", - " line_data,\n", - " x=\"Day\",\n", - " y=\"CTR\",\n", - " color=\"ExperimentGroup\",\n", - " title=\"[ENG] Daily Click-through Rate\",\n", - " acet_row=\"Channel\",\n", - " custom_data=[\"ExperimentGroup\"],\n", - " )\n", - "\n", - "fig.update_xaxes(tickfont=dict(size=10))\n", - "fig.update_yaxes(tickformat=\",.2%\")\n", - "yaxis_names = [\"yaxis\"] + [\n", - " axis_name for axis_name in fig.layout._subplotid_props if \"yaxis\" in axis_name\n", - "]\n", - "yaxis_layout_dict = {yaxis_name + \"_tickformat\": \",.2%\" for yaxis_name in yaxis_names}\n", - "fig.update_layout(yaxis_layout_dict)\n", - "height = max(640, 300 * len(line_data[\"Channel\"].unique()))\n", - "fig.update_layout(\n", - " xaxis_title=\"Day\", yaxis_title=\"CTR\", hovermode=\"x unified\", height=height\n", - ")\n", - "fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[1]))\n", - "fig = fig.update_traces(\n", - " hovertemplate=\"Day\"\n", - " + \" : %{x}\"\n", - " + \"
\"\n", - " + \"Experiment Group\"\n", - " + \" : %{customdata[0]}\"\n", - " + \"
\"\n", - " + \"CTR\"\n", - " + \" : %{y:.2%}\"\n", - " + \"\"\n", - ")\n", - "\n", - "fig.show()\n", - "line_data" + "ih.plots.experiment_gauges(\n", + " experiment_field=\"ExperimentGroup\",\n", + " by=\"Channel\",\n", + " reference_values={\"Web\": 0.20, \"Email\": 0.20},\n", + " title = \"Overall Engagement\",\n", + ")" ] } ], diff --git a/python/pdstools/__init__.py b/python/pdstools/__init__.py index 7d1a9cbd..78627fe3 100644 --- a/python/pdstools/__init__.py +++ b/python/pdstools/__init__.py @@ -7,6 +7,7 @@ from polars import enable_string_cache from .adm.ADMDatamart import ADMDatamart +from .ih.IH import IH from .infinity import Infinity from .pega_io import Anonymization, read_ds_export from .prediction.Prediction import Prediction @@ -23,6 +24,7 @@ __all__ = [ "ADMDatamart", + "IH", "Anonymization", "read_ds_export", "Prediction", diff --git a/python/pdstools/ih/Aggregates.py b/python/pdstools/ih/Aggregates.py index d4c9a8dd..00334ffd 100644 --- a/python/pdstools/ih/Aggregates.py +++ b/python/pdstools/ih/Aggregates.py @@ -1,4 +1,6 @@ -from typing import TYPE_CHECKING +from itertools import chain +from typing import TYPE_CHECKING, Dict, List, Optional +import polars as pl from ..utils.namespaces import LazyNamespace @@ -8,4 +10,71 @@ class Aggregates(LazyNamespace): def __init__(self, ih: "IH_Class"): + super().__init__() self.ih = ih + + def summary_by_experiment( + self, + experiment_field: str, + by: Optional[List[str]] = None, + positive_labels: List[str] = None, + negative_labels: List[str] = None, + ): + + if by is not None: + if isinstance(by, str): + by = [by] + else: + by = [] + + if positive_labels is None: + positive_labels = ["Accepted", "Accept", "Clicked", "Click"] + + if negative_labels is None: + negative_labels = [ + "Impression", + "Impressed", + "Pending", + "NoResponse", + ] + + summary = ( + self.ih.data.filter( + pl.col.ExperimentGroup.is_not_null() & (pl.col.ExperimentGroup != "") + ) + .group_by([experiment_field] + by + ["InteractionID"]) + .agg( + # Take only one outcome per interaction. TODO should perhaps be the last one. + InteractionOutcome=pl.when(pl.col.Outcome.is_in(positive_labels).any()) + .then(pl.lit(True)) + .when(pl.col.Outcome.is_in(negative_labels).any()) + .then(pl.lit(False)), + Outcomes=pl.col.Outcome.unique().sort(), # for debugging + ) + .group_by([experiment_field] + by) + .agg( + Positives=pl.col.InteractionOutcome.filter( + pl.col.InteractionOutcome + ).len(), + Negatives=pl.col.InteractionOutcome.filter( + pl.col.InteractionOutcome.not_() + ).len(), + Interactions=pl.len(), + Outcomes=pl.col.Outcomes.list.explode() + .unique() + .sort() + .drop_nulls(), # for debugging + ) + .with_columns(CTR=pl.col.Positives / (pl.col.Positives + pl.col.Negatives)) + .with_columns( + StdErr=( + ( + (pl.col.CTR * (1 - pl.col.CTR)) + / (pl.col.Positives + pl.col.Negatives) + ).sqrt() + ) + ) + .sort([experiment_field] + by) + ) + + return summary diff --git a/python/pdstools/ih/IH.py b/python/pdstools/ih/IH.py index d25b32db..08397335 100644 --- a/python/pdstools/ih/IH.py +++ b/python/pdstools/ih/IH.py @@ -1,14 +1,118 @@ +import datetime +import random +from typing import Optional import polars as pl from .Aggregates import Aggregates from .Plots import Plots +from ..utils.cdh_utils import to_prpc_date_time, _polars_capitalize +from ..pega_io.File import read_ds_export class IH: data: pl.LazyFrame def __init__(self, data: pl.LazyFrame): - self.data = data + self.data = _polars_capitalize(data) self.aggregates = Aggregates(ih=self) self.plots = Plots(ih=self) + + @classmethod + def from_ds_export( + cls, + ih_filename: Optional[str] = None, + ): + """Import from a Pega Dataset Export""" + + data = read_ds_export(ih_filename).with_columns( + # TODO this should come from some polars func in utils + pl.col("pxOutcomeTime").str.strptime(pl.Datetime, "%Y%m%dT%H%M%S%.3f %Z") + ) + return IH(data) + + @classmethod + def from_mock_data(cls, days=90, n=100000): + """Generate sample data""" + accept_rate = 0.2 + accept_avg_duration_minutes = 10 + convert_over_accept_rate_test = 0.5 + convert_over_accept_rate_control = 0.3 + convert_avg_duration_days = 2 + + now = datetime.datetime.now() + + # TODO maybe this should be changed in PDS tools - w/o __TimeStamp__ flag + # def to_prpc_time_str(__TimeStamp__): + # return to_prpc_date_time(__TimeStamp__)[0:15] + + ih_fake_impressions = pl.DataFrame( + { + "pxInteractionID": [str(int(1e9 + i)) for i in range(n)], + "pyChannel": random.choices(["Web", "Email"], k=n), + "pyIssue": "Acquisition", + "pyGroup": "Phones", + "pyName": "AppleIPhone1564GB", + "ExperimentGroup": ["Conversion-Test", "Conversion-Control"] + * int(n / 2), + "pxOutcomeTime": [ + (now - datetime.timedelta(days=i * days / n)) for i in range(n) + ], + "__AcceptDurationMinutes__": [ + random.uniform(0, 2 * accept_avg_duration_minutes) for i in range(n) + ], + "__ConvertDurationDays__": [ + random.uniform(0, 2 * convert_avg_duration_days) for i in range(n) + ], + } + ).with_columns( + pyOutcome=pl.when(pl.col.pyChannel == "Web") + .then(pl.lit("Impression")) + .otherwise(pl.lit("Pending")) + ) + ih_fake_accepts = ih_fake_impressions.sample(fraction=accept_rate).with_columns( + pl.col.pxOutcomeTime + + pl.duration(minutes=pl.col("__AcceptDurationMinutes__")), + pyOutcome=pl.when(pl.col.pyChannel == "Web") + .then(pl.lit("Clicked")) + .otherwise(pl.lit("Accepted")), + ) + ih_fake_converts_test = ( + ih_fake_accepts.filter(pl.col.ExperimentGroup == "Conversion-Test") + .sample(fraction=convert_over_accept_rate_test) + .with_columns( + pl.col.pxOutcomeTime + + pl.duration(days=pl.col("__ConvertDurationDays__")), + pyOutcome=pl.lit("Conversion"), + ) + ) + ih_fake_converts_control = ( + ih_fake_accepts.filter(pl.col.ExperimentGroup == "Conversion-Control") + .sample(fraction=convert_over_accept_rate_control) + .with_columns( + pl.col.pxOutcomeTime + + pl.duration(days=pl.col("__ConvertDurationDays__")), + pyOutcome=pl.lit("Conversion"), + ) + ) + + ih_data = ( + pl.concat( + [ + ih_fake_impressions, + ih_fake_accepts, + ih_fake_converts_test, + ih_fake_converts_control, + ] + ) + .filter(pl.col("pxOutcomeTime") < pl.lit(now)) + .drop( + [ + "__AcceptDurationMinutes__", + "__ConvertDurationDays__", + ] + ) + .sort("pxInteractionID", "pxOutcomeTime") + ) + + return IH(ih_data.lazy()) diff --git a/python/pdstools/ih/Plots.py b/python/pdstools/ih/Plots.py index 63f8924b..eab42c79 100644 --- a/python/pdstools/ih/Plots.py +++ b/python/pdstools/ih/Plots.py @@ -1,4 +1,10 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, List, Optional +import polars as pl +import plotly.io as pio +import plotly as plotly +import plotly.express as px +import plotly.graph_objs as go +from plotly.subplots import make_subplots from ..utils.namespaces import LazyNamespace @@ -8,4 +14,130 @@ class Plots(LazyNamespace): def __init__(self, ih: "IH_Class"): + super().__init__() self.ih = ih + + def experiment_gauges( + self, + experiment_field: str, + by: Optional[str] = "Channel", + positive_labels: Optional[List[str]] = None, + negative_labels: Optional[List[str]] = None, + reference_values: Optional[Dict] = None, + title: Optional[str] = "Experiment Overview", + return_df:Optional[bool] = False, + ): + # TODO currently only supporting a single by + + plot_data = self.ih.aggregates.summary_by_experiment( + experiment_field=experiment_field, + by=by, + positive_labels=positive_labels, + negative_labels=negative_labels, + ) + + if return_df: + return plot_data + + plot_data = plot_data.collect() + + cols = plot_data[by].unique().shape[0] + rows = plot_data[experiment_field].unique().shape[0] + + fig = make_subplots( + rows=rows, + cols=cols, + specs=[[{"type": "indicator"} for c in range(cols)] for t in range(rows)], + ) + fig.update_layout( + height=270 * rows, + autosize=True, + title=title, + margin=dict(b=10, t=120, l=10, r=10), + ) + index = 0 + for row in plot_data.iter_rows(named=True): + ref_value = ( + reference_values.get(row[by], None) if reference_values else None + ) + gauge = { + "axis": {"tickformat": ",.2%"}, + "threshold": { + "line": {"color": "red", "width": 2}, + "thickness": 0.75, + "value": ref_value, + }, + } + if ref_value: + if row["CTR"] < ref_value: + gauge = { + "axis": {"tickformat": ",.2%"}, + "bar": { + "color": ( + "#EC5300" + if row["CTR"] < (0.75 * ref_value) + else "#EC9B00" + ) + }, + "threshold": { + "line": {"color": "red", "width": 2}, + "thickness": 0.75, + "value": ref_value, + }, + } + + trace1 = go.Indicator( + mode="gauge+number+delta", + number={"valueformat": ",.2%"}, + value=row["CTR"], + delta={"reference": ref_value, "valueformat": ",.2%"}, + title={"text": f"{row[by]}: {row[experiment_field]}"}, + gauge=gauge, + ) + r, c = divmod(index, cols) + fig.add_trace(trace1, row=(r + 1), col=(c + 1)) + index = index + 1 + + return fig + + def tree_map( + self, + experiment_field: str, + by: Optional[List[str]] = None, + positive_labels: List[str] = None, + negative_labels: List[str] = None, + title: Optional[str] = "Detailed Click Through Rates", + return_df:Optional[bool] = False, + ): + if by is None: + by = [f for f in ["Channel", "Issue", "Group", "Name"] if f in self.ih.data.collect_schema().names()] + + plot_data = self.ih.aggregates.summary_by_experiment( + experiment_field=experiment_field, + by=by, + positive_labels=positive_labels, + negative_labels=negative_labels, + ) + + if return_df: + return plot_data + + plot_data = plot_data.collect() + + fig = px.treemap( + plot_data.with_columns( + CTR_DisplayValue=pl.col("CTR").round(3), + ), + path=[px.Constant("ALL")] + [experiment_field] + by, + values="CTR_DisplayValue", + color="CTR_DisplayValue", + color_continuous_scale=px.colors.sequential.RdBu, + title=title, + hover_data=["StdErr", "Positives", "Negatives"], + height=640, + ) + fig.update_coloraxes(showscale=False) + fig.update_traces(textinfo="label+value") + fig.update_layout(margin=dict(t=50, l=25, r=25, b=25)) + + return fig diff --git a/python/pdstools/ih/__init__.py b/python/pdstools/ih/__init__.py index e69de29b..9d3c5be9 100644 --- a/python/pdstools/ih/__init__.py +++ b/python/pdstools/ih/__init__.py @@ -0,0 +1,3 @@ +from .IH import IH + +__all__ = ["IH"] \ No newline at end of file diff --git a/python/pdstools/reports/HealthCheck.qmd b/python/pdstools/reports/HealthCheck.qmd index 359d5b27..d817674a 100644 --- a/python/pdstools/reports/HealthCheck.qmd +++ b/python/pdstools/reports/HealthCheck.qmd @@ -1032,7 +1032,6 @@ if datamart.predictor_data is None: ) ``` - ## Number of Predictors per Predictor Category The Predictor Categories identify the source of the predictors. By default we split by the first dot, so this distinguishes between between e.g. *Customer*, *Account*, *IH* and parameterized (*Param.*) predictors. @@ -1041,7 +1040,6 @@ You can override this behavior when the data is read. The numbers here can differ from the totals above, these ones are leading. - ::: {.callout-tip title="Guidance"} - Total number of predictors per model 200 - 700 to stay within service limits - There should be some “IH” predictors but no more than ca 100 of them From bca5bdebdd013474259d8cc113547a424e154f73 Mon Sep 17 00:00:00 2001 From: Stijn Kas <78410144+StijnKas@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:59:01 +0100 Subject: [PATCH 15/21] (patch): removed Plotly prerelease, added timestamp, increased DX client timeout (#301) * stick to GA version of plotly for now * Add another timestamp format * Increase timeout for infinity client * bump version for patch release --- pyproject.toml | 2 +- python/pdstools/__init__.py | 2 +- python/pdstools/infinity/internal/_base_client.py | 4 ++-- python/pdstools/utils/cdh_utils.py | 5 ++++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 00f7d9d3..88a8519c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = ['polars==1.16', 'typing_extensions'] version = {attr="pdstools.__version__"} [project.optional-dependencies] -adm = ['plotly[express]>=6.0.0rc0', 'requests'] +adm = ['plotly', 'requests'] pega_io = ['aioboto3', 'polars_hash'] api = ['httpx', 'pydantic', 'anyio'] healthcheck = ['pdstools[adm]', 'great_tables>=0.13', 'quarto', 'papermill', 'xlsxwriter>=3.0', 'pydot'] diff --git a/python/pdstools/__init__.py b/python/pdstools/__init__.py index 7d1a9cbd..dadd39e1 100644 --- a/python/pdstools/__init__.py +++ b/python/pdstools/__init__.py @@ -1,6 +1,6 @@ """Pega Data Scientist Tools Python library""" -__version__ = "4.0.0" +__version__ = "4.0.1" from pathlib import Path diff --git a/python/pdstools/infinity/internal/_base_client.py b/python/pdstools/infinity/internal/_base_client.py index 47e3aa25..bb5c066e 100644 --- a/python/pdstools/infinity/internal/_base_client.py +++ b/python/pdstools/infinity/internal/_base_client.py @@ -71,7 +71,7 @@ def __init__( auth: Union[httpx.Auth, PegaOAuth], verify: bool = False, pega_version: Union[str, None] = None, - timeout: float = 20, + timeout: float = 90, ): self._base_url = self._enforce_trailing_slash(httpx.URL(base_url)) self.auth = auth @@ -178,7 +178,7 @@ def __init__( auth: Union[httpx.Auth, PegaOAuth], verify: bool = False, pega_version: Union[str, None] = None, - timeout: float = 20, + timeout: float = 90, ): super().__init__( base_url=base_url, auth=auth, verify=verify, pega_version=pega_version diff --git a/python/pdstools/utils/cdh_utils.py b/python/pdstools/utils/cdh_utils.py index 86730a84..2e5a1088 100644 --- a/python/pdstools/utils/cdh_utils.py +++ b/python/pdstools/utils/cdh_utils.py @@ -258,6 +258,9 @@ def parse_pega_date_time_formats( pl.col(timestamp_col).str.to_datetime( "%d-%b-%y", strict=False, ambiguous="null" ), + pl.col(timestamp_col).str.to_datetime( + "%d%b%Y:%H:%M:%S", strict=False, ambiguous="null" + ), pl.col(timestamp_col).str.to_datetime( timestamp_fmt or "%Y", strict=False, ambiguous="null" ), @@ -562,7 +565,7 @@ def _capitalize(fields: Union[str, Iterable[str]]) -> List[str]: "Offline", "Update", "Strategy", - "ModelTechnique" + "ModelTechnique", ] if not isinstance(fields, list): fields = [fields] From e5c150cf95386ec1ede9adb555401cc3b974850a Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Thu, 19 Dec 2024 22:58:23 +0100 Subject: [PATCH 16/21] V1 of IH analysis on conversion models --- ...rting.ipynb => Conversion_Reporting.ipynb} | 41 +++--- examples/ih/ih_helper.py | 87 ------------- python/pdstools/ih/Aggregates.py | 74 ++++++++--- python/pdstools/ih/Plots.py | 120 ++++++++++++++++-- python/pdstools/utils/cdh_utils.py | 24 +++- 5 files changed, 214 insertions(+), 132 deletions(-) rename examples/ih/{Conversion_Modeling_Reporting.ipynb => Conversion_Reporting.ipynb} (81%) delete mode 100644 examples/ih/ih_helper.py diff --git a/examples/ih/Conversion_Modeling_Reporting.ipynb b/examples/ih/Conversion_Reporting.ipynb similarity index 81% rename from examples/ih/Conversion_Modeling_Reporting.ipynb rename to examples/ih/Conversion_Reporting.ipynb index f8c86aec..fa176603 100644 --- a/examples/ih/Conversion_Modeling_Reporting.ipynb +++ b/examples/ih/Conversion_Reporting.ipynb @@ -6,16 +6,10 @@ "metadata": {}, "outputs": [], "source": [ - "import polars as pl\n", - "from pdstools import read_ds_export, IH\n", - "from pdstools.utils import cdh_utils\n", - "from ih_helper import interaction_history\n", + "from pdstools import IH\n", "\n", "import plotly.io as pio\n", "import plotly as plotly\n", - "import plotly.express as px\n", - "import plotly.graph_objs as go\n", - "from plotly.subplots import make_subplots\n", "\n", "plotly.offline.init_notebook_mode()\n", "pio.renderers.default = \"vscode\"" @@ -25,7 +19,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Conversion Results" + "# Conversion Results\n", + "\n", + "Visualization of conversion modeling results from IH data." ] }, { @@ -37,7 +33,7 @@ "from pathlib import Path\n", "\n", "ih_export_file = Path(\n", - " \"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\"\n", + " \"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip \"\n", ")\n", "\n", "if not ih_export_file.exists():\n", @@ -103,17 +99,17 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# ih.plots.trend_line(\n", - "# experiment_field=\"ExperimentGroup\", # should be optional, can also give query to select only the Test group\n", - "# granularity=\"1d\", # string language polars\n", - "# positive_labels=[\"Conversion\"],\n", - "# negative_labels=[\"Impression\", \"Pending\"],\n", - "# title=\"Conversion Rate trends\",\n", - "# )" + "ih.plots.trend_bar(\n", + " experiment_field=\"ExperimentGroup\",\n", + " every=\"1w\",\n", + " positive_labels=[\"Conversion\"],\n", + " negative_labels=[\"Impression\", \"Pending\"],\n", + " title=\"Conversion Rates over Time\",\n", + ")" ] }, { @@ -136,6 +132,17 @@ " title = \"Overall Engagement\",\n", ")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ih.plots.trend_line(\n", + " title=\"Engagement Rates over Time\",\n", + ")" + ] } ], "metadata": { diff --git a/examples/ih/ih_helper.py b/examples/ih/ih_helper.py deleted file mode 100644 index 35a39661..00000000 --- a/examples/ih/ih_helper.py +++ /dev/null @@ -1,87 +0,0 @@ -import datetime -import random -import polars as pl -from pdstools.utils import cdh_utils - -# Some day will move into a proper IH class - -# ih.plots.gauge(conversion/engagement) etc -# constructor define objective (conversion and engagement) labels (positives/negatives) - -class interaction_history: - interactions_period_days = 21 - accept_rate = 0.2 - accept_avg_duration_minutes = 10 - convert_over_accept_rate_test = 0.5 - convert_over_accept_rate_control = 0.3 - convert_avg_duration_days = 2 - - def __init__(self, outcome_definitions): - pass - - def generate(self, n): - now = datetime.datetime.now() - - - # def _interpolate(min, max, i, n): - # return min + (max - min) * i / (n - 1) - - - def to_prpc_time_str(timestamp): - return cdh_utils.to_prpc_date_time(timestamp)[0:15] - - - ih_fake_impressions = pl.DataFrame( - { - "InteractionID": [str(int(1e9 + i)) for i in range(n)], - "pyChannel": random.choices(["Web", "Email"], k=n), - "pyIssue": "Acquisition", - "pyGroup": "Phones", - "pyName": "AppleIPhone1564GB", - "ExperimentGroup": ["Conversion-Test", "Conversion-Control"] * int(n / 2), - "TimeStamp": [ - (now - datetime.timedelta(days=i * self.interactions_period_days / n)) - for i in range(n) - ], - "AcceptDurationMinutes": [ - random.uniform(0, 2 * self.accept_avg_duration_minutes) for i in range(n) - ], - "ConvertDurationDays": [ - random.uniform(0, 2 * self.convert_avg_duration_days) for i in range(n) - ], - } - ).with_columns( - pyOutcome=pl.when(pl.col.pyChannel == "Web") - .then(pl.lit("Impression")) - .otherwise(pl.lit("Pending")) - ) - ih_fake_accepts = ih_fake_impressions.sample(fraction=self.accept_rate).with_columns( - pl.col.TimeStamp + pl.duration(minutes=pl.col("AcceptDurationMinutes")), - pyOutcome=pl.when(pl.col.pyChannel == "Web") - .then(pl.lit("Clicked")) - .otherwise(pl.lit("Accepted")), - ) - ih_fake_converts_test = ih_fake_accepts.filter(pl.col.ExperimentGroup=="Conversion-Test").sample( - fraction=self.convert_over_accept_rate_test - ).with_columns( - pl.col.TimeStamp + pl.duration(days=pl.col("ConvertDurationDays")), - pyOutcome=pl.lit("Conversion"), - ) - ih_fake_converts_control = ih_fake_accepts.filter(pl.col.ExperimentGroup=="Conversion-Control").sample( - fraction=self.convert_over_accept_rate_control - ).with_columns( - pl.col.TimeStamp + pl.duration(days=pl.col("ConvertDurationDays")), - pyOutcome=pl.lit("Conversion"), - ) - - ih_data=pl.concat([ih_fake_impressions, ih_fake_accepts, ih_fake_converts_test, ih_fake_converts_control]).with_columns( - pxOutcomeTime=pl.col("TimeStamp").map_elements( - to_prpc_time_str, return_dtype=pl.String - ), - ).filter(pl.col("TimeStamp") < pl.lit(now)).drop( - ["AcceptDurationMinutes", "ConvertDurationDays", "TimeStamp"] - ).sort( - "InteractionID", "pxOutcomeTime" - ).lazy() - - return ih_data \ No newline at end of file diff --git a/python/pdstools/ih/Aggregates.py b/python/pdstools/ih/Aggregates.py index 00334ffd..8264d451 100644 --- a/python/pdstools/ih/Aggregates.py +++ b/python/pdstools/ih/Aggregates.py @@ -1,8 +1,8 @@ -from itertools import chain -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, List, Optional import polars as pl from ..utils.namespaces import LazyNamespace +from ..utils.cdh_utils import safe_flatten_list if TYPE_CHECKING: from .IH import IH as IH_Class @@ -15,17 +15,42 @@ def __init__(self, ih: "IH_Class"): def summary_by_experiment( self, - experiment_field: str, + experiment_field: Optional[str] = None, + every: Optional[str] = None, by: Optional[List[str]] = None, - positive_labels: List[str] = None, - negative_labels: List[str] = None, - ): + positive_labels: Optional[List[str]] = None, + negative_labels: Optional[List[str]] = None, + ) -> pl.LazyFrame: + """Groups the IH data summarizing into success rate (CTR) and standard error (StdErr). - if by is not None: - if isinstance(by, str): - by = [by] - else: - by = [] + It groups by the "experiment field" (TODO in the future this can be optional or multiple). When + given, the 'every' argument is used to divide the timerange into buckets. It uses the same string + language as Polars. + + Every interaction is considered to have only one outcome: positive, negative or none. When any + outcome in the interaction is in the positive labels, the outcome is considered positive. Next, + when any is in the negative labels, the outcome of the interaction is considered negative. Otherwise + there is no defined outcome and the interaction is ignored in calculations of success rate or error. + + Parameters + ---------- + experiment_field : Optional[str], optional + Optional field that contains the experiments + every : Optional[str], optional + Every interval start and period length, by default None + by : Optional[List[str]], optional + Extra grouping keys, by default None + positive_labels : Optional[List[str]], optional + Outcome label(s) for the positive responses, by default None + negative_labels : Optional[List[str]], optional + Outcome label(s) for the negative responses, by default None + + Returns + ------- + pl.LazyFrame + A polars frame with the grouping keys and columns for the total number of Positives, Negatives, + number of Interactions, success rate (CTR) and standard error (StdErr). + """ if positive_labels is None: positive_labels = ["Accepted", "Accept", "Clicked", "Click"] @@ -38,11 +63,26 @@ def summary_by_experiment( "NoResponse", ] + if every is not None: + source = self.ih.data.with_columns(pl.col.OutcomeTime.dt.truncate(every)) + else: + source = self.ih.data + + group_by_clause = safe_flatten_list( + [experiment_field] + [by] + (["OutcomeTime"] if every is not None else []) + ) + if len(group_by_clause) == 0: + group_by_clause = None + summary = ( - self.ih.data.filter( + source.filter( pl.col.ExperimentGroup.is_not_null() & (pl.col.ExperimentGroup != "") ) - .group_by([experiment_field] + by + ["InteractionID"]) + .group_by( + (group_by_clause + ["InteractionID"]) + if group_by_clause is not None + else ["InteractionID"] + ) .agg( # Take only one outcome per interaction. TODO should perhaps be the last one. InteractionOutcome=pl.when(pl.col.Outcome.is_in(positive_labels).any()) @@ -51,7 +91,7 @@ def summary_by_experiment( .then(pl.lit(False)), Outcomes=pl.col.Outcome.unique().sort(), # for debugging ) - .group_by([experiment_field] + by) + .group_by(group_by_clause) .agg( Positives=pl.col.InteractionOutcome.filter( pl.col.InteractionOutcome @@ -74,7 +114,11 @@ def summary_by_experiment( ).sqrt() ) ) - .sort([experiment_field] + by) ) + if group_by_clause is None: + summary = summary.drop("literal") # created by empty group_by + else: + summary = summary.sort(group_by_clause) + return summary diff --git a/python/pdstools/ih/Plots.py b/python/pdstools/ih/Plots.py index eab42c79..036ce438 100644 --- a/python/pdstools/ih/Plots.py +++ b/python/pdstools/ih/Plots.py @@ -1,6 +1,5 @@ from typing import TYPE_CHECKING, Dict, List, Optional import polars as pl -import plotly.io as pio import plotly as plotly import plotly.express as px import plotly.graph_objs as go @@ -23,9 +22,9 @@ def experiment_gauges( by: Optional[str] = "Channel", positive_labels: Optional[List[str]] = None, negative_labels: Optional[List[str]] = None, - reference_values: Optional[Dict] = None, + reference_values: Optional[Dict[str, float]] = None, title: Optional[str] = "Experiment Overview", - return_df:Optional[bool] = False, + return_df: Optional[bool] = False, ): # TODO currently only supporting a single by @@ -38,7 +37,7 @@ def experiment_gauges( if return_df: return plot_data - + plot_data = plot_data.collect() cols = plot_data[by].unique().shape[0] @@ -104,13 +103,17 @@ def tree_map( self, experiment_field: str, by: Optional[List[str]] = None, - positive_labels: List[str] = None, - negative_labels: List[str] = None, + positive_labels: Optional[List[str]] = None, + negative_labels: Optional[List[str]] = None, title: Optional[str] = "Detailed Click Through Rates", - return_df:Optional[bool] = False, + return_df: Optional[bool] = False, ): if by is None: - by = [f for f in ["Channel", "Issue", "Group", "Name"] if f in self.ih.data.collect_schema().names()] + by = [ + f + for f in ["Channel", "Issue", "Group", "Name"] + if f in self.ih.data.collect_schema().names() + ] plot_data = self.ih.aggregates.summary_by_experiment( experiment_field=experiment_field, @@ -121,7 +124,7 @@ def tree_map( if return_df: return plot_data - + plot_data = plot_data.collect() fig = px.treemap( @@ -135,9 +138,108 @@ def tree_map( title=title, hover_data=["StdErr", "Positives", "Negatives"], height=640, + template="pega", ) fig.update_coloraxes(showscale=False) fig.update_traces(textinfo="label+value") fig.update_layout(margin=dict(t=50, l=25, r=25, b=25)) return fig + + def trend_bar( + self, + experiment_field: str, + every: str = "1d", + by: Optional[str] = None, + positive_labels: Optional[List[str]] = None, + negative_labels: Optional[List[str]] = None, + title: Optional[str] = "Click Through Trend", + return_df: Optional[bool] = False, + ): + + plot_data = self.ih.aggregates.summary_by_experiment( + experiment_field=experiment_field, + every=every, + by=by, + positive_labels=positive_labels, + negative_labels=negative_labels, + ) + if return_df: + return plot_data + + fig = px.bar( + plot_data.collect(), + x="OutcomeTime", + y="CTR", + color=experiment_field, + error_y="StdErr", + facet_row=by, + barmode="group", + custom_data=[experiment_field], + template="pega", + title=title, + ) + fig.update_yaxes(tickformat=",.3%").update_layout(xaxis_title=None) + return fig + + def trend_line( + self, + experiment_field: Optional[str] = None, + every: Optional[str] = "1d", + by: Optional[str] = None, + positive_labels: Optional[List[str]] = None, + negative_labels: Optional[List[str]] = None, + title: Optional[str] = "Click Through Trend", + return_df: Optional[bool] = False, + ): + plot_data = self.ih.aggregates.summary_by_experiment( + experiment_field=experiment_field, + every=every, + by=by, + positive_labels=positive_labels, + negative_labels=negative_labels, + ) + if return_df: + return plot_data + + fig = px.line( + plot_data.collect(), + x="OutcomeTime", + y="CTR", + color=experiment_field, + facet_row=by, + custom_data=[experiment_field] if experiment_field is not None else None, + template="pega", + title=title, + ) + + add_confidence_interval = (experiment_field is None) # doesn't work for multiple lines + if add_confidence_interval: + conf_data = ( + plot_data.select( + x=pl.col("OutcomeTime"), + y_upper=pl.col("CTR") + pl.col("StdErr"), + y_lower=pl.col("CTR") - pl.col("StdErr"), + ) + .collect() + .to_dict(as_series=False) + ) + + # Add continuous interval, see https://plotly.com/python/continuous-error-bars/ + x = conf_data["x"] + y_upper = conf_data["y_upper"] + y_lower = conf_data["y_lower"] + fig.add_trace( + go.Scatter( + x=x + x[::-1], # x, then x reversed + y=y_upper + y_lower[::-1], # upper, then lower reversed + fill="toself", + fillcolor="rgba(0,100,80,0.2)", + line=dict(color="rgba(255,255,255,0)"), + hoverinfo="skip", + showlegend=False, + ) + ) + + fig.update_yaxes(tickformat=",.3%").update_layout(xaxis_title=None) + return fig diff --git a/python/pdstools/utils/cdh_utils.py b/python/pdstools/utils/cdh_utils.py index c1cfa2da..7994b316 100644 --- a/python/pdstools/utils/cdh_utils.py +++ b/python/pdstools/utils/cdh_utils.py @@ -1,6 +1,8 @@ import datetime +from functools import partial import io import logging +from operator import is_not import re import tempfile import warnings @@ -476,7 +478,9 @@ def auc_to_gini(auc: float) -> float: return 2 * safe_range_auc(auc) - 1 -def _capitalize(fields: Union[str, Iterable[str]], extra:Optional[List[str]]=[]) -> List[str]: +def _capitalize( + fields: Union[str, Iterable[str]], extra_endwords: Optional[Iterable[str]] = None +) -> List[str]: """Applies automatic capitalization, aligned with the R couterpart. Parameters @@ -566,7 +570,7 @@ def _capitalize(fields: Union[str, Iterable[str]], extra:Optional[List[str]]=[]) "Strategy", "ModelTechnique", ] - + if not isinstance(fields, list): fields = [fields] fields = [re.sub("^p(x|y|z)", "", field.lower()) for field in fields] @@ -579,9 +583,9 @@ def _capitalize(fields: Union[str, Iterable[str]], extra:Optional[List[str]]=[]) return fields -def _polars_capitalize(df: F, extra:Optional[List[str]]=[]) -> F: +def _polars_capitalize(df: F, extra_endwords: Optional[Iterable[str]] = None) -> F: cols = df.collect_schema().names() - renamed_cols = _capitalize(cols, extra) + renamed_cols = _capitalize(cols, extra_endwords) def deduplicate(columns: List[str]): seen: Dict[str, int] = {} @@ -1151,3 +1155,15 @@ def create_working_and_temp_dir( else tempfile.mkdtemp(prefix="tmp_", dir=working_dir) ) return working_dir, Path(temp_dir_name) + + +# Safe flattening of nested lists, removing None elements, and not splitting strings +def safe_flatten_list(alist: List) -> List: + alist = list(filter(partial(is_not, None), alist)) + alist = [ + item + for sublist in [[item] if type(item) is not list else item for item in alist] + for item in sublist + ] + alist = list(filter(partial(is_not, None), alist)) + return alist From 13441cdbf76ccfc1f592153c0f3cc694a4de034e Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Fri, 20 Dec 2024 13:47:31 +0100 Subject: [PATCH 17/21] Streamlined signatures --- examples/ih/Conversion_Reporting.ipynb | 49 ++--- python/pdstools/ih/Aggregates.py | 123 +++++++----- python/pdstools/ih/IH.py | 27 ++- python/pdstools/ih/Plots.py | 251 +++++++++++++++++-------- python/pdstools/utils/cdh_utils.py | 4 +- 5 files changed, 288 insertions(+), 166 deletions(-) diff --git a/examples/ih/Conversion_Reporting.ipynb b/examples/ih/Conversion_Reporting.ipynb index fa176603..7cbb0fa2 100644 --- a/examples/ih/Conversion_Reporting.ipynb +++ b/examples/ih/Conversion_Reporting.ipynb @@ -31,22 +31,23 @@ "outputs": [], "source": [ "from pathlib import Path\n", + "import polars as pl\n", "\n", "ih_export_file = Path(\n", - " \"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip \"\n", + " \"./Data-pxStrategyResult_InteractionFiles_20241213T091932_GMT.zip\"\n", ")\n", "\n", "if not ih_export_file.exists():\n", " ih = IH.from_mock_data()\n", "else:\n", - " ih = IH.from_ds_export(ih_export_file)\n", + " ih = IH.from_ds_export(\n", + " ih_export_file,\n", + " query=pl.col.ExperimentGroup.is_not_null() & (pl.col.ExperimentGroup != \"\"),\n", + " )\n", "\n", - "ih.aggregates.summary_by_experiment(\n", - " experiment_field=\"ExperimentGroup\",\n", - " by=[\"Channel\"],\n", - " positive_labels=[\"Conversion\"],\n", - " negative_labels=[\"Impression\", \"Pending\"],\n", - ").collect()" + "ih.aggregates.summary_success_rates(by=[\"ExperimentGroup\", \"Channel\"]).drop(\n", + " \"Outcomes\"\n", + ").collect().to_pandas().style.hide()" ] }, { @@ -55,13 +56,10 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.experiment_gauges(\n", + "ih.plots.conversion_overall_gauges(\n", " experiment_field=\"ExperimentGroup\",\n", " by=\"Channel\",\n", - " positive_labels=[\"Conversion\"],\n", - " negative_labels=[\"Impression\", \"Pending\"],\n", " reference_values={\"Web\": 0.055, \"Email\": 0.09},\n", - " title = \"Overall Conversion\",\n", ")" ] }, @@ -69,7 +67,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Detailed View " + "## Detailed View \n", + "\n", + "Showing conversion rates for all actions." ] }, { @@ -78,21 +78,14 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO maybe this would only be useful for the test group\n", - "# then in the hovers show both test and control values\n", - "ih.plots.tree_map(\n", - " experiment_field=\"ExperimentGroup\", # should be optional, can also give query to select only the Test group\n", - " positive_labels=[\"Conversion\"],\n", - " negative_labels=[\"Impression\", \"Pending\"],\n", - " title=\"Conversion rates for all Actions\",\n", - ")" + "ih.plots.conversion_success_rates_tree_map()\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Trends\n", + "## Conversion Rate Trends\n", "\n", "side-by-side bars and lines (separate methods) with error bars" ] @@ -103,12 +96,9 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.trend_bar(\n", + "ih.plots.conversion_success_rates_trend_bar(\n", " experiment_field=\"ExperimentGroup\",\n", " every=\"1w\",\n", - " positive_labels=[\"Conversion\"],\n", - " negative_labels=[\"Impression\", \"Pending\"],\n", - " title=\"Conversion Rates over Time\",\n", ")" ] }, @@ -125,11 +115,10 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.experiment_gauges(\n", + "ih.plots.egagement_overall_gauges(\n", " experiment_field=\"ExperimentGroup\",\n", " by=\"Channel\",\n", " reference_values={\"Web\": 0.20, \"Email\": 0.20},\n", - " title = \"Overall Engagement\",\n", ")" ] }, @@ -139,8 +128,8 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.trend_line(\n", - " title=\"Engagement Rates over Time\",\n", + "ih.plots.conversion_success_rates_trend_line(\n", + " by=\"Channel\"\n", ")" ] } diff --git a/python/pdstools/ih/Aggregates.py b/python/pdstools/ih/Aggregates.py index 8264d451..b22b968e 100644 --- a/python/pdstools/ih/Aggregates.py +++ b/python/pdstools/ih/Aggregates.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, List, Optional, Union import polars as pl from ..utils.namespaces import LazyNamespace @@ -9,21 +9,19 @@ class Aggregates(LazyNamespace): + def __init__(self, ih: "IH_Class"): super().__init__() self.ih = ih - def summary_by_experiment( + def summary_success_rates( self, - experiment_field: Optional[str] = None, + by: Optional[Union[str, List[str]]] = None, every: Optional[str] = None, - by: Optional[List[str]] = None, - positive_labels: Optional[List[str]] = None, - negative_labels: Optional[List[str]] = None, ) -> pl.LazyFrame: - """Groups the IH data summarizing into success rate (CTR) and standard error (StdErr). + """Groups the IH data summarizing into success rates (SuccessRate) and standard error (StdErr). - It groups by the "experiment field" (TODO in the future this can be optional or multiple). When + It optionally groups by one or more dimensions (e.g. Experiment, Channel, Issue etc). When given, the 'every' argument is used to divide the timerange into buckets. It uses the same string language as Polars. @@ -34,85 +32,108 @@ def summary_by_experiment( Parameters ---------- - experiment_field : Optional[str], optional - Optional field that contains the experiments + by : Optional[Union[str, List[str]]], optional + Grouping keys, by default None every : Optional[str], optional Every interval start and period length, by default None - by : Optional[List[str]], optional - Extra grouping keys, by default None - positive_labels : Optional[List[str]], optional - Outcome label(s) for the positive responses, by default None - negative_labels : Optional[List[str]], optional - Outcome label(s) for the negative responses, by default None Returns ------- pl.LazyFrame A polars frame with the grouping keys and columns for the total number of Positives, Negatives, - number of Interactions, success rate (CTR) and standard error (StdErr). + number of Interactions, success rate (SuccessRate) and standard error (StdErr). """ - if positive_labels is None: - positive_labels = ["Accepted", "Accept", "Clicked", "Click"] - - if negative_labels is None: - negative_labels = [ - "Impression", - "Impressed", - "Pending", - "NoResponse", - ] - if every is not None: source = self.ih.data.with_columns(pl.col.OutcomeTime.dt.truncate(every)) else: source = self.ih.data group_by_clause = safe_flatten_list( - [experiment_field] + [by] + (["OutcomeTime"] if every is not None else []) + [by] + (["OutcomeTime"] if every is not None else []) ) - if len(group_by_clause) == 0: - group_by_clause = None + + # TODO filter out nulls for the by arguments + # source.filter( + # pl.col.ExperimentGroup.is_not_null() & (pl.col.ExperimentGroup != "") + # ) summary = ( - source.filter( - pl.col.ExperimentGroup.is_not_null() & (pl.col.ExperimentGroup != "") - ) - .group_by( + source.group_by( (group_by_clause + ["InteractionID"]) if group_by_clause is not None else ["InteractionID"] ) .agg( # Take only one outcome per interaction. TODO should perhaps be the last one. - InteractionOutcome=pl.when(pl.col.Outcome.is_in(positive_labels).any()) - .then(pl.lit(True)) - .when(pl.col.Outcome.is_in(negative_labels).any()) - .then(pl.lit(False)), + [ + pl.when( + pl.col.Outcome.is_in( + self.ih.positive_outcome_labels[metric] + ).any() + ) + .then(pl.lit(True)) + .when( + pl.col.Outcome.is_in( + self.ih.negative_outcome_labels[metric] + ).any() + ) + .then(pl.lit(False)) + .alias(f"Interaction_Outcome_{metric}") + for metric in self.ih.positive_outcome_labels.keys() + ], Outcomes=pl.col.Outcome.unique().sort(), # for debugging ) .group_by(group_by_clause) .agg( - Positives=pl.col.InteractionOutcome.filter( - pl.col.InteractionOutcome - ).len(), - Negatives=pl.col.InteractionOutcome.filter( - pl.col.InteractionOutcome.not_() - ).len(), + [ + pl.col(f"Interaction_Outcome_{metric}") + .filter(pl.col(f"Interaction_Outcome_{metric}")) + .len() + .alias(f"Positives_{metric}") + for metric in self.ih.positive_outcome_labels.keys() + ] + + [ + pl.col(f"Interaction_Outcome_{metric}") + .filter(pl.col(f"Interaction_Outcome_{metric}").not_()) + .len() + .alias(f"Negatives_{metric}") + for metric in self.ih.positive_outcome_labels.keys() + ], Interactions=pl.len(), Outcomes=pl.col.Outcomes.list.explode() .unique() .sort() .drop_nulls(), # for debugging ) - .with_columns(CTR=pl.col.Positives / (pl.col.Positives + pl.col.Negatives)) .with_columns( - StdErr=( + [ + ( + pl.col(f"Positives_{metric}") + / ( + pl.col(f"Positives_{metric}") + + pl.col(f"Negatives_{metric}") + ) + ).alias(f"SuccessRate_{metric}") + for metric in self.ih.positive_outcome_labels.keys() + ] + ) + .with_columns( + [ ( - (pl.col.CTR * (1 - pl.col.CTR)) - / (pl.col.Positives + pl.col.Negatives) - ).sqrt() - ) + ( + pl.col(f"SuccessRate_{metric}") + * (1 - pl.col(f"SuccessRate_{metric}")) + ) + / ( + pl.col(f"Positives_{metric}") + + pl.col(f"Negatives_{metric}") + ) + ) + .sqrt() + .alias(f"StdErr_{metric}") + for metric in self.ih.positive_outcome_labels.keys() + ] ) ) diff --git a/python/pdstools/ih/IH.py b/python/pdstools/ih/IH.py index 08397335..c97382be 100644 --- a/python/pdstools/ih/IH.py +++ b/python/pdstools/ih/IH.py @@ -1,27 +1,45 @@ import datetime +import os import random -from typing import Optional +from typing import Dict, List, Optional, Union import polars as pl + from .Aggregates import Aggregates from .Plots import Plots -from ..utils.cdh_utils import to_prpc_date_time, _polars_capitalize +from ..utils.cdh_utils import to_prpc_date_time, _polars_capitalize, _apply_query +from ..utils.types import QUERY from ..pega_io.File import read_ds_export class IH: data: pl.LazyFrame + positive_outcome_labels: Dict[str, List[str]] def __init__(self, data: pl.LazyFrame): self.data = _polars_capitalize(data) self.aggregates = Aggregates(ih=self) self.plots = Plots(ih=self) + self.positive_outcome_labels = { + "Engagement": ["Accepted", "Accept", "Clicked", "Click"], + "Conversion": ["Conversion"], + } + self.negative_outcome_labels = { + "Engagement": [ + "Impression", + "Impressed", + "Pending", + "NoResponse", + ], + "Conversion": ["Impression", "Pending"], + } @classmethod def from_ds_export( cls, - ih_filename: Optional[str] = None, + ih_filename: Union[os.PathLike, str], + query: Optional[QUERY] = None, ): """Import from a Pega Dataset Export""" @@ -29,6 +47,9 @@ def from_ds_export( # TODO this should come from some polars func in utils pl.col("pxOutcomeTime").str.strptime(pl.Datetime, "%Y%m%dT%H%M%S%.3f %Z") ) + if query is not None: + data = _apply_query(data, query=query) + return IH(data) @classmethod diff --git a/python/pdstools/ih/Plots.py b/python/pdstools/ih/Plots.py index 036ce438..4951001a 100644 --- a/python/pdstools/ih/Plots.py +++ b/python/pdstools/ih/Plots.py @@ -16,31 +16,28 @@ def __init__(self, ih: "IH_Class"): super().__init__() self.ih = ih - def experiment_gauges( + def overall_gauges( self, + metric: str, experiment_field: str, by: Optional[str] = "Channel", - positive_labels: Optional[List[str]] = None, - negative_labels: Optional[List[str]] = None, reference_values: Optional[Dict[str, float]] = None, - title: Optional[str] = "Experiment Overview", + title: Optional[str] = None, return_df: Optional[bool] = False, ): - # TODO currently only supporting a single by - - plot_data = self.ih.aggregates.summary_by_experiment( - experiment_field=experiment_field, - by=by, - positive_labels=positive_labels, - negative_labels=negative_labels, + plot_data = self.ih.aggregates.summary_success_rates( + by=[experiment_field, by], ) if return_df: return plot_data + if title is None: + title = f"{metric} Overall Rates" + plot_data = plot_data.collect() - cols = plot_data[by].unique().shape[0] + cols = plot_data[by].unique().shape[0] # TODO can be None rows = plot_data[experiment_field].unique().shape[0] fig = make_subplots( @@ -68,13 +65,13 @@ def experiment_gauges( }, } if ref_value: - if row["CTR"] < ref_value: + if row[f"SuccessRate_{metric}"] < ref_value: gauge = { "axis": {"tickformat": ",.2%"}, "bar": { "color": ( "#EC5300" - if row["CTR"] < (0.75 * ref_value) + if row[f"SuccessRate_{metric}"] < (0.75 * ref_value) else "#EC9B00" ) }, @@ -88,7 +85,7 @@ def experiment_gauges( trace1 = go.Indicator( mode="gauge+number+delta", number={"valueformat": ",.2%"}, - value=row["CTR"], + value=row[f"SuccessRate_{metric}"], delta={"reference": ref_value, "valueformat": ",.2%"}, title={"text": f"{row[by]}: {row[experiment_field]}"}, gauge=gauge, @@ -99,44 +96,80 @@ def experiment_gauges( return fig - def tree_map( + def conversion_overall_gauges( self, experiment_field: str, + by: Optional[str] = "Channel", + reference_values: Optional[Dict[str, float]] = None, + title: Optional[str] = None, + return_df: Optional[bool] = False, + ): + return self.overall_gauges( + metric="Conversion", + experiment_field=experiment_field, + by=by, + reference_values=reference_values, + title=title, + return_df=return_df, + ) + + def egagement_overall_gauges( + self, + experiment_field: str, + by: Optional[str] = "Channel", + reference_values: Optional[Dict[str, float]] = None, + title: Optional[str] = None, + return_df: Optional[bool] = False, + ): + return self.overall_gauges( + metric="Engagement", + experiment_field=experiment_field, + by=by, + reference_values=reference_values, + title=title, + return_df=return_df, + ) + + def success_rates_tree_map( + self, + metric: str, by: Optional[List[str]] = None, - positive_labels: Optional[List[str]] = None, - negative_labels: Optional[List[str]] = None, - title: Optional[str] = "Detailed Click Through Rates", + title: Optional[str] = None, return_df: Optional[bool] = False, ): if by is None: by = [ f - for f in ["Channel", "Issue", "Group", "Name"] + for f in ["Direction", "Channel", "Issue", "Group", "Name"] if f in self.ih.data.collect_schema().names() ] - plot_data = self.ih.aggregates.summary_by_experiment( - experiment_field=experiment_field, + plot_data = self.ih.aggregates.summary_success_rates( by=by, - positive_labels=positive_labels, - negative_labels=negative_labels, ) if return_df: return plot_data - plot_data = plot_data.collect() + if title is None: + title = f"{metric} Rates for All Actions" + + plot_data = plot_data.collect().with_columns( + CTR_DisplayValue=pl.col(f"SuccessRate_{metric}").round(3), + ) fig = px.treemap( - plot_data.with_columns( - CTR_DisplayValue=pl.col("CTR").round(3), - ), - path=[px.Constant("ALL")] + [experiment_field] + by, + plot_data, + path=[px.Constant("ALL")] + by, values="CTR_DisplayValue", color="CTR_DisplayValue", color_continuous_scale=px.colors.sequential.RdBu, title=title, - hover_data=["StdErr", "Positives", "Negatives"], + hover_data=[ + f"StdErr_{metric}", + f"Positives_{metric}", + f"Negatives_{metric}", + ], height=640, template="pega", ) @@ -146,33 +179,58 @@ def tree_map( return fig - def trend_bar( + def conversion_success_rates_tree_map( + self, + by: Optional[List[str]] = None, + title: Optional[str] = None, + return_df: Optional[bool] = False, + ): + return self.success_rates_tree_map( + metric="Conversion", + by=by, + title=title, + return_df=return_df, + ) + + def engagement_success_rates_tree_map( self, + by: Optional[List[str]] = None, + title: Optional[str] = None, + return_df: Optional[bool] = False, + ): + return self.success_rates_tree_map( + metric="Engagement", + by=by, + title=title, + return_df=return_df, + ) + + def success_rates_trend_bar( + self, + metric: str, experiment_field: str, every: str = "1d", by: Optional[str] = None, - positive_labels: Optional[List[str]] = None, - negative_labels: Optional[List[str]] = None, - title: Optional[str] = "Click Through Trend", + title: Optional[str] = None, return_df: Optional[bool] = False, ): - plot_data = self.ih.aggregates.summary_by_experiment( - experiment_field=experiment_field, + plot_data = self.ih.aggregates.summary_success_rates( every=every, - by=by, - positive_labels=positive_labels, - negative_labels=negative_labels, + by=[experiment_field] + [by], ) if return_df: return plot_data + if title is None: + title = f"{metric} Rates over Time" + fig = px.bar( plot_data.collect(), x="OutcomeTime", - y="CTR", + y=f"SuccessRate_{metric}", color=experiment_field, - error_y="StdErr", + error_y=f"StdErr_{metric}", facet_row=by, barmode="group", custom_data=[experiment_field], @@ -182,22 +240,51 @@ def trend_bar( fig.update_yaxes(tickformat=",.3%").update_layout(xaxis_title=None) return fig - def trend_line( + def conversion_success_rates_trend_bar( self, - experiment_field: Optional[str] = None, - every: Optional[str] = "1d", + experiment_field: str, + every: str = "1d", by: Optional[str] = None, - positive_labels: Optional[List[str]] = None, - negative_labels: Optional[List[str]] = None, - title: Optional[str] = "Click Through Trend", + title: Optional[str] = None, return_df: Optional[bool] = False, ): - plot_data = self.ih.aggregates.summary_by_experiment( + return self.success_rates_trend_bar( + metric="Conversion", experiment_field=experiment_field, every=every, by=by, - positive_labels=positive_labels, - negative_labels=negative_labels, + title=title, + return_df=return_df, + ) + + def engagement_success_rates_trend_bar( + self, + experiment_field: str, + every: str = "1d", + by: Optional[str] = None, + title: Optional[str] = None, + return_df: Optional[bool] = False, + ): + return self.success_rates_trend_bar( + metric="Engagement", + experiment_field=experiment_field, + every=every, + by=by, + title=title, + return_df=return_df, + ) + + def success_rates_trend_line( + self, + metric: str, + every: Optional[str] = "1d", + by: Optional[str] = None, + title: Optional[str] = None, + return_df: Optional[bool] = False, + ): + plot_data = self.ih.aggregates.summary_success_rates( + every=every, + by=by, ) if return_df: return plot_data @@ -205,41 +292,43 @@ def trend_line( fig = px.line( plot_data.collect(), x="OutcomeTime", - y="CTR", - color=experiment_field, + y=f"SuccessRate_{metric}", + color=by, facet_row=by, - custom_data=[experiment_field] if experiment_field is not None else None, + # custom_data=[experiment_field] if experiment_field is not None else None, template="pega", title=title, ) - add_confidence_interval = (experiment_field is None) # doesn't work for multiple lines - if add_confidence_interval: - conf_data = ( - plot_data.select( - x=pl.col("OutcomeTime"), - y_upper=pl.col("CTR") + pl.col("StdErr"), - y_lower=pl.col("CTR") - pl.col("StdErr"), - ) - .collect() - .to_dict(as_series=False) - ) - - # Add continuous interval, see https://plotly.com/python/continuous-error-bars/ - x = conf_data["x"] - y_upper = conf_data["y_upper"] - y_lower = conf_data["y_lower"] - fig.add_trace( - go.Scatter( - x=x + x[::-1], # x, then x reversed - y=y_upper + y_lower[::-1], # upper, then lower reversed - fill="toself", - fillcolor="rgba(0,100,80,0.2)", - line=dict(color="rgba(255,255,255,0)"), - hoverinfo="skip", - showlegend=False, - ) - ) - fig.update_yaxes(tickformat=",.3%").update_layout(xaxis_title=None) return fig + + def conversion_success_rates_trend_line( + self, + every: Optional[str] = "1d", + by: Optional[str] = None, + title: Optional[str] = None, + return_df: Optional[bool] = False, + ): + return self.success_rates_trend_line( + metric="Conversion", + every=every, + by=by, + title=title, + return_df=return_df, + ) + + def engagement_success_rates_trend_line( + self, + every: Optional[str] = "1d", + by: Optional[str] = None, + title: Optional[str] = None, + return_df: Optional[bool] = False, + ): + return self.success_rates_trend_line( + metric="Engagement", + every=every, + by=by, + title=title, + return_df=return_df, + ) diff --git a/python/pdstools/utils/cdh_utils.py b/python/pdstools/utils/cdh_utils.py index 7994b316..953ffbb2 100644 --- a/python/pdstools/utils/cdh_utils.py +++ b/python/pdstools/utils/cdh_utils.py @@ -1159,6 +1159,8 @@ def create_working_and_temp_dir( # Safe flattening of nested lists, removing None elements, and not splitting strings def safe_flatten_list(alist: List) -> List: + if alist is None: + return None alist = list(filter(partial(is_not, None), alist)) alist = [ item @@ -1166,4 +1168,4 @@ def safe_flatten_list(alist: List) -> List: for item in sublist ] alist = list(filter(partial(is_not, None), alist)) - return alist + return alist if len(alist) > 0 else None From 0db173a7e77b5036b8245dcf0b81daabd97c925d Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Sat, 21 Dec 2024 21:25:29 +0100 Subject: [PATCH 18/21] Refactoring old notebook --- examples/ih/Conversion_Reporting.ipynb | 29 +- examples/ih/Example_IH_Analysis.ipynb | 898 +++++-------------------- python/pdstools/ih/Aggregates.py | 40 +- python/pdstools/ih/Plots.py | 182 ++--- 4 files changed, 265 insertions(+), 884 deletions(-) diff --git a/examples/ih/Conversion_Reporting.ipynb b/examples/ih/Conversion_Reporting.ipynb index 7cbb0fa2..1ef101cd 100644 --- a/examples/ih/Conversion_Reporting.ipynb +++ b/examples/ih/Conversion_Reporting.ipynb @@ -56,8 +56,9 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.conversion_overall_gauges(\n", - " experiment_field=\"ExperimentGroup\",\n", + "ih.plots.overall_gauges(\n", + " metric=\"Conversion\",\n", + " condition=\"ExperimentGroup\",\n", " by=\"Channel\",\n", " reference_values={\"Web\": 0.055, \"Email\": 0.09},\n", ")" @@ -78,7 +79,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.conversion_success_rates_tree_map()\n" + "ih.plots.success_rates_tree_map(metric=\"Conversion\")\n" ] }, { @@ -96,10 +97,20 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.conversion_success_rates_trend_bar(\n", - " experiment_field=\"ExperimentGroup\",\n", + "ih.plots.success_rates_trend_bar(\n", + " metric=\"Conversion\",\n", + " condition=\"ExperimentGroup\",\n", " every=\"1w\",\n", - ")" + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ih.plots.success_rates_trend_line(metric=\"Conversion\", every=\"1d\")" ] }, { @@ -115,8 +126,8 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.egagement_overall_gauges(\n", - " experiment_field=\"ExperimentGroup\",\n", + "ih.plots.overall_gauges(\n", + " condition=\"ExperimentGroup\",\n", " by=\"Channel\",\n", " reference_values={\"Web\": 0.20, \"Email\": 0.20},\n", ")" @@ -128,7 +139,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.conversion_success_rates_trend_line(\n", + "ih.plots.success_rates_trend_line(\n", " by=\"Channel\"\n", ")" ] diff --git a/examples/ih/Example_IH_Analysis.ipynb b/examples/ih/Example_IH_Analysis.ipynb index 2430ac13..c0fc2cf0 100644 --- a/examples/ih/Example_IH_Analysis.ipynb +++ b/examples/ih/Example_IH_Analysis.ipynb @@ -2,56 +2,37 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'cdhtools'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[1], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msys\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcdhtools\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mIHanalysis\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcdhtools\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcdh_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m readDSExport\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyplot\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mplt\u001b[39;00m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'cdhtools'" - ] - } - ], - "source": [ - "import pandas as pd\n", - "import sys\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pdstools import IH\n", "\n", - "from cdhtools.IHanalysis import *\n", - "from cdhtools.cdh_utils import readDSExport\n", + "import plotly.io as pio\n", + "import plotly as plotly\n", "\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline" + "plotly.offline.init_notebook_mode()\n", + "pio.renderers.default = \"vscode\"" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Importing: ../../data/Data-pxStrategyResult_pxInteractionHistory_20210101T010000_GMT.zip\n" - ] - } - ], "source": [ - "df_orig = readDSExport(\"Data-pxStrategyResult_pxInteractionHistory_20210101T010000_GMT.zip\", path=\"../../data\")" + "# IH Example Analysis\n", + "\n", + "This notebook uses sample data shipped with PDStools. Replace with actual IH data." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "df = initial_prep(df_orig, referenceTime='pxOutcomeTime')" + "ih = IH.from_ds_export(\n", + " \"../../data/Data-pxStrategyResult_pxInteractionHistory_20210101T010000_GMT.zip\"\n", + ")" ] }, { @@ -65,490 +46,48 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pySubjectTypepxInteractionIDControlGroupValidityStartpyStagepyJourneyCustomerIDChannelSubGrouppyChannelpyCustomerSubSegmentpyStep...pyResponsepyCategoryControlGroupValidityEndpxDecisionTimepyLabelChannelGrouppyStrategyDateWeekOfYearWeek
0CDHSample-Data-Customer-3586780626931683381Customer-4118SMS...2021-01-27 13:22:05.810000+00:00U+ Personal CardInitializeModelsSmall2021-01-2741
1CDHSample-Data-Customer-3586780626931683381Customer-4118Web...2021-01-27 13:22:05.810000+00:00U+ Personal CardInitializeModelsSmall2021-01-2741
2CDHSample-Data-Customer-3586780626931683381Customer-4118Web...2021-01-27 13:22:05.810000+00:00Visa Gold CardInitializeModelsSmall2021-01-2741
3CDHSample-Data-Customer-3586780626931683381Customer-4118SMS...2021-01-27 13:22:05.810000+00:00MasterCard GoldInitializeModelsSmall2021-01-2741
4CDHSample-Data-Customer-3586780626931683381Customer-4118Web...2021-01-27 13:22:05.810000+00:00AMEXPersonalInitializeModelsSmall2021-01-2741
\n", - "

5 rows × 52 columns

\n", - "
" - ], - "text/plain": [ - " pySubjectType pxInteractionID ControlGroupValidityStart \\\n", - "0 CDHSample-Data-Customer -3586780626931683381 \n", - "1 CDHSample-Data-Customer -3586780626931683381 \n", - "2 CDHSample-Data-Customer -3586780626931683381 \n", - "3 CDHSample-Data-Customer -3586780626931683381 \n", - "4 CDHSample-Data-Customer -3586780626931683381 \n", - "\n", - " pyStage pyJourney CustomerID ChannelSubGroup pyChannel \\\n", - "0 Customer-4118 SMS \n", - "1 Customer-4118 Web \n", - "2 Customer-4118 Web \n", - "3 Customer-4118 SMS \n", - "4 Customer-4118 Web \n", - "\n", - " pyCustomerSubSegment pyStep ... pyResponse pyCategory \\\n", - "0 ... \n", - "1 ... \n", - "2 ... \n", - "3 ... \n", - "4 ... \n", - "\n", - " ControlGroupValidityEnd pxDecisionTime pyLabel \\\n", - "0 2021-01-27 13:22:05.810000+00:00 U+ Personal Card \n", - "1 2021-01-27 13:22:05.810000+00:00 U+ Personal Card \n", - "2 2021-01-27 13:22:05.810000+00:00 Visa Gold Card \n", - "3 2021-01-27 13:22:05.810000+00:00 MasterCard Gold \n", - "4 2021-01-27 13:22:05.810000+00:00 AMEXPersonal \n", - "\n", - " ChannelGroup pyStrategy Date WeekOfYear Week \n", - "0 InitializeModelsSmall 2021-01-27 4 1 \n", - "1 InitializeModelsSmall 2021-01-27 4 1 \n", - "2 InitializeModelsSmall 2021-01-27 4 1 \n", - "3 InitializeModelsSmall 2021-01-27 4 1 \n", - "4 InitializeModelsSmall 2021-01-27 4 1 \n", - "\n", - "[5 rows x 52 columns]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head()" + "outputs": [], + "source": [ + "ih.data.head().collect()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Count
pyIssuepyGrouppyDirectionpyChannelpyNamepyOutcome
Churned5072
Loyal4928
SalesCreditCardsInboundWebAMEXPersonalClicked1487
NoResponse6331
UPlusFinGoldAccepted367
Rejected6468
UPlusFinPersonalAccepted367
Rejected6534
UPlusGoldAccepted1843
Clicked1204
NoResponse7004
Rejected5487
UPlusPersonalAccept2635
Accepted970
Rejected4361
VisaGoldClicked1777
NoResponse5538
OutboundSMSAMEXPersonalClicked1002
NoResponse6775
MasterCardGoldClicked296
NoResponse6438
MasterCardWorldClicked342
NoResponse5846
UPlusFinGoldAccepted297
Clicked265
NoResponse7081
Rejected6645
UPlusFinPersonalAccepted311
Rejected6482
UPlusGoldAccepted1463
Rejected5474
UPlusPersonalAccept5206
Accepted684
Clicked581
NoResponse4984
Rejected4578
\n", - "
" - ], - "text/plain": [ - " Count\n", - "pyIssue pyGroup pyDirection pyChannel pyName pyOutcome \n", - " Churned 5072\n", - " Loyal 4928\n", - "Sales CreditCards Inbound Web AMEXPersonal Clicked 1487\n", - " NoResponse 6331\n", - " UPlusFinGold Accepted 367\n", - " Rejected 6468\n", - " UPlusFinPersonal Accepted 367\n", - " Rejected 6534\n", - " UPlusGold Accepted 1843\n", - " Clicked 1204\n", - " NoResponse 7004\n", - " Rejected 5487\n", - " UPlusPersonal Accept 2635\n", - " Accepted 970\n", - " Rejected 4361\n", - " VisaGold Clicked 1777\n", - " NoResponse 5538\n", - " Outbound SMS AMEXPersonal Clicked 1002\n", - " NoResponse 6775\n", - " MasterCardGold Clicked 296\n", - " NoResponse 6438\n", - " MasterCardWorld Clicked 342\n", - " NoResponse 5846\n", - " UPlusFinGold Accepted 297\n", - " Clicked 265\n", - " NoResponse 7081\n", - " Rejected 6645\n", - " UPlusFinPersonal Accepted 311\n", - " Rejected 6482\n", - " UPlusGold Accepted 1463\n", - " Rejected 5474\n", - " UPlusPersonal Accept 5206\n", - " Accepted 684\n", - " Clicked 581\n", - " NoResponse 4984\n", - " Rejected 4578" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.groupby(['pyIssue', 'pyGroup', 'pyDirection', 'pyChannel', 'pyName', 'pyOutcome']).count()[[\n", - " 'pxInteractionID']].rename(columns={'pxInteractionID':'Count'})" + "outputs": [], + "source": [ + "# df.groupby(['pyIssue', 'pyGroup', 'pyDirection', 'pyChannel', 'pyName', 'pyOutcome']).count()[[\n", + "# 'pxInteractionID']].rename(columns={'pxInteractionID':'Count'})\n", + "\n", + "# TODO tree map\n", + "import plotly.express as px\n", + "\n", + "plot_data = ih.aggregates.summary_outcomes(\n", + " by=[\"Issue\", \"Group\", \"Direction\", \"Channel\", \"Name\"]\n", + ").collect()\n", + "fig = px.treemap(\n", + " plot_data,\n", + " path=[px.Constant(\"ALL\")]\n", + " + [\"Outcome\"]\n", + " + [\"Issue\", \"Group\", \"Direction\", \"Channel\", \"Name\"],\n", + " values=\"Count\",\n", + " color=\"Count\",\n", + " branchvalues=\"total\",\n", + " # color_continuous_scale=px.colors.sequential.RdBu_r,\n", + " # title=title,\n", + " # hover_data=[\n", + " # f\"StdErr_{metric}\",\n", + " # f\"Positives_{metric}\",\n", + " # f\"Negatives_{metric}\",\n", + " # ],\n", + " height=640,\n", + " template=\"pega\",\n", + ")\n", + "fig.update_coloraxes(showscale=False)\n", + "fig.update_traces(textinfo=\"label+value+percent parent\")\n", + "fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))\n", + "fig" ] }, { @@ -562,23 +101,16 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAFwCAYAAACGgdwmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA7VElEQVR4nO3deXxcd33v//dH0miXtVvet3iJEzskIAIhgTiEbNxAIAWSsBQKIeX3aMgN0FK2UgKXXm4J3eByiy9NQynNwtJAWZM2cLM4QOSQxcGJ7Sx25DiWN9nWYmmk+fz+OGcWySNpZOtoNKPX8/GYx8x8zzmajxAnfuurz/kec3cBAAAAmFol+S4AAAAAKEYEbQAAACACBG0AAAAgAgRtAAAAIAIEbQAAACACBG0AAAAgAmX5LiAqLS0tvmzZsnyXAQAAgCK2efPm/e7emm1b0QbtZcuWqaOjI99lAAAAoIiZ2c6xttE6AgAAAESAoA0AAABEgKANAAAARKBoe7QBAAAwfeLxuDo7O3Xs2LF8lxKJyspKLVq0SLFYLOdjCNoAAAA4aZ2dnaqrq9OyZctkZvkuZ0q5uw4cOKDOzk4tX7485+NoHQEAAMBJO3bsmJqbm4suZEuSmam5uXnSs/UEbQAAAEyJYgzZSSfyvRG0AQAAUBS++MUv6vTTT9cZZ5yhM888U7/5zW+0YcMGLVmyRO6e2u8tb3mLamtrJUmJREI33HCD1q1bp/Xr1+uVr3ylnnvuuSmphx5tAAAAFLyHHnpIP/7xj/XII4+ooqJC+/fv1+DgoCSpoaFBDz74oM477zx1d3drz549qePuuOMOvfjii3r88cdVUlKizs5O1dTUTElNBG0AyOQuHe6UDoWzGSVlxz9KY1JJqVQSyxjL3CfcXsR/QgWAmWbPnj1qaWlRRUWFJKmlpSW17eqrr9btt9+u8847Tz/4wQ905ZVX6sknn0wdN3/+fJWUBI0eixYtmrKaCNoAZqdEQjrSKXU9Je3LfDwtDfZMzWdYaRjKy0YG81RQL8sI65n7Zgv1ue5bltsvAdmOz+nzMrYnx6yEXyoA5N3FF1+sz3/+81q9erXe8IY36KqrrtL5558vSbrwwgv1wQ9+UMPDw7r99tu1ceNGfeELX5AkveMd79B5552n+++/XxdeeKHe/e5366yzzpqSmgjaAIpbIiEd3hUE6K6twXMyUMd70/vVtkmta6Qz3xU8N68MAmViSBoeCp4TQ1IiLiWGw/F4xvhQxthwxr5Dwfsx9w23JzK2Dw9JQwNjHJ9l38zPypeswT5bUB/rl4AJfokY65eA0iz7jfVLx3G/cJRJFXVS0/JgfwBT5qb/eFK/f/HIlH7N0xbM0V++6fQxt9fW1mrz5s26//779ctf/lJXXXWVvvSlL0mSSktLdd555+n2229Xf3+/li1bljpu0aJFevrpp3Xvvffq3nvv1YUXXqjvfve7uvDCC0+6ZoI2gOKQSEjdO8MgvTUdrPdvk+J96f1q5wVB+uXvCZ5b1wbP1U35q32quEueyB7qR/wSkC2on+AvERMen/F5xx0f/gIx2JfDvqN+4fDhqfvfraQs+MWq9VRp7tr0c9MKAjhQYEpLS7VhwwZt2LBB69ev17e+9a3Utquvvlpvfetb9bnPfe644yoqKnTZZZfpsssuU1tbm+666y6CNoBZKJGQup8fNUO9Vdq3TRrqT+9XNz8M1O8NnueGgbqqMW+lR84saFcpKc13JdFLJIKwPdFfBib6y0L/oeAvHF1PSXsek37/Q0nhygQlMall1cgA3npqGMD55xMYz3gzz1F5+umnVVJSolWrVkmSHn30US1dulRbtmyRJL32ta/VJz/5SV1zzTUjjnvkkUc0b948LViwQIlEQo8//rjOOOOMKamJ/1IAmJkSw9Kh57PMUG8fFagXSHNPldr/KB2EWtdIVQ35qhzToaREUkkw4xyrmrqvO9gX/BVk31PhL3JPSbs3S0/+IL1PabnUvCr4/13r2vRz0/LZ8UsOMEP19PTowx/+sLq7u1VWVqaVK1dq48aNetvb3iYpWAf7T//0T487rqurSx/84Ac1MDAgSTr77LN1/fXXT0lNlrmmYDFpb2/3jo6OfJcBYCKpQP3UyBnq/duloYw7cM1ZmA7SczMCdWV93krHLDLYm9HfH86A79sqde9K71NaIbWsDv//GbYlzV0rNS4jgGNW2Lp1q9auXZvvMiKV7Xs0s83u3p5tf2a0AUyPxLB08LkwqCRnqJ8KZg+HB9L7zVkUBJXl54+coa6ck7/agfIaaeHLg0emgR5p/9Pp4N31lLTr19IT303vU1YZtqCsHTkL3rAsnJkHUKwI2gCm1vBQsAZ1atYvfOzfPjJQ1y8OQvSK89P9ry2rCdQoLBW10sJXBI9MA0fTM+DJFpSdm6Qn7kzvU1YVBPDMCzBbT5UalhLAgSIxLUHbzG6RdLmkLndfF459QdIVkhKSuiS9z91fzHLssKQnwre73P3N01EzgAkMD0kHnx25BnXXU9KB7dLwYHq/+iXB7N0pF4QrfJwqta4OllUDilVFnbSoPXhkOnYk3R6VnAV/7n7p8TvS+8SqwxaUUQG8fjEBHCgw0zWjfaukr0n6l4yxL7v7X0iSmd0g6bOSPpTl2H53PzPqAgGMYTieDtSjZ6gT8fR+DUuCIL3ywvQKHy1rghk/AIHKOdLiVwaPTP3dQRtVcva7a6v07K+kx25L7xOrCX5JHd2CUr+YGwYBM9S0BG13v8/Mlo0ay1zFvEap9ZQA5MVwXDrwTJYZ6h2jAvXSIEivuii9BnXrmqCHFcCJqWqQFp8dPDL1H8pYyjIM4M/cKz32b+l9ymtHrgmfmgFfRAAH8iyvPdpm9kVJfyjpsKQLxtit0sw6JA1J+pK73zVN5QHFaWhQOvjMqLskJgN18s6CFqyU0HqqtObS9AWJLasJ1MB0qmqUlrw6eGTqO3h8C8r2u6VH/zW9T3ldGLxHLUM4ZwEBHJgmeQ3a7v5pSZ82s09Kul7SX2bZbam77zazFZLuNbMn3P2ZbF/PzK6TdJ0kLVmyJKqygcIwNBiE58w1qPc9HYTszEDdtDwM1G9ML53XvEoqr85r+QDGUd0kLT0neGTqOxie6xnn/bZfSL/LCOAVc9K/PGf2gdfNJ4CjoH3kIx/R0qVLdeONN0qSLrnkEi1evFjf/OY3JUkf+9jHtHDhQn30ox897tgNGzbo5ptvVnt71lX6TthMWXXkO5J+qixB2913h8/PmtmvJJ0lKWvQdveNkjZKwTraURULzChDA0GgHnGXxKeDNpDkbaqtRGoMA/Xay9PL5rWsmtqbfQDIr+omadm5wSNT74Fw9ntrui3s6Z9Kv/t2ep+K+vQa9ZkBvLaNAI6CcO655+rOO+/UjTfeqEQiof379+vIkXSn8qZNm/S3f/u301pT3oK2ma1y9+3h2yskPZVln0ZJfe4+YGYtks6V9NfTWCYwc8SPhTPUT2nEzV0OPjsyUDetCAP1mzNmqFcSqIHZrKZZqjlPWnbeyPGefaP+6vWUtPU/pEe+ld6nsiHjRlEZLSi1cwngmFFe85rX6CMf+Ygk6cknn9S6deu0Z88eHTp0SNXV1dq6davMTOeff756enrU0tKiW2+9VfPnz5ckffvb39a1116roaEh3XLLLTr77LPH+7icTNfyfrdJ2iCpxcw6Fcxcv9HM1ihY3m+nwhVHzKxd0ofc/VpJayV9w8wSkkoU9Gj/fjpqBvImfixYIi9zhY99T4WBOhHsY6VhoF4jnf6W9Ax180opVpnX8gEUkNrW4LH8dekxd6l338gLMPc9JT15l3Ts1vR+VY0ZwTtjJrymlQCOvFiwYIHKysq0a9cubdq0Seecc452796thx56SPX19Vq7dq0+8pGP6Ic//KFaW1t1xx136NOf/rRuueUWSVJfX58effRR3XfffXr/+9+vLVu2nHRN07XqyDVZhv9pjH07JF0bvt4kaX2EpQH5E+8PlsgbcUvnp4KbvWQG6uZTgn+8Tr8y3VPZvFIqq8hv/QCKk1kwW107N7ihVJK71NM18gLMrqekLd+Xjh1O71fVdPwa4HPXSjUt0/+9IH9+9gnppScm3m8y5q2XLvvSuLu85jWv0aZNm7Rp0yZ99KMf1e7du7Vp0ybV19dr4cKFuvvuu3XRRRdJkoaHh1Oz2ZJ0zTVBXH3d616nI0eOqLu7Ww0NDSdV8kzp0QaKV7w/XB931Az1oefTgbqkTGo6RWo7XVr/tvRSXc2nEKgBzAxmUl1b8FixIT3uLh19aWQA3/e09MT3pIGMAF7dkl5jPxXC1wZtLcAUOffcc7Vp0yY98cQTWrdunRYvXqyvfOUrmjNnjjZs2JCa4c7GRv0lZvT7E0HQBqbKYF8QqI+boX5eqWXiS8qC2eh5Z0jr35GeoW46RSorz2f1AHBizKQ584PHKa9Pj7tLR/cc34Ly2B3S4NH0fjWtx89+t54aXNiJwjXBzHNUXvOa1+jmm2/WihUrVFpaqqamJnV3d+vJJ5/UN77xDX31q1/VQw89pHPOOUfxeFzbtm3T6aefLkm64447dMEFF+iBBx5QfX296uvrT7oegjaQi0QiuGlLYihY5aN7Z5YZ6p1KB+pYEKgXnCm97OqMHupTpNJYPr8TAJgeZsGa3XMWBHeMTXKXjuzOmP0OJyYe/TdpsCe9X21b+q97mRdiVjVO//eCgrF+/Xrt379f73znO0eM9fT0aO7cufre976nG264QYcPH9bQ0JBuvPHGVNCurKzUWWedpXg8nurbPlnmXpyr4LW3t3tHR0e+y5gdEsNBAB0Og2jyMWXvh9Mhdzi5/UTeD2d8zYnej6pjrBuXlsSCJfJSFwKFz00rCNQAMBnu0uHOkbPfydWV4r3p/WrnjVoBJXxUNeStdAS2bt2qtWvX5ruMSGX7Hs1ss7tnXYCbGe0ouQc9uDmHu0mGv+PeZwbSyb6fILBmqyP5fqwQGqWSsvARk0pKg1Cb9X34SL4vq5BKasbYXhoen8P7+kUZgZrTCABOmpnUsDh4rLooPZ5ISIdfGNWWtzVYgjDel96vbn6WFpQ1UuXJ//kfOFEkhKl02zXS8w+MDMr5kO8QOuH7zPrKsoyNV3P4nqWjAGB2KCmRGpcGj9WXpMcTCenwrpEroOzbKnX8szTUn95vzsLghl1184JHbVvG63nBxZ0Vc/h3BZEgaE+lFRdIjcumNoRO9J4QCgCYjUpKgn9zG5dJay5NjycSwXU0mS0o3buk3ZuD1VEyQ3hSWVW4osr8dBCvDd/XtYWBfF7QH86/sZgEgvZUetV1+a4AAIDZraREaloePNZcNnKbuzRwRDq6V+p5KQjeR1+SevamX+/dIu34r5EroySVlqdnwbMF8eRYdXNQxyzk7lOyLN5MdCLXNRK0AQDA7GAW9GxX1kutq8ffd7A3I4i/dHw4P7AjaBc91n38sSVlUs3ciWfJa1qL6jqfyspKHThwQM3NzUUXtt1dBw4cUGXl5O6+XDw/XQAAgKlSXhMsydp8yvj7xfvDGfG96UB+dE96lrx7l/TCb6S+A8cfayXBjXzG7B8Px2rbCuJeC4sWLVJnZ6f27duX71IiUVlZqUWLFk3qGII2AADAiYpVpXvFxzM0KPV2ZQTxl44P53sek3r3pe8anKm6OaNtZd7Y4Tw2uRnXqRSLxbR8+fK8ff5MRNAGAACIWll5sDRs/QQzosNDUt/+IHhnBvHMtpV9Twcz5omh44+vrM8exFNtK+Hritpovk+MQNAGAACYKUrL0iF5PIlE0I4yIoiPCue7HgqC+fDg8ceX145c4nCsWXKWPjwpBG0AAIBCU1Ii1bYGj3nrx97PXeo/NHJlldFtK7sfCbZn3gAoKbn0YTKQZ724k6UPx0LQBgAAKFZmUnVT8Jg7zu3R3aWBo2Eg33N8u0rPXmnv76VnfhkskThaaXn6ws3RNwTKDOfVLbNq6UOCNgAAwGxnJlXOCR4tq8bfN7n0YXKWfHQ4P/DM2EsfWmkYusebJZ8XLI9YBEsfFv53AAAAgOmT89KHx4IQPlbbyuEXpM6Hg4s/j2PBOuNj9o/PL4ilDwnaAAAAmHqxSqlxafAYz3Bc6unKCOIvHR/OX3oiWB4x29KHVU1B+H7Vh6RXvDea7+UEEbQBAACQP6UxqX5h8BhPYjhYZ3ystpUZuGQhQRsAAAAzX0lpbksfziCz57JPAAAAYBoRtAEAAIAIELQBAACACBC0AQAAgAgQtAEAAIAIELQBAACACBC0AQAAgAhMW9A2s1vMrMvMtmSMfcHMHjezR83sbjNbMMax7zWz7eFjZt3yBwAAAMhiOme0b5V06aixL7v7Ge5+pqQfS/rs6IPMrEnSX0p6laSzJf2lmTVGWyoAAABwcqYtaLv7fZIOjho7kvG2RpJnOfQSSfe4+0F3PyTpHh0f2AEAAIAZJe+3YDezL0r6Q0mHJV2QZZeFkl7IeN8ZjgEAAAAzVt4vhnT3T7v7YknfkXT9yXwtM7vOzDrMrGPfvn1TUyAAAABwAvIetDN8R9IfZBnfLWlxxvtF4dhx3H2ju7e7e3tra2sEJQIAAAC5yWvQNrNVGW+vkPRUlt1+IeliM2sML4K8OBwDAAAAZqxp69E2s9skbZDUYmadClYSeaOZrZGUkLRT0ofCfdslfcjdr3X3g2b2BUkPh1/q8+5+8LgPAAAAAGYQc8+20Efha29v946OjnyXAQAAgCJmZpvdvT3btpnUow0AAAAUDYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEIGyXHc0sxWSmiQdlPScu3tkVQEAAAAFbtwZbTN7uZl9y8wOSNoh6bfh84Fw/OXTUSQAAABQaMYM2mb2L5L+XdILkq6U1CKpPHx+i6Rdkn4Q7gcAAAAgw3itIw9K+iN3Hx41flDSfZLuM7PPSfpARLUBAAAABWvMoO3u35jo4DCEb5zSigAAAIAikPPFkElmdqGksyTtcPe7prwiAAAAoAhMank/M/uUpM9Kapb0CTNjNhsAAADIYtwZbTN7ubs/kjF0saQN7u5mViVpt6TrJvoQM7tF0uWSutx9XTj2ZUlvkjQo6RkF/eDdWY59XtJRScOShty9PYfvCwAAAMiriWa0/6+Z/ZWZlYfv90l6j5mtkvRBSc/n+Dm3Srp01Ng9kta5+xmStkn65DjHX+DuZxKyAQAAUCgmCtqvkjQg6WEzO0fSDQoC812SLpT0rlw+xN3vU7BaSebY3e4+FL79taRFuZcNAAAAzGzjto6EQfgmM/u+pP+r4IY117p73xTX8X5Jd4xVhqS7zcwlfcPd6QsHAADAjJfTxZDuvkXSuQpuXvNwuPLIlDCzT0sakvSdMXY5z91fLukySX9iZq8b52tdZ2YdZtaxb9++qSoRAAAAmLSJbsF+jpk9ZmY9kh6Q9GNJV0j6jJl908zmnMyHm9n7FFwk+S5392z7uPvu8LlLwZ0qzx7r67n7Rndvd/f21tbWkykNAAAAOCkTzWj/s6TPKVjO7yuS/t7dd7j7BZI2S/rNiX6wmV0q6eOS3jxWK4qZ1ZhZXfK1glVPtpzoZwIAAADTZaKg3SLpP919QNK9klLTxO7+fyRdlMuHmNltkh6StMbMOs3sA5K+JqlO0j1m9qiZ/WO47wIz+2l4aJukB8zsMQX94T9x95/n/u0BAAAA+THRnSH/UdJmM3tYQcvGlzM3untnLh/i7tdkGf6nMfZ9UdIbw9fPSnpZLp8BAAAAzCQTrTrymXDFkZWS/oe7b52esgAAAIDCNtGMttz9d5J+Nw21AAAAAEVjzB5tM7spvABxTGZWa2Y3TX1ZAAAAQGEbb0a7QtJzZvZDBbdL/72kI5LmSDpN0hskvUXSNyOuEQAAACg4YwZtd/+EmX1d0ocULPF3qoK7NJqkpxXchr3d3XdFXyYAAABQWCa6GHKXpE9J+pSZVUpqlHTI3Y9NR3EAAABAoZrwYsikMFzvibAWAAAAoGhMdMMaAAAAACeAoA0AAABEgKANAAAARICgDQAAAEQg56BtZu82s3vM7PHw/evM7MroSgMAAAAKV05B28w+KukmST+TtCQc3ifp4xHVBQAAABS0XGe0/z9Jl7n73yi4aY0kbZO0MpKqAAAAgAKXa9Bucvdt4etk0LaM1wAAAAAy5Bq0f29ml48au1TSY1NcDwAAAFAUcr0z5Kck/cTM7pRUYWZflXS1pNHhGwAAAIBynNF29/slvVpSv6RfhsdtcPffRFgbAAAAULBymtE2s2Xu/ntJHx41vtTdd0ZSGQAAAFDAcu3RfnyM8d9NVSEAAABAMck1aNtxA2YxseoIAAAAkNW4rSNmdo+CMF1hZneP2rxE0iNRFQYAAAAUsol6tB8In8+X9GDGeELSS5K+G0VRAAAAQKEbN2i7+02SZGZb3f3O6SkJAAAAKHw5rTqSDNlmVimpVRk92+6+K5rSAAAAgMKV6/J+KyT9q6RXZdlcOqUVAQAAAEUg11VHvibpBUkvk3RU0hmS7pL0gWjKAgAAAApbrkH7VZKudfctkuTuT0r6Y0l/lsvBZnaLmXWZ2ZaMsS+b2VNm9riZ/buZNYxx7KVm9rSZ7TCzT+RYLwAAAJBXuQbthILbr0tSTxiKDypY4i8Xt0q6dNTYPZLWufsZkrZJ+uTog8ysVNL/lnSZpNMkXWNmp+X4mQAAAEDe5Bq0n5R0bvj6N5L+VtI/SHoul4Pd/T4FwTxz7G53Hwrf/lrSoiyHni1ph7s/6+6Dkm6XdEWONQMAAAB5k2vQvkHpoPxnkhZKalfQPjIV3i/pZ1nGFyroDU/qDMcAAACAGS3X5f0ez3j9rKSLJcnM6k+2ADP7tKQhSd+Zgq91naTrJGnJkly7WgAAAICpl+uM9ghmVmlmH5f07Ml8uJm9T9Llkt7l7p5ll92SFme8XxSOZeXuG9293d3bW1tbT6Y0AAAA4KSMG7TNbJmZ3WtmR8zst2a22swukbRdwdJ+N5zoB5vZpZI+LunN7t43xm4PS1plZsvNrFzS1ZJ+dKKfCQAAAEyXiWa0/yZ8/oSCVUe+K+mbkv5C0lp3z6ndw8xuk/SQpDVm1mlmH1CwNnedpHvM7FEz+8dw3wVm9lNJCi+WvF7SLyRtlXRnuLQgAAAAMKNZ9o6NcKPZS5JOd/cDZjZP0ouS2t39kekq8ES1t7d7R0dHvssAAABAETOzze7enm3bRDPa1e5+QJLc/SVJPYUQsgEAAIB8m2jVETOz+ZIsfD886r3c/cWoigMAAAAK1URBu0bB2tVJlvHeJLmk0gjqAgAAAAraREF7+bRUAQAAABSZcYO2u++crkIAAACAYnJCN6wBAAAAMD6CNgAAABABgjYAAAAQgZyCtpktG2N86ZRWAwAAABSJXGe0Hx9j/HdTVQgAAABQTHIN2nbcgFlMwTraAAAAAEYZd3k/M7tHQZiuMLO7R21eIonbsQMAAABZTHTDmgfC5/MlPZgxnpD0kqTvRlEUAAAAUOgmumHNTZJkZlvd/c7pKQkAAAAofBPNaEuS3P1OM6uVdLmkRZI6Jf3E3Y9GWRwAAABQqHIK2mbWLumnkvol7VLQn/0PZvZGd++IsD4AAACgIOW66sjXJX3F3Ze6+2vdfamkmyX9n+hKAwAAAApXrkF7raSvjBr7G0mnTm05AAAAQHHINWg/KmndqLH14TgAAACAUXLq0ZZ0t6Qfm9k3Je2UtEzS+yVtNLN3Jndy93+b8goBAACAApRr0H6/pLik92aMDYXjSS6JoA0AAAAo9+X9lkddCAAAAFBMcu3RliSZ2QIze3VUxQAAAADFIqegbWZzzew/Fdyo5j/DsavM7OtRFgcAAAAUqlxntP9B0nOSWhX0akvSvZIuiqIoAAAAoNDlejHkBZKWuvsxM3NJcvd9ZjY3utIAAACAwpXrjPaARoVyM2uSdHDKKwIAAACKQK5B+25JXzGzWMbYTZJ+ksvBZnaLmXWZ2ZaMsbeb2ZNmljCz9nGOfd7MnjCzR82sI8d6AQAAgLzKNWh/XMFt2A9JmmNm3ZLOkPSZHI+/VdKlo8a2SLpS0n05HH+Bu5/p7mMGcgAAAGAmyXUd7YOSXmdmr5C0XMHdITvc3XM8/j4zWzZqbKskmdmkCgYAAAAKQU5B28waJQ26+2ZJm8OxGjOLuXt3hPVJwR0n7w4vwvyGu2+M+PMAAACAk5Zr68iPJJ0+amydpB9ObTlZnefuL5d0maQ/MbPXjbWjmV1nZh1m1rFv375pKA0AAADILtegfbqk0RcidkhaP7XlHM/dd4fPXZL+XdLZ4+y70d3b3b29tbU16tIAAACAMeUatI9Jqh41VqP0zWsiEban1CVfS7pYwUWUAAAAwIyWa9B+QNJfmVmJJFlwBePnJT2Yy8FmdpukhyStMbNOM/uAmb3VzDolnSPpJ2b2i3DfBWb20/DQNkkPmNljkn4r6Sfu/vNcvzkAAAAgXyyXhUPMbKmCW65XSnpWwcojg5Je7+7PR1ngiWpvb/eODpbdBgAAQHTMbPNYS1DnurzfTjNbJ+lyScskPa9gdrlvqooEAAAAiklOQVuS3L1f0ncjrAUAAAAoGjn1aJvZ3Wb2+lFjF5rZz6IpCwAAAChsuV4M+XIdf6v0+yW9cmrLAQAAAIpDrkE7ISk2aqxUEvdPBwAAALLINWhvlvThUWPXS3pkassBAAAAikOuF0P+uaRfmdkfSNomaZWkNZI2RFQXAAAAUNBymtF298clnSbpe5KOSPq+pNPc/bEIawMAAAAK1mSW93tJ0peT783sdDP7pLvfEEllAAAAQAHLtUdbkmRmFWb2h2b2oKQnFKxGAgAAAGCUnGa0zew0SX8s6d2SqhUE9Evd/e4IawMAAAAK1rgz2mb2HjO7X9IWSedL+pykhZIOSno06uIAAACAQjXRjPa3JB2Q9N/cPXUXSDOWzwYAAADGM1GP9l9I6pF0l5n9u5m9ycwm1dcNAAAAzEbjhmZ3/6KkFZLeIskVLOu3W1KDpAUR1wYAAAAUrAlnpz3wM3e/UtJSSV+X9JKkh83szqgLBAAAAArRpNpA3H2Pu39BwSz3FZLKI6kKAAAAKHA537Amk7u7pJ+GDwAAAACjcGEjAAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQgWkJ2mZ2i5l1mdmWjLG3m9mTZpYws/Zxjr3UzJ42sx1m9onpqBcAAAA4WdM1o32rpEtHjW2RdKWk+8Y6yMxKJf1vSZdJOk3SNWZ2WkQ1AgAAAFNmWoK2u98n6eCosa3u/vQEh54taYe7P+vug5Jul3RFRGUCAAAAU2am92gvlPRCxvvOcAwAAACY0WZ60J4UM7vOzDrMrGPfvn35LgcAAACz2EwP2rslLc54vygcy8rdN7p7u7u3t7a2Rl4cAAAAMJaZHrQflrTKzJabWbmkqyX9KM81AQAAABOaruX9bpP0kKQ1ZtZpZh8ws7eaWaekcyT9xMx+Ee67wMx+KknuPiTpekm/kLRV0p3u/uR01AwAAACcDHP3fNcQifb2du/o6Mh3GQAAAChiZrbZ3bPeE2amt44AAAAABYmgDQAAAESAoA0AAABEgKANAAAARKAs3wUAADBdDvfHtetAn3YdDB7d/YOqr4qpoapcjdUx1VfH1FhdrsbqcjVUx1QZK813yQAKGEEbAFA0hoYT2nP4WCpIpx5huD7cHx+xf6zUFB8ee/WtyliJGqqC0J0M3w3Vyffh66qYGmuC5+S2WCl/MAZA0AYAFJijx+LadbBPLxzs084DIwP17kP9Gkqkg3NZiWlRY5UWN1XrZYvna0lTdfio0eKmKtVVxtQ/OKzu/kEd6o2ru29Q3f1xHeobVHdf8P5QXzz1entXT7BPX3zE54xWW1EWhvJkQA8DeXVM9dXB7HljdXlqBr2hKqY5VTGVlth0/E8IFBV318BQQpJm3F+hCNoAgBklkXC9dOSYdh4IwvSug33aGT6/cLBPB3sHR+xfXxXT0uZqrVtYrzeun6+lYZhe3FSt+fWVKptgdrmqvFRV5VWaX1+Vc43urp6BoTCAB8H8UN+gDvfHg8DeP5ga7+6L64WDferuj+twf1xj3b7CTGEbS+aseTiLXlWuxpqY6qtGtrY0VMdUW1EmMwI6Cou7qz8+rJ5jQ+oZGFLvwLCODsTVOzCs3oFgLBgPXx8bUu/gkHoGhtVzLNgvc5+hhOtPL16t61+/Kt/f2ggEbQDAtOsdGNILh/pG9EsnWzw6D/VrcDiR2re0xLSgoVJLmqp1yenzUrPSS5urtbixWvXVsWmv38xUVxlTXWVMi5tyP2444TrSH0/Nmh9OhfS4Didnz/uD2fP9PQPa0dWj7r64egaGxvyaZSWWbmkJQ3pjGMLHCuwNVeWqKp9ZM3+Y+RIJD8NuMgBnBuUs4Th8HYTo5Ot0aB7nj0IpZsFfiGorylQTPtdWlKm1rkK1FTHVVpSqJtz26hWTOBmnCUEbADDlEgnXvp6Bka0dB3rD1/3a3zMwYv+6ijItaa7Wmnl1uui0Ni1prk4F6gUNVUXT81xaYmqsKVdjTbmWqybn4+LDiVT7Snd/XId6B1OBPLO1pbsvrs5DfdqyO5hVPxZPjPk1K8pKRsyMp0J4qs2lPOPi0OBC0YaqcpWXFcfPYrYYGk6MmC3OOlM8MKSewYzXA8PqyTK73Dc4nNNnlpWYaivLVFMeBuPKMtVXxbSwofK4wFyTJUTXVJSqtjJ4XRUrLei/2BC0AQAn5Fh8ON3acSDd2rEzfE72TErBrNSC+iotaarWhafOHRGklzRVq6E6VtD/mEYtVlqi1roKtdZVTOq4Y/Hhka0tffFw1jxsbckI7M/s69GhncHr8frPa8pLg1nzcGY8ay96TUz1VeledPrPJ2dgaDgMvUEw7g1D8Ngzx8OpmeLR+2eeh+MpLytRXRh4ayrKVFdRppbaci1trlZdGJprKsqC16lgXKrailgQjDMCc0VZCedziKANAMjK3bW/Z1C7DoYz0Qf6tfNgbypc7z0ycla6urxUS5qqtaKlRhesaU31SS9trtHChipmQvOgMlaqefWlmldfmfMx7q7ewWEd6g17zke3tqQuEg1C+u7ufnWH/elj5XMzaU5lup2lsXqMXvTUtnI11MRUVyD95+6uY/HEiB7jo+HscLLVItts8VitFuOthJOpKpae+U2G3QUNlVlnjGtTIbo0HZbL08G5WP5qNNMQtAFgFhsYGlbnof4RS+Blvu6Pj/xT8bw5lVrSXK3XrmpN90mHs9LNNeUFEYowPjNLBbPFkzgukXAdOZa+ODTV2tKb7jtPbjvYG8ygd/fGdXSc/vPSEgsD+ajlFZNLKlZnXwO9MjbxjGoi4eqLT6LHOCM4p14PDKV6j3PtN062U6RmgSvL1FxTnXp9fEgeOWtck9yvvIy/EhQAgjYAFDF316G+uHaG/dEvjGrz2HPk2IhVMCpjJal2jnNXtmhJU1XY5lGjRY1VM27pLMwcJSUWzlCXa9kk+88P98dHLac4eFxg7+6La3f3MT354hF198WP+yUwU3lZSXpmvDpoWxkRoo8NqTfHfuPSElNNeanqKoOwm2yfmF+fZea4MgjGNeVlGTPNZamWjKpYqUoIx7MKQRsACtzgUEIvdvenlsF7IZyRTr4evWJFa12FljZV69UrmsPWjnSvdGtdBbPSmFax0hK11FaopXby/eeHx1nzPHN5xcGhhJpqyrW4qVq15Zkzx9l7jGsr0kGZfmOcDII2ABSAw31x7Uz2So9q83ixu3/En63Ly0q0uLFKS5tr9KrlTanWjqXN1VrUWKXqcv7Tj8JXGStVZaxUbXNy7z8Hphv/tQWAGSDz1uEjV/Do1a4DfTpybOSsdEttMDv3iqWNuvKshRlhukZz6yr48zQAzAAEbQCYJkePxY+722Hy9ehbh8dKTYsag/B81uLGoLWjOX3Hw9oK/vMNADMd/6UGgCkyHN46PGjrSN+cJXmjlkN98RH7N1bHtKSpWusX1uvyM+aPWA5v3pxKVhQAgAJH0AaASegdGMraJ73rYJ86D/WNWP+2tMS0sKFKS5urddn6IEgvDcP0kuZqzamc/luHAwCmD0EbAEY52DuoHV092nmg97g2j/09gyP2rass09Lmaq2dX6dLTp+XWr1jaXO15tdXqoybQADArEXQBjArubsO9A5q296j2tHVo+17e1KvD/Smw3SJSQsagluHv2FtW5Zbh5fn8bsAAMxkBG0ARc3dte/ogLZ39Wj73qPa1tWjHXt7tL3r6Iie6bqKMq1qq9Ub1rZpVVutVs6t1bLmGi3g1uEAgBNE0AZQFNxde48MaNveo9re1aMdXUe1fW+Ptnf16HB/OlDPqSzT6rY6XbpuvlbNrdWqtlqtmluntjncqAUAMLUI2gAKirvrxcPHtD2z5aPrqHbs7dHRjDsgNlbHtKqtTpefEQTq1W11WtlWq9ZaAjUAYHoQtAHMSImEa3d3v3Z09aRmqbd39WjH3qPqHRxO7ddSW66Vc2v1lrMWanVbrVbOrdOqttpJ384ZAICpRtAGkFeJhKvzUH9GmA5mqnd09agvI1C31lVodVut3t6+WCvn1oZtH3VqquFiRADAzDQtQdvMbpF0uaQud18XjjVJukPSMknPS3qHux/KcuywpCfCt7vc/c3TUTOAqTWccO062KftyUAdPj+zr0fH4onUfvPmVGpVW62ueuVirZpbF85S17K6BwCg4EzXjPatkr4m6V8yxj4h6b/c/Utm9onw/Z9nObbf3c+MvEIAU2JoOKGdyUAdXoyYDNSDQ+lAvaC+Uivb6nTOiuZwlY86rZxbq/oqbuICACgO0xK03f0+M1s2avgKSRvC19+S9CtlD9oAZqD4cELP7+8NZ6eDlo/te3v03P5eDQ6nA/Wixiqtmlur165q0crwosRTWmtUx10RAQBFLp892m3uvid8/ZKktjH2qzSzDklDkr7k7ndNR3EAAgNDw3p+f5+2dx3Vtr3pZfOe29+roURwu3EzaXFjtVbNrdWGU1u1Orwg8ZTWWtVUcCkIAGB2mhH/Arq7m5mPsXmpu+82sxWS7jWzJ9z9mWw7mtl1kq6TpCVLlkRULVCcjsWH9dz+3pF3Suw6qp0H+jQcBuoSk5Y0VWtVW50uOq0ttQb1Ka21qiovzfN3AADAzJLPoL3XzOa7+x4zmy+pK9tO7r47fH7WzH4l6SxJWYO2u2+UtFGS2tvbxwruwKx2LD6cWtUjPUvdo50HehXmaZWWmJY2BzPUb1w3PxWoV7TWqDJGoAYAIBf5DNo/kvReSV8Kn384egcza5TU5+4DZtYi6VxJfz2tVQIFqm9wSM909Y5s+ejq0a6DffIwUJeVmJa11OjUeXV608sWpO6UuLylRhVlBGoAAE7GdC3vd5uCCx9bzKxT0l8qCNh3mtkHJO2U9I5w33ZJH3L3ayWtlfQNM0tIKlHQo/376agZKBS9A0Opm7rsCFf42Lb3qDoP9af2iZWaVrTUat3Cer31rIWpZfOWNteovKwkj9UDAFC8zL04Oyza29u9o6Mj32UAU+bIsXjQ8rF3ZMvH7u50oC4vLdGK1hqtaqsLbzseLJu3tLlasVICNQAAU83MNrt7e7ZtM+JiSABph/vj2hEG6eSyeTu6erTn8LHUPhVlJTqltVavXNaod7YtSd0pcUlTtcoI1AAAzAgEbSBPuvsGgzAdLpeXbP/oOjqQ2qcqVqqVc2t1zopmrWyrTS2bt6ixWqUllsfqAQDARAjaQMQO9Ayk7o6YebfE/T3pQF1dXhre1KVVq9tqU6t8LGyoUgmBGgCAgkTQBqaAu2t/z2Bqdjr93KODvYOp/WoryrSqrVavP7VVq+bWBbPUbXWaP6eSQA0AQJEhaAM5cnf1x4d1sHcwdafE1Cx1V4+6++Kpfesqy7S6rU4Xn9aWujBxVVut5s2plBmBGgCA2YCgjVlpaDih7v64uvsGdagvrkO9g+rui6u7P3jf3TeoQ71xHeobOT44lBjxdeqrYlrdVqs3rp8fhOlw2bzWugoCNQAAsxxBGwXN3XV0YEjdvaNDcvp1d388Pd43qO7euI4ODI35NctKTA3V5WqsjqmxulxLm6t1ZnWDGmpiaqgKxpc0V2vV3Dq11JYTqAEAQFYEbcwYA0PD6u4LZpEP9WaG5GBWORmeD2cE6u6+uIYSY68FP6eyTI015WFwLteKlprU68aamOqrgjDdWF2uhuqYGmvKVVNeSngGAAAnjaCNKZdIuA73xzNCcroN43A4ltmekQzUfYPDY37NirKSdBiuLtfqtlrVV6VnnZPjQXgOxuurYqwpDQAA8oagjTElL/7L2sN8XGvGYGo2+nB/XGPdcLTEpIbqcjVUxdRQHdP8+kqtnT8nCMw15RkzzLFg5rkmeF8ZK53ebx4AAOAkEbRnifhwIphlPsmL/zLVlJeOCMMLG6qOC8kNVRmzzdXlqqssYxk7AAAwKxC0C8y0XvyXDMwZz401wWx0fXVMFWXMMgMAAIyFoJ1HuV78lxyf1MV/YQsGF/8BAADkB0F7CnUe6lPX0YEx2zCiuPgvc9aZi/8AAABmDoL2FPrYnY/pN88dHDFWYkrNIo+++C8dko/va64qpy0DAACgkBG0p9BHLlqt/sFhLv4DAAAAQXsqvXpFc75LAAAAwAxBQy8AAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAYI2AAAAEAGCNgAAABABgjYAAAAQAXP3fNcQCTPbJ2lnHj66RdL+PHwuphc/59mBn3Px42c8O/Bznh3y9XNe6u6t2TYUbdDOFzPrcPf2fNeBaPFznh34ORc/fsazAz/n2WEm/pxpHQEAAAAiQNAGAAAAIkDQnnob810ApgU/59mBn3Px42c8O/Bznh1m3M+ZHm0AAAAgAsxoAwAAABEgaAMAAAARIGgDAAAAESBoAzkws7KM17Vm1m5mTfmsCcCJM7MmzmEAUSNoAxMws/dJ2mtm28zsMkmPS/pfkh4zs2vyWhyAnJnZEjO7Pbxz8G8k/dbMusKxZXkuD8AUMLMn8l1DprKJdwFmvY9JWiOpTtJjks5y92fMrE3SPZJuy2dxAHJ2h6S/k/Qudx+WJDMrlfR2SbdLenX+SgOQKzO7cqxNkuZNZy0TYXk/YAJm9qi7nxm+ftHdF2Rse9zdz8hbcQByZmbb3X3VZLcBmFnMLC7pO5Kyhdi3uXvdNJc0Jma0gYntMrP/qWBG+ykz+4qkH0h6g6Q9ea0MwGRsNrOvS/qWpBfCscWS3ivpd3mrCsBkPS7pZnffMnqDmb0hD/WMiRltYAJmNkfSnyj4zflrki6R9EeSdkr6H+5O2AYKgJmVS/qApCskLQyHd0v6kaR/cveBfNUGIHdm9lpJO919V5Zt7e7ekYeysiJoAwAAABFg1RFgAmZWamZ/bGZfMLNzR237TL7qAjA5ZlZtZh83sz8zs0oze6+Z/cjM/trMavNdH4DcmFlZ+O/yz83s8fDxMzP7kJnF8l1fJma0gQmY2TclVUv6raT3SPp/7v7RcNsj7v7yfNYHIDdmdqeC3uwqBSsJbVWwEsmbJc1z9/fksTwAOTKz2yR1K7jeojMcXqTgeosmd78qT6Udh6ANTCBzZZHwxjVfl9Qi6RpJv3b3s/JZH4DcJFcQMjNTcCHzfHf38P1jrCAEFAYz2+buqye7LR9oHQEmVp584e5D7n6dpEcl3SuJPzcDBcaDGaafhs/J98w6AYXjoJm93cxSOdbMSszsKkmH8ljXcQjawMQ6zOzSzAF3/7ykf5a0LC8VATgRHclebHd/f3LQzE6RdDRvVQGYrKslvU3puzZvk7RX0pXhthmD1hEAwKxnZub8gwgUHDNrliR3P5DvWrJhRhs4AWa2Md81ADh5yXOZkA0UJnc/4O4HZuq/ywRt4MS057sAAFOCcxkoDjPyXCZoAyemK98FAJgSnMtAcZiR5zI92gAAAEAEmNEGTsJM7QkDMDmcy0DhKKQ7NjOjDUzAzJrG2qTgJheLprMeACeGcxkoDoV0x2aCNjABMxuWtFPBP8ZJHr5f6O7lWQ8EMKNwLgPFoZDu2FyW7wKAAvCspAvdfdfoDWb2Qh7qAXBiOJeB4jDijs2SrjOzz2oG3rGZHm1gYn8nqXGMbX89jXUAODl/J85loBgUzB2baR0BAAAAIkDrCJADMztV0hWSFoZDuyX9yN235q8qAJPFuQwUh0I5l2kdASZgZn8u6XYFF0z9NnyYpNvM7BP5rA1A7jiXgeJQSOcyrSPABMxsm6TT3T0+arxc0pPuvio/lQGYDM5loDgU0rnMjDYwsYSkBVnG54fbABQGzmWgOBTMuUyPNjCxGyX9l5ltl5RcAmyJpJWSrs9XUQAm7UZxLgPF4EYVyLlM6wiQAzMrkXS2Rl508bC7D+evKgCTxbkMFIdCOZcJ2sAJMLPr3H1jvusAcHI4l4HiMFPPZXq0gRPzoXwXAGBKcC4DxWFGnssEbeDEWL4LADAlOJeB4jAjz2VaR4ATYGaL3L0z33UAODmcy0BxmKnnMjPawAlInsxm9kf5rgXAieNcBorDTD2XmdEGToKZ7XL3JfmuA8DJ4VwGisNMO5dZRxuYgJk9PtYmSW3TWQuAE8e5DBSHQjqXCdrAxNokXSLp0Khxk7Rp+ssBcII4l4HiUDDnMkEbmNiPJdW6+6OjN5jZr6a9GgAninMZKA4Fcy7Tow0AAABEgFVHAAAAgAgQtIEJmNkZZvZrM3vBzDaaWWPGtt/mszYAueNcBopDIZ3LBG1gYl+X9DlJ6yVtk/SAmZ0SbovlqygAk8a5DBSHgjmXuRgSmFidu/88fH2zmW2W9HMze48kLnIACgfnMlAcCuZcJmgDOTCzenc/LEnu/ksz+wNJ35fUlN/KAEwG5zJQHArlXKZ1BJjY/5K0NnPA3R+XdKGkH+SlIgAngnMZKA4Fcy6zvB8AAAAQAWa0gQmYWb2ZfcnMnjKzg2Z2wMy2hmMN+a4PQG44l4HiUEjnMkEbmNidCm7zusHdm9y9WdIF4didea0MwGRwLgPFoWDOZVpHgAmY2dPuvmay2wDMLJzLQHEopHOZGW1gYjvN7ONm1pYcMLM2M/tzSS/ksS4Ak8O5DBSHgjmXCdrAxK6S1Czp/5nZITM7KOlXCpYQekc+CwMwKZzLQHEomHOZ1hEgB2Z2qqRFkn7t7j0Z45dmLJoPYIbjXAaKQ6Gcy8xoAxMwsxsk/VDS9ZK2mNkVGZv/Kj9VAZgszmWgOBTSucydIYGJfVDSK9y9x8yWSfqemS1z97+XZPktDcAkcC4DxaFgzmWCNjCxkuSfpdz9eTPboOCkXqoZdkIDGBfnMlAcCuZcpnUEmNheMzsz+SY8uS+X1CJpfb6KAjBpnMtAcSiYc5mLIYEJmNkiSUPu/lKWbee6+4N5KAvAJHEuA8WhkM5lgjYAAAAQAVpHAAAAgAgQtAEAAIAIELQBAACACBC0AaDImNmvzGzAzI6a2WEze9bMvm1mr5jE13jezN4dZZ0AUOwI2gBQnL7g7nXuXi/pAkk7Jf3azN6a57oAYNYgaANAkXP3ne7+GUn/IumrFvjvZvZUOOu9y8z+p5mVSpKZ/YekJZK+aWY9ZnZ3OF5mZp8ys21m1m1mD5pZe/6+MwCY2QjaADB73C5poaQ1kjolXSZpjqQrJL1f0rWS5O5vkrRL0rXuXuvuF4fH3xTue6mkZkm3SPq5mTVO5zcBAIWCoA0As0dn+Nzs7t939+c88DtJ35Z04VgHmplJukHSn7n7s+4+7O7/JGmPpP8WeeUAUIDK8l0AAGDaLAqfD5jZNZI+KmmFgn8LyiX9epxjWyTVSvoPM8u801ks4+sCADIQtAFg9rhK0m5JvZL+VdKVkn7m7oNmdrOkzH7rxKhj94fHvcHdH56OYgGg0NE6AgBFzswWm9lNkt4n6b8rmJkukbRPUtzMXi3pPaMOe0nSquQbd3dJfy/pZjNbFX7dWjO7xMwWRP9dAEDhseC/nQCAYmFmv5J0jqRBSS7pgKRNkv7e3X8b7vNZSR9W0DLyS0nPSzrT3TeE298o6auSmiT92t0vM7MyBX3a1ypoF+lV0G7yYXdP9n8DAEIEbQAAACACtI4AAAAAESBoAwAAABEgaAMAAAARIGgDAAAAESBoAwAAABEgaAMAAAARIGgDAAAAESBoAwAAABEgaAMAAAAR+P8BIQ1/KBVrJGgAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plot_daily_accept_rate(df, 'Accepted', 'Rejected', \n", - " **{'hue':['pyChannel'], 'allTime':True, 'shrinkTicks':True})" + "# plot_daily_accept_rate(\n", + "# df,\n", + "# \"Accepted\",\n", + "# \"Rejected\",\n", + "# **{\"hue\": [\"pyChannel\"], \"allTime\": True, \"shrinkTicks\": True},\n", + "# )\n", + "# TODO more or less fits the engagement trend charts\n", + "ih.plots.success_rates_trend_line(by=\"Channel\")" ] }, { @@ -592,22 +124,12 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAFGCAYAAABUojiWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAtTUlEQVR4nO3deZTkdXnv8ffTy3TPAiMwIwmOyKDDJptjB6OgokQF9TLG6DWIyqISvFH03hwN0RgkRi9XNN64O1E05CLuIuIGshzEoNIjI4Ks4gCDIgMDwjBLb8/9o6qb6uqq7urp+nX18n6d06e7vlW/qqf7p/Ez3zy/5xeZiSRJkqTmamt1AZIkSdJcZNCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCmDQliRJkgrQ0eoCirJs2bLcZ599Wl2GJEmS5rB169Y9kJnLaz03Z4P2PvvsQ29vb6vLkCRJ0hwWEXfVe87WEUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpANMStCPivIi4PyJurFh7f0TcEBHrI+LSiNirzrGD5desj4iLp6NeSZIkaaqma0f7i8CxVWvnZuahmXk4cAnwT3WO3ZaZh5e/ji+wRkmSJKlppiVoZ+bVwOaqtUcqHi4GcjpqkSRJkqZDS29YExEfAN4A/BF4QZ2XdUdELzAAnJOZF01TeZIkSdJOa2nQzsz3AO+JiH8A3gqcVeNlT8nMeyNiX+CKiPhVZv6m1vtFxGnAaQB77713UWXXdfnNf+DBx/pYtKCdRQvaWdjZ8fjPC9pZtKD0uKujjYiY9vokSZI0fWbKLdgvAL5HjaCdmfeWv98ZEVcBzwBqBu3MXAusBejp6Zn2VpS1V9/Jz367ecLXtQUs7Gxn4YLqIP54OC89317xfMVrO0uhffTz5bXOdtrbDPGSJEmt1rKgHRGrMvP28sM1wC01XrMbsDUzd0TEMuBI4EPTWOakrH19D49s72db/yBb+wbZ2jfAtr7Sz9vKj7f2l37e1jc48vPWvoGR1zz02Lby8Y+vDQxN7t8MXR1t5QDeURHg20fWujurA3o5yHeO3X0fCfOdpfda0OFESEmSpEZMS9COiAuBo4FlEbGR0s71SyNif2AIuAs4vfzaHuD0zHwTcCDw2YgYonTh5jmZ+evpqHlnLF3UydJFnU1/376BoXIwH6gI7aODfCm0l5/vHxwb8PsGefCxPu55aNuotR0DQ5OqpaMtKnbSO0YCfGWgH7NTX7UDPyrIj/oHgC01kiRp7ojMuTnso6enJ3t7e1tdxow3OJQjO+iPB/jHA/rju/OlID/8eNRrx/wDoBz6+weZzH+8otxS83hAH90e010RyocDfM3d9zr98bbUSJKkZouIdZnZU+u5mdKjrRZpbwuWdHWwpKv5/1HITHYMDI3dfa8I55UBfVvlrnzVTv3v/9g/eie/f5D+wcn9I3HBcEtNRUAfvRM/dqe+sj++1u788PqCdnfjJUnSaAZtFSYi6O4s7UTvvnhB09+/f3BoTHvM4zvuo4P8mPWK/viHHuvj3oqwv7VvgO39k2upaW+LigA/9uLVMRe7Vl/oOk5/fHdHO23uxkuSNOsYtDVrdba3sXRhG0sXNr8vfmikpWZwwv74UT3xVTv1j+0YYNOjO6pabgaY5PWtY3vhy+F84aid9vEn2dQM8p3tdLR7gaskSUUwaEs1tLUFi7s6WFxgS822itaY6jaa4faYbVU77aNabfoGuf/R7WPW+gYntxu/oL2t6kLVx/vjawb8qjGT9XboFzozXpI0zxm0pWlW2VKzWwHvPzA4VNEaU78/frwxk1v7Bnl4Wz+//+O2MX30k9EW1BwzubDmTnutSTa1x0wOv5ctNZKkmcygLc0xHe1t7Nrexq7dxbTUbB+osftedfHq6AA/uj9+uI1m82PbRl/82j/I4CR7aro728ZcvDpmOk3FFJuau+9VYyaHn++0pUaSNEUGbUkNa2uL8i5zMS01fYNDNcdMbq1qoxkznaaqP/6BLX1s7ds6EuC39g3SN8mZ8Z3tMaY9Zsx0mhr98RONmVxkS40kzRsGbUkzQkTQ1dFOV0c7T1jU/PcfGBwa1fde6+LVelNrKnvpH90+wP2P7GBr/+gLYic7M35RjZs7Lewc2+devfte2R9fPWZyeHffmfGSNDMYtCXNCx3tbezS3sYuBbTUZCbb+4fGjJmsO52mxpjJ4VD/8Nb+MRe/DkyypaZreGZ8RRjv7hxvp33iMZPD/fELOmypkaRGGbQlaYoiYmTU4h4FvH/fyJSaxsZMjtptrwjymx/rY+NDo9d2TLKlpqMtxly8OvZi16ogX7UDPyrIV/THd3faUiNpbjFoS9IMt6CjjQUdbSyl+bvxgyMz4wdq9sdPNGZyuAVnS3lm/KiLX3eipWbU1JnODrpH9cGP7Y+vuftepz/elhpJ082gLUnzWHtbsKSrgyUFzoyvN2ZyW9/QSJgfM52mapLNfY/0j5kz3z84uZaaBcMtNZ3tpQBfPTKyxpjJMRe61umPX9DubryksQzakqRCVM6M333xgqa/f//g0Jj2mMoLWkffkbV+f/zDW/v43cOjd+2390+upaa9LUZdwFprzGTlTnutmzzV64/v7nBmvDRbGbQlSbNSZ3sbSxe2sXRhMTPjRwX1mjd6GntX1+pJNlv7Bnhgy46q0D/AJK9vHdlhH3VR6zhjJkdd6FpnzOTwcR3OjJcKY9CWJKlKW1uwuKuDxQW21NQM7LWm09Tpj9/WN8j9j24fs9Y3OLnd+AXtbaMuaK0/iab2mMl6O/QLnRkvGbQlSZpOlS01uxXw/gODQ1V3Zh2omhNfNbWmxpjJrX2D/HFbP/f9cVvVjv3gpGppC0Z23RdWh/YaYyYr++PHGzM5HPRtqdFMZ9CWJGkO6WhvY9f2NnYtYGb80FCyfaB2YK9uj9lafYOoqj76zY9tG33xa/8gg5PsqenubKs7ZnLMSMl6/fF1jum0pUZNYNCWJEkNaWuL8i5zMS01fYNDE46ZrGy3qXen1we29LG1b+tIgN/aN0jfJGfGd7ZHRYCv3mmv3x8/0ZjJRbbUzCsGbUmS1HIRQVdHO10d7TxhUfPff3AoJz1msnK3fXjt0e0D3P/IjtE3htqJmfHDfewLq1piqvvcq3ffK/vjq8dMDv+DwJnxM4dBW5IkzXntbcEu3Z3sUkBLTWayvX+oxpjJWpNoao+ZHH7+4a39Yy5+HZhkS01XR9uYmzqNv9M+8ZjJ4X8MLOiwpWYyDNqSJElTEBEju9N7FPD+fSNTairGTPbX2X0vv25b3+j2m1JffB8bHxq9tmOSLTUdbTF2Ok1Vf/yYMZNVO/CjgnxFf3x359xrqTFoS5IkzWALOtpY0NHGUpq/Gz84MjN+dCvMmOk0dcZMDrfgbNkxwKZHd4xuv5lkSw0wdjpNjTGTlf3xlaH96Xvtyqo9d2n632gqDNqSJEnzVHtbsKSrgyUFzowfDuzbR/XBVwT5Gv3x1Re/3vdI/5hj+gdHp/i/e9F+Bm1JkiTNfZUz43dfvKDp798/ODSqlWbX7pkXa2deRZIkSdIEOtvbWLqwjaULm99S0yxeOipJkiQVwKAtSZIkFcCgLUmSJBXAoC1JkiQVwKAtSZIkFcCgLUmSJBXAoC1JkiQVwKAtSZIkFcCgLUmSJBVg2oJ2RJwXEfdHxI0Va++PiBsiYn1EXBoRe9U59qSIuL38ddJ01SxJkiTtrOnc0f4icGzV2rmZeWhmHg5cAvxT9UERsTtwFvAs4AjgrIjYrdhSJUmSpKmZtqCdmVcDm6vWHql4uBjIGoe+BLgsMzdn5kPAZYwN7JIkSdKM0tHqAiLiA8AbgD8CL6jxkicB91Q83lhekyRJkmasll8MmZnvycwnAxcAb53Ke0XEaRHRGxG9mzZtak6BkiRJ0k5oedCucAHwVzXW7wWeXPF4RXltjMxcm5k9mdmzfPnyAkqUJEmSGtPSoB0RqyoergFuqfGyHwIvjojdyhdBvri8JkmSJM1Y09ajHREXAkcDyyJiI6VJIi+NiP2BIeAu4PTya3uA0zPzTZm5OSLeD1xXfqt/zszNYz5AkiRJmkEis9agj9mvp6cne3t7W12GJEmS5rCIWJeZPbWem0k92pIkSdKcYdCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCmDQliRJkgpg0JYkSZIKYNCWJEmSCtDR6AsjYl9gd2Az8NvMzMKqkiRJkma5cXe0I2J1RPxHRDwI3AH8vPz9wfL66ukoUpIkSZpt6gbtiDgf+BZwD/BKYBmwoPz9FcDdwDfLr5MkSZJUYbzWkZ8Ap2TmYNX6ZuBq4OqIeB/wxoJqkyRJkmatukE7Mz870cHlEL62qRVJkiRJc0DDF0MOi4hjgGcAd2TmRU2vSJIkSZoDJjXeLyLeDfwTsAdwZkS4my1JkiTVMO6OdkSszsxfVCy9GDg6MzMiFgL3AqdN9CERcR7wcuD+zDy4vHYu8N+APuA3lPrBH65x7AbgUWAQGMjMngZ+L0mSJKmlJtrR/veI+GBELCg/3gS8PiJWAW8GNjT4OV8Ejq1auww4ODMPBW4D/mGc41+QmYcbsiVJkjRbTBS0nwXsAK6LiGcDZ1AKzBcBxwAnNvIhmXk1pWkllWuXZuZA+eFPgRWNly1JkiTNbOO2jpSD8NkR8Q3g3yndsOZNmbm1yXWcCnylXhnApRGRwGcz075wSZIkzXgNXQyZmTcCR1K6ec115ckjTRER7wEGgAvqvOSozFwNHAf8bUQ8b5z3Oi0ieiOid9OmTc0qUZIkSZq0iW7B/uyI+GVEbAGuAS4B1gD/GBGfi4hdp/LhEXEypYskT8zMrPWazLy3/P1+SneqPKLe+2Xm2szsycye5cuXT6U0SZIkaUom2tH+AvA+SuP8PgL8W2bekZkvANYBP9vZD46IY4F3AcfXa0WJiMURscvwz5Smnty4s58pSZIkTZeJgvYy4EeZuQO4AhjZJs7MTwMvauRDIuJC4Fpg/4jYGBFvBD4B7AJcFhHrI+Iz5dfuFRHfKx+6J3BNRPySUn/4dzPzB43/epIkSVJrTHRnyM8A6yLiOkotG+dWPpmZGxv5kMw8ocby5+u89nfAS8s/3wkc1shnSJIkSTPJRFNH/rE8ceRpwL9k5s3TU5YkSZI0u020o01mXg9cPw21SJIkSXNG3R7tiDi7fAFiXRGxJCLObn5ZkiRJ0uw23o52F/DbiPg2pdul/xp4BNgVOAj4C+AVwOcKrlGSJEmadeoG7cw8MyI+BZxOacTfAZTu0hjArZRuw96TmXcXX6YkSZI0u0x0MeTdwLuBd0dEN7Ab8FBmbp+O4iRJkqTZasKLIYeVw/XvC6xFkiRJmjMmumGNJEmSpJ1g0JYkSZIKYNCWJEmSCmDQliRJkgrQcNCOiNdFxGURcUP58fMi4pXFlSZJkiTNXg0F7Yj4X8DZwPeBvcvLm4B3FVSXJEmSNKs1uqP9FuC4zPxXSjetAbgNeFohVUmSJEmzXKNBe/fMvK3883DQjoqfJUmSJFVoNGj/OiJeXrV2LPDLJtcjSZIkzQmN3hny3cB3I+KrQFdEfBz4a6A6fEuSJEmiwR3tzPwx8OfANuDK8nFHZ+bPCqxNkiRJmrUa2tGOiH0y89fA26rWn5KZdxVSmSRJknZaf38/GzduZPv27a0uZU7o7u5mxYoVdHZ2NnxMo60jNwC71li/Hti94U+TJEnStNi4cSO77LIL++yzDxHR6nJmtczkwQcfZOPGjaxcubLh4xq9GHLM2YmITpw6IkmSNCNt376dPfbYw5DdBBHBHnvsMen/78C4O9oRcRmlMN0VEZdWPb038ItJfZokSZKmjSG7eXbmbzlR68g15e/PB35SsT4E3Ad8bdKfKEmSJM0D4wbtzDwbICJuzsyvTk9JkiRJmguWLFnCli1bxn3NPvvsQ29vL8uWLZuWmq666io+/OEPc8kllxT+WQ1dDDkcsiOiG1hORc92Zt5dTGmSJEnS7NXQxZARsW9E/BfwGLAB+G3FlyRJklTXVVddxdFHH82rXvUqDjjgAE488UQyH5+p8aEPfYhDDjmEI444gjvuuAOADRs28MIXvpBDDz2UY445hrvvLu3tnnzyyXz9618fOXbJkiUTfsYPfvADDjjgAFavXs03v/nN6fq1Gx7v9wngHuA0Sn3bRwL/DHynoLokSZLUJGd/5yZ+/btHmvqeB+21K2f9t6c3/Prrr7+em266ib322osjjzySn/zkJxx11FEALF26lF/96lecf/75vOMd7+CSSy7hbW97GyeddBInnXQS5513HmeccQYXXXTRpD+jp6eHN7/5zVxxxRU87WlP4zWvec1Ufu1JaXS837OAN2XmjQCZeRPwN8A7iypMkiRJc8cRRxzBihUraGtr4/DDD2fDhg0jz51wwgkj36+99loArr32Wl772tcC8PrXv55rrrlmzHs28hm33HILK1euZNWqVUQEr3vd65r/y9XR6I72EKXbrwNsiYgnAJspjfiTJEnSDDaZneeidHV1jfzc3t7OwMDAyOPK0XkTjdHr6OhgaGgIgKGhIfr6+hr6jFZodEf7JkrtIgA/Az4KfAx7tCVJkjRFX/nKV0a+P/vZzwbgOc95Dl/+8pcBuOCCC3juc58LlKaUrFu3DoCLL76Y/v7+cd/7gAMOYMOGDfzmN78B4MILLyzkd6il0R3tM3j8LpDvBD5D6Zbsf1NEUZIkSZo/HnroIQ499FC6urpGgvDHP/5xTjnlFM4991yWL1/OF77wBQDe/OY3s2bNGg477DCOPfZYFi9ePO57d3d3s3btWl72spexaNEinvvc5/Loo48W/jsBROUVn5M+OGJpZv6xifU0TU9PT/b29ra6DEmSpJa4+eabOfDAA1tdxpxS628aEesys6fW6xttHal+w+6IeBdw584cL0mSJM114wbtiNgnIq6IiEci4ucRsV9EvAS4HXgjpZYSSZIkSVUm2tH+1/L3MylNHfka8DngvcCBmXlBIx8SEedFxP0RcWPF2rkRcUtE3BAR3ypPMql17LERcWtE3BERZzbyeZIkSVKrTRS0nwO8OjM/BbwGOARYk5lfzMyhSXzOF4Fjq9YuAw7OzEOB24B/qD4oItqBTwLHAQcBJ0TEQZP4XEmSJKklJgraizLzQYDMvA/Ykpm/mOyHZObVlOZuV65dmpnDww1/CqyocegRwB2ZeWdm9gFfBtZM9vMlSZKk6TbReL+IiD8FhieHD1Y9JjN/14Q6TgW+UmP9SZRu/T5sI6W7VEqSJEkz2kQ72osphdt7yl9LKx4Pf5+SiHgPMAA01O89wXudFhG9EdG7adOmqb6dJEmSpmDjxo2sWbOGVatW8dSnPpW3v/3to+7kWMsHP/jBkZ83bNjAwQcfXHSZo7zvfe/jwx/+cFPea6KgvRLYt+JrZY2fd1pEnAy8HDgxaw/0vhd4csXjFeW1mjJzbWb2ZGbP8uXLp1KaJEmSpiAzeeUrX8krXvEKbr/9dm677Ta2bNnCe97znnGPqwzas924QTsz75roa2c/OCKOBd4FHJ+ZW+u87DpgVUSsjIgFwF8DF+/sZ0qSJGl6XHHFFXR3d3PKKacA0N7ezkc/+lHOO+88PvWpT/HWt7515LUvf/nLueqqqzjzzDPZtm0bhx9+OCeeeCIAAwMDnHjiiRx44IG86lWvYuvWUmy8/PLLecYznsEhhxzCqaeeyo4dO4DSLdofeOABAHp7ezn66KOB0k71qaeeytFHH82+++7Lxz72sZHP/8AHPsB+++3HUUcdxa233tq0v0Gjt2Cfkoi4EDgaWBYRG4GzKE0Z6QIuiwiAn2bm6RGxF/C5zHxpZg5ExFuBHwLtwHmZedN01CxJkjRnfP9MuO9XzX3PPzkEjjun7tM33XQTz3zmM0et7brrruy9994MDAzUPOacc87hE5/4BOvXrwdKrSO33norn//85znyyCM59dRTR0L6ySefzOWXX85+++3HG97wBj796U/zjne8Y9ySb7nlFq688koeffRR9t9/f97ylrdwww038OUvf5n169czMDDA6tWrx9S9s3bqzpCTlZknZOafZmZnZq7IzM9n5tMy88mZeXj56/Tya3+XmS+tOPZ7mblfZj41Mz8wHfVKkiRpZnjyk5/MkUceCcDrXvc6rrnmGm699VZWrlzJfvvtB8BJJ53E1VdfPeF7vexlL6Orq4tly5bxxCc+kT/84Q/8+Mc/5i//8i9ZtGgRu+66K8cff3zTap+WHW1JkiS10Dg7z0U56KCD+PrXvz5q7ZFHHuHuu+/mCU94AkNDj9+SZfv27XXfp9z5UPdxtY6OjpH3rn7frq6ukZ/b29vr7qw3S0M72hGxT531pzS1GkmSJM0JxxxzDFu3buX8888HYHBwkL/7u7/j5JNPZt9992X9+vUMDQ1xzz338POf/3zkuM7OTvr7+0ce33333Vx77bUAfOlLX+Koo45i//33Z8OGDdxxxx0A/Od//ifPf/7zgVKP9rp16wD4xje+MWGdz3ve87jooovYtm0bjz76KN/5znea8weg8daRG+qsX9+sQiRJkjR3RATf+ta3+NrXvsaqVavYb7/96O7u5oMf/CBHHnkkK1eu5KCDDuKMM85g9erVI8eddtppHHrooSMXQ+6///588pOf5MADD+Shhx7iLW95C93d3XzhC1/g1a9+NYcccghtbW2cfvrpAJx11lm8/e1vp6enh/b29gnrXL16Na95zWs47LDDOO644/izP/uz5v0Nak/Vq3pRxKOZuUvVWidwX2bu0bRqmqinpyd7e3tbXYYkSVJL3HzzzRx44IGtLmNOqfU3jYh1mdlT6/Xj9mhHxGVAAl0RcWnV03sDk74duyRJkjQfTHQx5DXl788HflKxPgTcB3ytiKIkSZKk2W7coJ2ZZwNExM2Z+dXpKUmSJEnNkJkTTulQYxppt67W0Hi/zPxqRCyhdLv0FcBG4LuZ+eikP1GSJEmF6+7u5sEHH2SPPfYwbE9RZvLggw/S3d09qeMaCtoR0QN8D9gG3E2pP/tjEfHSzPSKQ0mSpBlmxYoVbNy4kU2bNrW6lDmhu7ubFStWTOqYRm9Y8yngI5n5f4YXIuJdwKeB5s1AkSRJUlN0dnaycuXKVpcxrzU6R/tA4CNVa/8KHNDcciRJkqS5odGgvR44uGrtkPK6JEmSpCqNto5cClwSEZ8D7gL2AU4F1kbEa4dflJlfanqFkiRJ0izUaNA+FegHTqpYGyivD0vAoC1JkiTR+Hg/O+klSZKkSWi0RxuAiNgrIv68qGIkSZKkuaKhoB0RT4yIH1G6Uc2PymuviYhPFVmcJEmSNFs1uqP9MeC3wHJKvdoAVwAvKqIoSZIkabZr9GLIFwBPycztEZEAmbkpIp5YXGmSJEnS7NXojvYOqkJ5ROwObG56RZIkSdIc0GjQvhT4SER0VqydDXy3+SVJkiRJs1+jrSPvAi4CHgK6I+Jh4JfAmmLKkiRJkma3RudobwaeFxHPBFZSujtkb2ZmkcVJkiRJs1VDQTsidgP6MnMdsK68tjgiOjPz4QLrkyRJkmalRnu0LwaeXrV2MPDt5pYjSZIkzQ2NBu2nA71Va73AIc0tR5IkSZobGg3a24FFVWuLefzmNZIkSZIqNBq0rwE+GBFtABERwD8DPymqMEmSJGk2a3S83zsp3XL9ryLiTkqTR/qAFxZVmCRJkjSbNTre766IOBh4ObAPsAH4bmZuLa40SZIkafZqdEebzNwGfK3AWiRJkqQ5o6Ee7Yi4NCJeWLV2TER8v5iyJEmSpNmt0YshVwNXV639GPiz5pYjSZIkzQ2NBu0hoLNqrR2I5pYjSZIkzQ2NBu11wNuq1t4K/KKRgyPivIi4PyJurFh7dUTcFBFDEdEzzrEbIuJXEbE+IqpvmiNJkiTNSI1eDPn3wFUR8VfAbcAqYH/g6AaP/yLwCeD8irUbgVcCn23g+Bdk5gMNfpYkSZLUcg3taGfmDcBBwNeBR4BvAAdl5i8bPP5qYHPV2s2ZeevkypUkSZJmh8mM97sPOHf4cUQ8PSL+ITPPKKSyio8GLo2IBD6bmWsL/jxJkiRpyhrt0QYgIroi4g0R8RPgV5SmkRTtqMxcDRwH/G1EPG+c+k6LiN6I6N20adM0lCZJkiTV1ugc7YMi4t+A31Hqqe4Bjs3Mo4osDiAz7y1/vx/4FnDEOK9dm5k9mdmzfPnyokuTJEmS6ho3aEfE6yPix5QuXHw+8D7gSZT6rdcXXVxELI6IXYZ/Bl5crkWSJEma0Sbq0f4P4EHgZZk5chfIiMmNz46ICylNKFkWERuBsyiF9Y8Dy4HvRsT6zHxJROwFfC4zXwrsCXyr/HkdwJcy8weT+nBJkiSpBSYK2u8F3gRcFBHfA84DvjvZD8nME+o89a0ar/0d8NLyz3cCh0328yRJkqRWG7d1JDM/AOwLvILS9I9vAPcCTwD2Krg2SZIkadaa8GLILPl+Zr4SeArwKeA+4LqI+GrRBUqSJEmz0aTG+2Xm7zPz/ZR2udcACwqpSpIkSZrlGr5hTaXMTOB75S9JkiRJVSa1oy1JkiSpMQZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkABm1JkiSpAAZtSZIkqQAGbUmSJKkA0xK0I+K8iLg/Im6sWHt1RNwUEUMR0TPOscdGxK0RcUdEnDkd9UqSJElTNV072l8Ejq1auxF4JXB1vYMioh34JHAccBBwQkQcVFCNkiRJUtNMS9DOzKuBzVVrN2fmrRMcegRwR2bemZl9wJeBNQWVKUmSJDXNTO/RfhJwT8XjjeW1miLitIjojYjeTZs2FV6cJEmSVM9MD9qTkplrM7MnM3uWL1/e6nIkSZI0j830oH0v8OSKxyvKa5IkSdKMNtOD9nXAqohYGRELgL8GLm5xTZIkSdKEpmu834XAtcD+EbExIt4YEX8ZERuBZwPfjYgfll+7V0R8DyAzB4C3Aj8Ebga+mpk3TUfNkiRJ0lREZra6hkL09PRkb29vq8uQJEnSHBYR6zKz5j1hZnrriCRJkjQrGbQlSZKkAhi0JUmSpAIYtCVJkqQCGLQlSZKkAhi0JUmSpAIYtCVJkqQCGLQlSZKkAnS0ugBJkiRppw0NwsB2iHbo7G51NaMYtCVJkjQ1Q0MwuAP6t8HAjlLwHfkqP+6vfLytznrVcY2831B/qYYX/iM8752t/TtUMWhLkiTNBZlNDLkTBeCq9xvsm1rtbR3Q0V3x1QWdC0vfO7qheyl07Pn4486K13WUX/eU5zTn79hEBm1JkqRmySyFzkmF3Cbt/g5sn1rt0fZ4aK0MucNfC5bA4uUV6xUht174rRWaK786u6G9C9rnZiSdm7+VJEmavzJhaKCq7aDgkFv5WeQUio86obQcVhcsgkW7TyHk1jumG9o7m3UGVGbQliRJxRgcKCjkVgXoWn28OTS12sdrY+joKrcyNBpyJ7H7294JEc35+6vlDNqSJM1lwxMZRgJrnYvLGtr9bTDkDq/n4NRqb68MpTV2eBctqx9YO6teO5nd3/YFhl01hUFbkqSiTTSRYWdCbqNtDMMTGXZW+4LxQ+mi3Sfe/d2ZFof2Lmjzdh+a3QzakqT5od5EhimNI6sRdPtrrDVlIsM4vbWVExnq9feOF3Kr36/yuLb25vz9pXnIoC1Jmj7VExkaCrlN2v0d3DG12ocnMtQLpWMmMkw15HbP+YkM0lznf3Mlab6pOZFhZ0LuZHd/p3Miw862MYwTjA27kibJ/6shSa0yMpFhoiDbxJDbtIkME8zIHZ7I0FDIncTurxMZJM0iBm1J81vlRIb+bY1dXNascWRFT2Ro6MYSNXaFJzrGiQyS1BCDtqTWGxqqCq+TGB821XFkRU1kGA6lE05k2MkWh/YFTmSQpBnOoC2pZGQiQzND7ng7vxXPFzqRYSF0P6G5IbfyeMOuJKkOg7Y0k1ROZGh0Rm6jLQ4TBeNCJjJUhNKuXepPZNipcWTln53IIEmaofxfJ6laJgz2T2/ILWQiQ40d2QknMkwUcsc5xrArSdIo/i+jZq4JJzI0MFlhZy9ia+ZEhlo7sjUnMkwUfhsIxk5kkCRpxjBoa3x1JzIUGHKH15sxkWG8W/1Oqo1hEru/TmSQJEkYtGeH8SYyNDpZYWcnNQwNTK32kYkMdUJp9USG8W4DPJmL2JzIIEmSWsyg3Ux3/wy23DeJkNvgDu+UJzJ0jh9KhycyjLf7uzOTGpzIIEmS5jGDdjNd+S/w26trPxft44fSURMZmhByncggSZLUUiawZnrZv5Z2oZ3IIEmSNO+Z/ppp2apWVyBJkqQZwgZaSZIkqQDTErQj4ryIuD8ibqxY2z0iLouI28vfd6tz7GBErC9/XTwd9UqSJElTNV072l8Ejq1aOxO4PDNXAZeXH9eyLTMPL38dX2CNkiRJUtNMS9DOzKuBzVXLa4D/KP/8H8ArpqMWSZIkaTq0skd7z8z8ffnn+4A967yuOyJ6I+KnEfGK6SlNkiRJmpoZMXUkMzMiss7TT8nMeyNiX+CKiPhVZv6m1gsj4jTgNIC99967oGolSZKkibVyR/sPEfGnAOXv99d6UWbeW/5+J3AV8Ix6b5iZazOzJzN7li9f3vyKJUmSpAa1MmhfDJxU/vkk4NvVL4iI3SKiq/zzMuBI4NfTVqEkSZK0k6ZrvN+FwLXA/hGxMSLeCJwDvCgibgf+ovyYiOiJiM+VDz0Q6I2IXwJXAudkpkFbkiRJM9609Ghn5gl1njqmxmt7gTeVf/4v4JACS5MkSZIKEZn1rkGc3SJiE3BXCz56GfBACz5X08vzPD94nuc+z/H84HmeH1p1np+SmTUvDpyzQbtVIqI3M3taXYeK5XmeHzzPc5/neH7wPM8PM/E8t/JiSEmSJGnOMmhLkiRJBTBoN9/aVhegaeF5nh88z3Of53h+8DzPDzPuPNujLUmSJBXAHW1JkiSpAAZtSZIkqQAGbUmSJKkA03JnSEmSJKmZImJP4Enlh/dm5h9aWU8tXgzZBLPhREuavIg4PjMvbnUdKkZEPA04DLg5M3/d6nrUPBHRkZkD5Z+XAAcAd2bm5tZWpmaIiMOBzwBLgXvLyyuAh4H/kZm/aE1lYxm0p2A2nWjtvIg4BPh3Sv+Y+j7w95n5UPm5n2fmEa2sT80REa+sXgI+CfwPgMz85rQXpaaKiCuBV2fmAxHxeuC9wNXAs4C1mfnxlhaopoiIk4GPAA8Cb6f03+PfAvsB78rMC1tXnZohItYDf5OZP6ta/3Pgs5l5WEsKq8GgPQWz6URr50XENcC/AD8F3gScAhyfmb+JiOsz8xktLVBNERH9wA+B+ymFbIBXAV8HMjNPbVVtao6IuDEzDy7/fB1wbGY+GBGLgJ9m5qGtrVDNEBG/Al4A7AL8EnhG+f9e7wlc5nme/SLi9sxcVee5OzLzadNdUz32aE/N4uqQDZCZP42Ixa0oSIXYJTN/UP75wxGxDvhBeUfMf6nOHc8BzgGuy8xPA0TE0Zl5SmvLUhP1R8STMvNeYAvwWHl9B9DeurLUZIOZ+QDwQERsyczfAGTmHyJigkM1S3w/Ir4LnA/cU157MvAG4Ad1j2oBg/bUzJoTramJiKWZ+UeAzLwyIv4K+Aawe2srU7Nk5nUR8SLgbeUWg7/Hf0jNNf8TuDQivgHcBFwRET8EjgK+0NLK1Ex3R8T/prSjfUtEfAT4JvAXwO9bWpmaIjPPiIjjgDVUXCMHfDIzv9e6ysaydWSK6pzoi2faidbOi4jXUrqI5qdV63sD783MN7emMhUlIvYC/i/Qk5n7trgcNVFELAVeS6lftwPYCHw7M29paWFqmojYFfhbSv9Q/gTwEkotf3cB/5KZhm1NG4O2JEmS5oSIOC0z17a6jmHesKYgEXFaq2tQ8TzP84Pnee7zHM8Pnud5YUY14hu0izOjTrQK43meHzzPc5/neH7wPM8REXFARBxTnpNe6a6WFFSHQbs4fa0uQNPC8zw/eJ7nPs/x/OB5ngMi4gzg28DbgBsjYk3F0x9sTVW12aNdkIi4OzP3bnUdKpbneX7wPM99nuP5wfM8N5RnpT87M7dExD6U7nfwn5n5bzPt/haO95uCiLih3lPAntNZi4rjeZ4fPM9zn+d4fvA8zwttmbkFIDM3RMTRwNcj4inMsPYgg/bU7ElpbNBDVesB/Nf0l6OCeJ7nB8/z3Oc5nh88z3PfHyLi8MxcD1De2X45cB5wSEsrq2LQnppLgCXDJ7pSRFw17dWoKJ7n+cHzPPd5jucHz/Pc9wZgoHIhMweAN0TEZ1tTUm32aEuSJEkFcOqIJEmSVACDtiRJklQAg7YkadIi4n0R8aNW1yFJM5lBW5LmgIg4IyJ+U7X2tojIiDiuYm1hRGyPiOOnv0pJml8M2pI0N1wO7FueIzvsGOAm4IUVa0cC7cBV01eaJM1PBm1JmgMy8ybg95TCNRHRDjwfOGt4rewY4Dpga0S8OyJui4iHI+InEdFT+Z4R8eaIuDEi/hgR10fEi+t9fkScEhEbI+JZzf7dJGm2MmhL0txxBY+H6mcC9wEXA0+NiD3K68cAPwLOBtYAxwJ7ULrRww8iYjcohWzg74ETgd2A9wDfjIinVX9oRLy//NrnZebPivnVJGn2MWhL0tzxIx5vEzkGuCIz+yndDe8FEbEUWE2pzeQM4J2ZeWdmDmbm5yntiL+sfPzbgX/OzF9m5lBmfg+4Evjris9bEBH/j9LO+XMy886if0FJmk28M6QkzR2XA38SEQdRCtyfKa9fWX7cD2wHfg0sAb4TEZV3LesEVpR/Xgl8MiI+VvF8B7Cx4vGBwLOBF2fm5ib/LpI06xm0JWmOyMx7IuI2SrvSzwZeU37qCuACSrcsvhp4AHgM+IvMvK7O290FnJWZXxvnI38JfAr4RkT898x03J8kVbB1RJLmlsuB/wXcXrHLfD3wRODVwI8yM4F/Az4cEasAImJJRLwkIvYqH/NR4H0RcXiULIyIoyLigMoPy8xvAicAX42INcX/epI0exi0JWlu+RHwJ5R2sQHIzEFKO9l/Un4eStNIvg18OyIeAW4HTqf8vwuZ+e/Ah4AvAA8BdwPvpdReMkpm/pDShZXnRcSJhfxWkjQLRWljQ5IkSVIzuaMtSZIkFcCgLUmSJBXAoC1JkiQVwKAtSZIkFcCgLUmSJBXAoC1JkiQVwKAtSZIkFcCgLUmSJBXAoC1JkiQV4P8De19eP146xM4AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plot_weekly_accept_rate(df, 'Accepted', 'Rejected', **{'showOutlier':True, 'hue':'pyDirection'})" + "# plot_weekly_accept_rate(\n", + "# df, \"Accepted\", \"Rejected\", **{\"showOutlier\": True, \"hue\": \"pyDirection\"}\n", + "# )\n", + "# Same" ] }, { @@ -621,24 +143,20 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAGACAYAAACTE8U/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABQzUlEQVR4nO3ddXicVfrG8e8TaeqapO6SeguE4tCixWWhwA+XBRZZFvddWFwKC7vL7iLFF7ZFFndaitWpUXdvUvfo8/tjpmwokYlM3szk/lzXXDN5Zd47Ew59cnLec8zdERERERGRqpUQdAARERERkXikQltEREREJApUaIuIiIiIRIEKbRERERGRKFChLSIiIiISBSq0RURERESiQIW2iEg1M7OPzeyCUvb/08zuqs5MFWFmY8zs0qBzBMnM7jazV4POISI1kwptEYk7ZrbEzHaa2TYzW2tmL5pZw6Bz7ebux7r7SwBmdqGZfbvH/ivc/d5o5wh/Lvftsa2TmbmZJYW/jupnGf7+C8Lvv8XMpprZCVX1/iIiQVKhLSLx6kR3bwjsDWQCd+55wO5iUspU5mdZST+E378p8Dww0syalecN9LMUkZpIhbaIxDV3Xwl8DPQFCPfWXmVm84H54W2/NbMFZrbBzN4zsza7zw8f/3szW2Rm68zsUTNLCO9LMLM7zWypmWWZ2ctm1iS8r66ZvWpm681sk5lNNLOW4X1jzOxSM+sF/BM4INyjuym8/xc9zRHku8LM5oev83czs+r4LIvacwhFMT3jF4Y/w61mttjMzinm/QuBEUA9oKuZpZjZY2a2LNyb/k8zqxd+v8FmtsLMbjGzNcALZpZqZh+EP4cNZvZNkZ9Vr/DnvsnMfjKzk4pkfTH8uX0YzjfezLoW2f+kmS0P97hPNrNDquozFZH4pkJbROKambUHjgN+LLL5FGA/oLeZHQ48CAwDWgNLgTf2eJtTCfXk7g2cDFwc3n5h+DEE6AI0BP4W3ncB0ARoD7QArgB2Fn1Td58d3v6Duzd096bF5I8k3wnAvkD/8HHHhM/tEC4sOxT74ZRTCZ9lJOc1AJ4CjnX3RsCBwNRijksCLgW2Efol6CGgBzAQ6Aa0Bf5Y5JRWQHOgI3AZcAOwAkgDWgK3A25mycD7wGdAOnAN8JqZZRR5r7OAe4BmwALg/iL7JoYzNAf+DYwys7rl+QxEpHZSoS0i8eq/4R7ib4GvgQeK7HvQ3Te4+07gHGCEu09x9xzgNkI9zJ2KHP9w+PhlwF+As8PbzwEed/dF7r4tfO5Z4YIxj1CB3c3dC9x9srtvqcD3EUm+h9x9UzjfaEJFIe6+zN2bhrdXRmmfZaQKgb5mVs/dV7v7T0X27R9+/zWEPttTgS2Eiufrwp/91vB1z9rjPf/k7jnhn2UeoV9GOrp7nrt/4+4O7E/ol6CH3D3X3b8CPuB/P0eAd9x9grvnA68R/gwB3P1Vd1/v7vnuPhxIAYoW6SIixVKhLSLx6pRwkdnR3a8MF2K7LS/yug2hXmIAwgXzekK9p8UdvzR8zq/ODb9OItSb+grwKfCGma0ys0fCPavlFUm+NUVe7yBUVEYiH9gzUzKhArawyLbSPssyuft24ExCvferw0M0ehY5ZFz4/VPdfX93/4JQr3R9YHK4V34T8El4+27Z7r6ryNePEuqN/iw8TOXW8PY2wPLw0JTdlhLhZ2hmN5rZbDPbHM7RBEgtz2cgIrWTCm0RqY28yOtVhIYeAD8Pc2gBrCxyTPsirzuEz/nVueF9+cDacI/qPe7em9BQiROA88vIUpxI8lXUMqDTHts68+uiNBLbCRXGu7UqutPdP3X3owj1OM8Bni3j/dYRGmrTJ1yEN3X3JuGbJn9+2z2usdXdb3D3LsBJwPVmdgShz7D97vHaYR2I4DMMj8e+mdCQnGbh4T2bgaiMgxeR+KJCW0Rqu9eBi8xsoJmlEBqeMN7dlxQ55iYzaxYeo3wt8J8i515nZp0tNOXdA8B/3D3fzIaYWT8zSyQ0DCKPX/YS77YWaGdmdSqRr6LeAo43s6PNLDF8k+Wd/HoMeCSmAoeGx4U3ITTEBQAza2lmJ4d/ScghNAa71EI+XOg/CzxhZunh92lrZseUdI6ZnWBm3cI3g24GCsLXGU+ol/pmM0s2s8HAiRF+n40I/fKUDSSZ2R+BxhGcJyKiQltEarfwMIW7CBWdq4Gu/HIcMMC7wGRCxeSHhKagg9AMGa8AY4HFwC5CN9pBqEf3TUJF9mxCY5tfKSbCV8BPwBozW1fBfMUKF73bSroZMjxO+mxCN1tuAH4gVJTeE8n77/FenxP6BWQ6oc/qgyK7E4DrCfUsbwAOA34XwdveQmgoyDgz2wJ8Qeljo7uHj9lG6Ht52t1Hu3suocL6WEI95U8D57v7nAgyfEpoyMo8QsNNdvHLoUQiIiWy0H0iIiJSHDNzoLu7Lwg6i4iIxBb1aIuIiIiIRIEKbRERERGRKNDQERERERGRKFCPtoiIiIhIFKjQFhERERGJgqSgA0RLamqqd+rUKegYIiIiIhLHJk+evM7d04rbF7eFdqdOnZg0aVLQMUREREQkjpnZ0pL2aeiIiIiIiEgUqNAWEREREYkCFdoiIiIiIlGgQltEREREJApUaIuIiIiIRIEKbRERERGRKFChLSIiIiISBSq0RURERESiQIW2iIiIiEgUqNCuQsvW72Dq8k1BxxARERGRGkCFdhVxd6769xSuem0KW3flBR1HRERERAKmQruKmBn3nNyH1Zt3cu8Hs4KOIyIiIiIBU6Fdhfbu0IwrDuvKyEkr+HL22qDjiIiIiEiAVGhXsWuP7E7PVo249e0ZbNyeG3QcEREREQmICu0qlpKUyOPDBrJpRy53vTsz6DgiIiIiEhAV2lHQu01j/nBkDz6Yvpr3p60KOo6IiIiIBECFdpRcfmgXBrZvyl3vziRry66g44iIiIhINVOhHSVJiQkMHzaAnbkF3Pb2DNw96EgiIiIiUo1UaEdR17SG3DK0J1/OyWLUpBVBxxERERGRaqRCO8ouPLAT+3dpzp8/mMWKjTuCjiMiIiIi1USFdpQlJBiPnj4Ad+emUdMpLNQQEhEREZHaoFoKbTMbYWZZZjazyLa7zWylmU0NP44r4dzrzOwnM5tpZq+bWd3qyFyV2jevz10n9OaHRet5+YclQccRERERkWpQXT3aLwJDi9n+hLsPDD8+2nOnmbUFfg9kuntfIBE4K6pJo+TMfdszJCONhz6Zw6LsbUHHEREREZEoq5ZC293HAhsqeHoSUM/MkoD6QExOTG1mPPSb/qQkJXLDqGnkFxQGHUlEREREoijoMdpXm9n08NCSZnvudPeVwGPAMmA1sNndP6vukFWlZeO63HtKX35ctolnvlkUdBwRERERiaIgC+1/AF2BgYSK6OF7HhAuvk8GOgNtgAZmdm5Jb2hml5nZJDOblJ2dHZXQlXVi/9Yc3681T3w+j9mrtwQdR0RERESiJLBC293XunuBuxcCzwKDijnsSGCxu2e7ex7wNnBgKe/5jLtnuntmWlpadIJXkplx7yl9aVKvDtePnEZuvoaQiIiIiMSjwAptM2td5MtTgZnFHLYM2N/M6puZAUcAs6sjXzQ1b1CHB0/rx+zVW3jqy/lBxxERERGRKKiu6f1eB34AMsxshZldAjxiZjPMbDowBLgufGwbM/sIwN3HA28CU4AZ4bzPVEfmaDuqd0tO36cdT49ZwI/LNgYdR0RERESqmLnH5wIqmZmZPmnSpKBjlGrLrjyGPjGWunUS+ej3h1A3OTHoSCIiIiJSDmY22d0zi9sX9KwjtVrjusk8esYAFmVv55FP5gYdR0RERESqkArtgB3ULZULDujIiO8W88PC9UHHEREREZEqokK7Brjl2J50alGfG0dNY1tOftBxRERERKQKqNCuAerXSWL4sAGs3ryT+z+cFXQcEREREakCKrRriH06NueyQ7vy+oTljJ6TFXQcEREREakkFdo1yHVHdSejZSNueWs6m3bkBh1HRERERCpBhXYNkpKUyPBhA9iwPZc/vvtT0HFEREREpBJUaNcwfds24fdHdOe9aav4cPrqoOOIiIiISAWp0K6BrhzclQHtmnDnf2eQtXVX0HFEREREpAJUaNdASYkJDB82gO25Bdz+9kzidfVOERERkXimQruG6pbeiJuPyeCL2Wt5c/KKoOOIiIiISDmp0K7BLj6oM4M6N+fP789i5aadQccRERERkXJQoV2DJSQYw88YQIE7N785jcJCDSERERERiRURF9pm1sXMMsPPFs1Q8j/tm9fnzuN7892C9bw6fmnQcUREREQkQqUW2ma2t5m9ZGbrgQXAhPDz+vD2vasjZG139qD2HNYjjQc/msPidduDjiMiIiIiESix0Dazl4F3gOXAaUAqUCf8fAqwDHg7fJxEkZnx8G/6k5xo3DhqGgUaQiIiIiJS45XWo/0d0MXd73T3r919g7vnh5/HuvtdQFfg2+qJWru1alKXe0/py+SlG3n2m0VBxxERERGRMpRYaLv7v9y9oLST3b3A3Z+p+lhSnJMGtOHYvq14/LN5zFmzJeg4IiIiIlKKcs86YmZHmNmNZnZKFPJIKcyM+07pS+N6Sdwwchq5+YVBRxIRERGREpSr0Daz24E/Ai2AW81MvdnVrEXDFO4/tR8/rdrC376aH3QcERERESlBmbOO7LHpaGCwu98GDAFOj1YwKdkxfVpx2t5t+fuYhUxbvinoOCIiIiJSjLJ6tJ81swfMrE7462zgPDPrDvwWWBLNcFKyP53Yh/RGKVw/ciq78kodSi8iIiIiASir0N4PyAEmmtkBwO+BocB/gSOAc6KaTkrUpF4yj5zen4XZ23ns07lBxxERERGRPSSVttPd84F7zOwt4FlCC9Zc6u47qiOclO6Q7mmct39Hnv9uMUf1bsl+XVoEHUlEREREwiK6GdLdZwIHEVq8ZqKZHRHVVBKx247rSYfm9bnxzWlsy8kPOo6IiIiIhJV1M+QBZjbNzLYRWpjmA+Bk4E4ze87MGldHSClZ/TpJDD9jACs27uSBj2YHHUdEREREwsrq0X4BuJvQdH7DgSfdfYG7DwEmA+OjG08ikdmpOZcd0oV/j1/GmLlZQccREREREcoutFOBL9w9B/gKSNu9w93/ARwVxWxSDtcd1YMeLRtyy1vT2bwjL+g4IiIiIrVeWYX2P4HJZvYaoRsh/1l0p7uviFYwKZ+6yYkMP2Mg67fl8qf3ZgYdR0RERKTWK7XQdvc7gTMJTed3krtXaCVIMxthZllmNrPItrvNbKWZTQ0/jivmvIwi+6ea2RYz+0NFMtQG/do14erDu/Hfqav4eMbqoOOIiIiI1Gplzjri7j+6+yh3r8yddi8Smn97T0+4+8Dw46Nirj13935gH2AH8E4lcsS9q4Z0o1/bJtzx35lkb80JOo6IiIhIrVVioW1m95hZg9JONrOGZnZPWRdx97HAhgrkK+oIYKG7L63k+8S15MQEHh82gG05+dzxzgzcPehIIiIiIrVSaT3aKcBiM3vWzIaZWV8z6xB+HmZmzwCLgORKXP9qM5seHlrSrIxjzwJer8S1ao3uLRtx09EZfDZrLW9PWRl0HBEREZFaqcRC291vBTKBbEJT/E0HFoef7wHWA5nufnsFr/0PoCswEFhNaPrAYplZHeAkYFRpb2hml5nZJDOblJ2dXcFY8eHigzszqFNz7n7/J1Zt2hl0HBEREZFap6ybIZe5++3u3huoD7QD6rt7L3e/zd2XVfTC7r7W3QvcvZDQ8u6DSjn8WGCKu68t4z2fcfdMd89MS0sr7dC4l5hgPHpGfwoKnVvemq4hJCIiIiLVLKIl2AHcfZe7r3b3XVVxYTNrXeTLU4HS5qQ7Gw0bKbeOLRpw+3G9+Gb+Ol4dX+HfiURERESkAiIutCvDzF4HfgAyzGyFmV0CPGJmM8xsOjAEuC58bBsz+6jIuQ0ILYzzdnVkjTfn7NeBQ7qn8sCHs1m6fnvQcURERERqDYvXIQWZmZk+adKkoGPUCKs37+ToJ8aS0bIR/7n8ABITLOhIIiIiInHBzCa7e2Zx+6qlR1uC1bpJPe45qQ+Tlm7k+W8XBR1HREREpFZQoV1LnLpXW47p05LHPp3HvLVbg44jIiIiEvciLrTN7Fwz+zw8phozO9TMToteNKlKZsb9p/ajUd0krh85lbyCwqAjiYiIiMS1iAptM7ue0NzZHwMdwpuzgZujlEuiILVhCvef2peZK7fwt68WBB1HREREJK5F2qP9O+BYd38c2H335DygW1RSSdQM7duaU/dqy99GL2DGis1BxxERERGJW5EW2s3dfV749e5C24q8lhhy94l9SGuYwvUjp7IrryDoOCIiIiJxKdJCe5aZnbDHtqHAtCrOI9WgSf1kHj69P/OztvH45/PKPkFEREREyi3SQvt24N9m9hyQYmZ/BV4A7ohaMomqw3qk8X/7deDZbxYxccmGoOOIiIiIxJ2ICm13/wbYH9gJjA6fN9jdx0cxm0TZHcf1ol2zetwwchrbc/KDjiMiIiISVyKddaSTu89y92vc/QR3v8rdfzKzjtEOKNHTICWJ4WcMZPnGHTz48eyg44iIiIjElUiHjkwvYfuPVRVEgjGoc3MuPbgzr45bxth52UHHEREREYkbkRba9qsNZslo1pG4cMPRGXRLb8jNb05n8868oOOIiIiIxIVSC+3wSpCfEboB8rOiD2AGMKVaUkpU1U1O5PFhA8jelsM97/0UdBwRERGRuJBUxv5vw8+HAd8V2V4IrAFGRSOUVL/+7Zpy1ZBuPPXlfI7u04qhfVsFHUlEREQkppVaaLv7PQBmNtvdR1ZPJAnKNYd348vZa7njnRlkdmpGasOUoCOJiIiIxKxIp/cbCWBmdc2svZl12P2IbjypTsmJCTw+bCBbd+Vz5zszcdcQfBEREZGKinR6vy5m9j2wHVgCLC7ykDiS0aoR1x/dg09+WsO7U1cFHUdEREQkZkU668jfgOXAAGAr0B/4L3BJdGJJkH57SBf26diMP747kzWbdwUdR0RERCQmRVpo7wdc6u4zAdz9J+By4KZoBZPgJCYYw88YQF6Bc/Nb0zWERERERKQCIi20Cwktvw6wzcyaAhsAjdGOU51SG3D7cT0ZOy+bf09YFnQcERERkZgTaaH9E3BQ+PV44AngKTRGO66ds19HDu6Wyv0fzmbZ+h1BxxERERGJKZEW2r8n1IMNoeEibYFMQsNHJE4lJBiPnN6fRDNuHDWNgkINIRERERGJVKTT+0139xnh14vc/Wh33x+YFdV0Erg2Tevxp5P6MGHJBl74Tn/AEBEREYlUWStDFsvM6hLq5b4FaFGliaTG+c3ebfn0pzU88ulcDuuRRveWjYKOJCIiEpHJSzcycckG6iYlkJKcSN3kBOomJZJS5DklKbQ9Zff25ETqJiWSnGiYWdDfgsSwUgttM+sEjCA0TGQOcC7QGXgO2EGo2JY4Z2Y8cGo/jvnLWG4YNY23fncgyYmRjjoSEREJxth52Vzy0kTyCio29NGMXxXlpRXpxT3XTU4kJenXzylFvt7zPBX48aOsHu3Hw8+3AmcCo4DmwF3Ay+5eGMVsUoOkNUrhvlP6cuVrU3h69EKuPbJ70JFERERKNHnpBi5/ZTLd0hvx0kX7kpSYwK68AnLyC0t/juCYXXmF5OQXsHVXPuvyc4s9J78S9zUlGCUW7CklFO7FFex7PqeUUvjXTU4kKUEFflUrq9A+EOjj7uvN7G1gFZDp7lOiH01qmuP6tebkgW3461fzOaJXOn3bNgk6koiIyK/MWrWFi16YSKsmdXn54kGkNUqp9gz5BYURF+w54eddxTwXd97WXflkb80hd8/3yy+s1MQFCUa4YC+59/3noj4pofie/l8cW8JfA/Z4n6Q4/it5WYV2fXdfD+Dua8xsm4rs2u2ek/rww8L1XD9yKu9fczApSYlBRxIREfnZ4nXbOX/EeBqkJPHKJcEU2QBJiaECskFKhW6Hq7D8gkJ25ReSEy68c/L2KNyLFPY5Rb7+ZcG+e98v32fzzrxie+935RVQmYnJEhPsf2PoixT2u78urRe+aG/+wPbN6NeuZnUClvXTNzNrDez+O0LBHl/j7quiFU5qnqb16/Dw6f256IWJPP75PG47tlfQkURERABYtWkn5z43nkKHVy7Zj3bN6gcdqdolJSbQMDGBhtVc4OcV04O/Z2988UV6cb35vzxm047cEv8qULTAv/HoHjFXaDcAVhT52op8bYAD6tKsZYZkpHP2oPY8M3YRR/duyT4dmwcdSUREarn123I49/nxbNmZx+uX7U+39IZBR6pVkhMTSK7mAt/dyS/0nwv2usk1bwhKWYk6A12KPDoX87pMZjbCzLLMbGaRbXeb2Uozmxp+HFfCuU3N7E0zm2Nms83sgEiuKdF1x/G9adu0HtePnMaO3Pyg44iISC22ZVceF7wwgZUbd/L8hfvqHqJawsxITkygUd1k0hql0KhuctCRfqXUQtvdl5b1iPA6LwJDi9n+hLsPDD8+KuHcJ4FP3L0nMACYHeE1JYoapiTx2BkDWLZhBw99PCfoOCIiUkvtzC3g0hcnMWf1Vv557j4M6qy/skrNUS197O4+lv8t4R4xM2sCHAo8H36fXHffVLXppKL279KCiw/qzMs/LOXb+euCjiMiIrVMbn4hV742mYlLN/DEmQMZ0jM96EgivxD0YJarzWx6eGhJs2L2dwaygRfM7Ecze87MGlRzRinFTcdk0DWtATe9OY0tu/KCjiMiIrVEQaFz/cipjJ6bzf2n9OPEAW2CjiTyK0EW2v8AugIDgdXA8GKOSQL2Bv7h7nsB2wktnlMsM7vMzCaZ2aTs7OyqTyy/Ujc5keHDBpK1NYd73psVdBwREakF3J07/zuTD6av5tZje/J/+3UIOpJIsSIqtMNLsRe3vWNFL+zua929ILy65LPAoGIOWwGscPfx4a/fJFR4l/Sez7h7prtnpqWlVTSalNPA9k25cnBX3pqygs9+WhN0HBERiXMPfzKX1ycs48rBXbnisK5BxxEpUaQ92tNL2P5jRS8cno97t1OBmXse4+5rgOVmlhHedASgbtMa6JrDu9O7dWNuf2cG67flBB1HRETi1NNjFvDPrxdy7v4duOmYjLJPEAlQpIX2rxa+N7NkQvNol32y2evAD0CGma0ws0uAR8xshplNB4YA14WPbWNmRWcguQZ4LXzcQOCBCDNLNaqTlMDjZw5g88487vzvTNwrsUSUiIhIMV4dt5RHPpnLSQPa8OeT+mL2q/JEpEYpdVZxM/ucUDGdYmaf7bG7AxDRcuzufnYxm58v4dhVwHFFvp4KZEZyHQlWz1aNue6oHjzyyVzem7aKkwe2DTqSiIjEiXenruSud2dyeM90hg8bQEKCimyp+cpavufb8PNhwHdFthcCa4BR0QglsevyQ7vy+ay1/PHdn9i/SwtaNq4bdCQREYlxX81Zyw0jpzGoU3OePmdvkhODnjRNJDKlFtrufg+Amc1295HVE0liWWKC8fiwgRz75FhueWs6L1y4r/60JyIiFTZu0Xp+9+oUerVuzHMXZFI3OTHoSCIRi+hXQncfaWYNzewsM7sx/Nwo2uEkNnVObcBtx/ZizNxs3pi4POg4IiISo6av2MSlL02iffP6vHTxoBq5xLZIaSKd3i8TWAQ8DJwcfl4Y3i7yK+ft35EDu7bgvg9msXzDjqDjiIhIjFmQtZULRkygSb1kXrlkEM0b1Ak6kki5RTrI6WlguLt3dPdD3L0j8BihRWdEfiUhwXj0jAGYGTeOmkZhoWYhERGRyCzfsINzn5tAYkICr126H62b1As6kkiFRFpo9+LXKzc+DvSs2jgST9o2rccfT+zN+MUbeOH7JUHHERGRGJC1dRfnPj+eHbn5vHLJIDqlNgg6kkiFRVpoTwX67rGtX3i7SInO2KcdR/RM55FP5rAga1vQcUREpAbbvCOP85+fQPbWHF68eBC9WjcOOpJIpURaaH8GfGBmd5vZRWZ2D/Ae8JmZ/d/uR/RiSqwyMx78TT/q1UnkhpFTyS8oDDqSiIjUQNtz8rnwxQksyt7OM+dlsneHZkFHEqm0subR3u1iIA+4oMi2/PD23Rz4dxXlkjiS3qgu953Sl6v//SP//HohVx/ePehIIiJSg+TkF3DFq5OZtnwTT5+zDwd3Tw06kkiViKjQdvfO0Q4i8e2E/m349Ke1PPnlfIb0TKdPmyZBRxIRkRogv6CQa1+fyjfz1/HYGQMY2rdV0JFEqky5llYyszZmtn+0wkh8+/NJfWhavw43jJxGTn5B0HFERCRghYXOrW/P4JOf1vDHE3pz+j7tgo4kUqUinUc73cy+AFYAX4S3nWlmT0cznMSXZg3q8PBv+jFnzVb+8sX8oOOIiEiA3J17P5zFm5NX8Icju3PxwfrjucSfSHu0nwIWA2mExmoDfAUcFY1QEr8O79mSMzPb86+vFzJ56cag44iISECe/HI+L3y3hIsO6sS1R+jeHYlPkRbaQ4Br3H09oZsecfdsID1awSR+3XlCL1o3qceNo6axIzc/6DgiIlLNRny7mL98MZ/T92nHXcf3xsyCjiQSFZEW2jnsceOkmTUHNlR5Iol7jeom8+gZ/Vm8bjuPfDI36DgiIlKNRk1azp8/mMXQPq146LR+JCSoyJb4VZ55tIebWXKRbfcAH1Z9JKkNDuyayoUHduLF75fw3YJ1QccREZFq8MnMNdzy1nQO7pbKk2cPJCmxXHMyiMScSP8Lv5nQMuwbgcZmtgnoD9wZpVxSC9wytCddUhtw85vT2bIrr+wTREQkZn07fx2/f/1HBrRvyr/O24eUpMSgI4lEXUSFtrtvcPdDgcOAswjdBDnY3TdFMZvEuXp1Enls2ABWb97Jve/PCjqOiIhEyZRlG7nslUl0SWvAixcOokFKpOvlicS2SKf3a2ZmDdx9sru/6e4Tgfpm1jS68STe7d2hGb8b3JVRk1fwxay1QccREZEqNnv1Fi4cMYG0Rim8fMkgmtRPLvskkTgR6dCR94A+e2zrC7xbtXGkNvr9Ed3p2aoRt749g43bc4OOIyIiVWTJuu2c9/wE6tdJ4tVL9iO9Ud2gI4lUq0gL7T7ApD22TQL6VW0cqY1SkhJ5fNhANu/M5c53ZwYdR0REqsCazbs457nxFBQW8uqlg2jfvH7QkUSqXaSF9i5gzxbSgP8tXiNSKb3bNOYPR/bgw+mreX/aqqDjiIhIJWzYnsu5z49n8848Xrp4EN3SGwUdSSQQkRba3wIPmFkCgIVmlv8z8F20gkntc/mhXRjYvil3vTuTrC27go4jIiIVsHVXHheMmMDyDTt47oJM+rdrGnQkkcBEWmjfBBwPLDezb4DlwEnA9dEKJrVPUmICw4cNYGduAbe+PQN3DzqSiIiUw668Ai59aRKzV2/hH+fuzf5dWgQdSSRQkU7vt5TQzY9/IHRj5HVAX3dfErVkUit1TWvILUN78tWcLEZOWh50HBERiVBeQSFXvjaFCUs2MHzYAA7v2TLoSCKBi3giS3ffCYyKYhYRAC48sBOfzVrDn9+fxYFdU3UDjYhIDVdY6Nw4ahpfzcnivlP6cvLAtkFHEqkRIp1H+zMzO3yPbUeY2cfRiSW1WUKC8ejpAzAzbnpzGoWFGkIiIlJTuTt/fG8m705dxc1DMzh3/45BRxKpMSIdo703MHaPbd8A+1ZtHJGQ9s3rc9cJvRi3aAMv/bAk6DgiIlKCRz+dy6vjlnH5YV24cnC3oOOI1CiRFtqFwJ5LOSUCVrVxRP5nWGZ7Du+ZzkMfz2Fh9rag44iIyB7++fVCnh6zkLMHdeDWoT2DjiNS40RaaE8Grtlj29XAlEhONrMRZpZlZjOLbLvbzFaa2dTw47gSzl1iZjPCx+y5aI7EMTPjodP6UTc5kRtGTiO/oDDoSCIiEvbv8ct46OM5nNC/Nfed0pfQzL8iUlSkhfYtwK1mNt7MXjGzccDtwI0Rnv8iMLSY7U+4+8Dw46NSzh8SPiYzwutJnEhvXJd7T+nL1OWb+NfYRUHHERER4P1pq7jjvzMYnJHG48MGkpigIlukOJFO7zcd6A28CWwB3gJ6u/u0CM8fC2yoaEip3U4a0Ibj+7fmL1/MY9aqLUHHERGp1UbPyeK6/0xl347N+cc5+1AnKdI+O5HaJ+LW4e5r3P1Rd7/K3R8FmpvZU5W8/tVmNj08tKRZSZcGPjOzyWZ2WSWvJzHq3pP70qReHa4fOZXcfA0hEREJwoTFG7ji1cn0bN2I5y7MpF6dxKAjidRo5fo11MxSzOx8M/sOmEFoNpKK+gfQFRgIrAaGl3Dcwe6+N3AscJWZHVpKvsvMbJKZTcrOzq5ENKlpmjeow0On9WPOmq08+eW8oOOIiNQ6M1du5pIXJ9KuWT1eumgQjevuOUeCiOwp0nm0e5vZk8Aq4F9AJjDU3Q+u6IXdfa27F7h7IfAsMKiE41aGn7OAd0o6LnzMM+6e6e6ZaWlpFY0mNdSRvVtyxj7t+MeYhfy4bGPQcUREao0FWds4f8QEGtdL5pVL9qNFw5SgI4nEhFILbTM7z8y+AWYChwF3A20JjbeeWpkLm1nrIl+eGr7Gnsc0MLNGu18DRxd3nNQed53Ym9ZN6nHDyGnszC0IOo6ISNxbsXEH5z0/ngSDVy/djzZN6wUdSSRmlNWj/RLQEzg+POvHX9293Dc1mtnrwA9AhpmtMLNLgEfC0/ZNB4YA14WPbWNmu2cgaQl8a2bTgAnAh+7+SXmvL/Gjcd1kHjm9P4vWbeeRT+cEHUdEJK5lb83h3OfGsy0nn5cv3o/OqQ2CjiQSU5LK2H8XcCnw33DxOwL4sLwXcfezi9n8fAnHrgKOC79eBAwo7/Ukvh3ULZULDujIC98t4ajeLTmwa2rQkURE4s7mnXmcP2ICa7fk8Oqlg+jdpnHQkURiTqk92u5+P9AFOIXQ7B9vASuBpkCbKGcTKdGtx/aic2oDbho1na278oKOIyISV3bk5nPxixNZkLWVf523D/t0bB50JJGYVObNkB7ysbufBnQEngbWABPNbGS0A4oUp16dRB47YwCrN+/k/g9nBx1HRCRu5OQXcPkrk/lx2UaeOmsvDu2hyQVEKqpc0/u5+2p3v5dQL/fJQJ2opBKJwD4dm3H5YV15Y+JyvpqzNug4IiIxL7+gkD+8MZVv5q/jodP6c2y/1mWfJCIlqtByTuFe7o/c/ZQqziNSLn84sjs9WzXilrdmsHF7btBxRERilrtz+zsz+HjmGu48vhfD9m0fdCSRmKd1UyWmpSQlMnzYADZuz+WP7/0UdBwRkZjk7tz/4WxGTlrB7w/vxqWHdAk6kkhcUKEtMa9PmyZce0R33p+2ig+mrwo6johIzPnbVwt47tvFXHhgJ647qkfQcUTihgptiQu/G9yVAe2acNd/Z5K1dVfQcUREYsaL3y1m+OfzOG3vtvzxhN6YWdCRROJGuQptC9GdEVLjJCUmMHzYQHbkFnDbWzNw96AjiYjUeG9NXsHd78/i6N4teeQ3/UlIUJEtUpUiKrTNrKGZPQ/sBBaEt51iZn+KZjiR8uiW3pCbjsngyzlZjJq8Iug4IiI12mc/reHmt6ZzYNcWPHX2XiQl6o/cIlUt0lY1nNBy6AcBu6d2mAicGY1QIhV18UGd2a9zc/78/ixWbNwRdBwRkRrpuwXruPrfP9K3bROeOT+TusmJQUcSiUuRFtonAOe4+2RCK0Ti7ivR6pBSwyQkGI+dMQB35+Y3p1NYqCEkIiJF/bhsI799eRKdUxvw0kX70jAlKehIInEr0kI7gdCwkZ+ZWUNgW5UnEqmk9s3rc+cJvfl+4XpeGbc06DgiIjXG3DVbufCFiaQ2TOGVSwbRtL7WnROJpkgL7W+B2/bYdg0wumrjiFSNs/Ztz+CMNB78eDaL120POo6ISOCWrt/Oec+Pp25yAq9duh/pjesGHUkk7kVaaF8PnGtm84GGZjYDuBC4NVrBRCrDzHj4N/1JSUrkhpFTKdAQEhGpxdZu2cW5z48nt6CQVy7Zj/bN6wcdSaRWiKjQdvflQF9Cvdq3A38GBobHaYvUSC0b1+XPJ/dhyrJNPDN2UdBxREQCsXF7Luc+N54N23J56aJB9GjZKOhIIrVGRHdAmFkHd18GvBnlPCJV6qQBbfhk5hqe+HweQ3qm0bNV46AjiYhUm205+Vz4wgSWbtjBixfty4D2TYOOJFKrRDp0ZJGZfW5mZ5lZSlQTiVQhM+O+U/rSuF4S1/9nGrn5hUFHEhGpFrvyCvjtS5OYuWoLT//f3hzYNTXoSCK1TqSFdnfge+BBYLWZPW1mmdGLJVJ1WjRM4YFT+zFr9Rb++tX8oOOIiERdXkEhV//7R8YtXs/wMwZwZO+WQUcSqZUiHaO92N3/5O6dgTOAhsBoM5sW1XQiVeToPq34zd7teHrMQqYu3xR0HBGRqCksDK0j8MXstfz5pD6cslfboCOJ1FoVWW91DPAOMInQDZIiMeGPJ/YmvVEKN4ycyq68gqDjiIhUOXfn7vd/4p0fV3Lj0T0474BOQUcSqdUiLrTNrL+ZPQGsAp4EvgMyohVMpKo1qZfMo6cPYGH2dh79dG7QcUREqtzwz+bx8g9LuezQLlw1pFvQcURqvYgKbTP7ERgHtAbOBzq5+53uviCa4USq2sHdUzlv/46M+G4x4xatDzqOiEiVeXbsIv42egFn7due247tiZkFHUmk1ou0R/tZoI27n+Xun7q7pm6QmHXbcT3p0Lw+N46axrac/KDjiIhU2hsTlnH/R7M5vl9r7j+1n4pskRoi0pshn3b3TVHOIlIt6tdJYvgZA1i5aSf3fzg76DgiIpXy4fTV3PbODA7rkcYTZw4kMUFFtkhNUeKCNWb2rrufHH79WUnHufvR0QgmEk2ZnZpz2SFd+NfYRRzdpyVDMtKDjiQiUm5j5mbxh//8yD4dmvHPc/ehTlJF5jgQkWgpbWXIcUVefw94lLOIVKvrjurB6LlZ3PLmdD677lCa1q8TdCQRkYhNXLKBK16dTPf0Rjx/4b7Uq5MYdCQR2YO5x2f9nJmZ6ZMmTQo6htRwM1du5pS/f8fx/Vvz5Fl7BR1HRCQiM1du5uxnxpHWKIWRVxxAakMt2iwSFDOb7O7FLuQY6awjxQ5kNbMZlQkmErS+bZtwzeHdeXfqKj6asTroOCIiZVqYvY0LRkygUd0kXrl0PxXZIjVYpIO52pVzu0jMuHJIV/q1bcId78wge2tO0HFEREq0ctNOzntuPACvXrofbZvWCziRiJSmtDHamNntu48r8nq3bsDyqKQSqUbJiQk8PmwAx//1W257ewbPnr+PpsYSkRpn3bYczntuPFtz8nn9t/vTJa1h0JFEpAxl9WgfFX4kF3l9FHAEkAJcHMlFzGyEmWWZ2cwi2+42s5VmNjX8OK6U8xPN7Ecz+yCS64mUV/eWjbjp6Ay+mL2Wt6asDDqOiMgvbN6Zx/nPT2DV5p28cOG+9G3bJOhIIhKBUnu03X0IgJn91d2vqcR1XgT+Bry8x/Yn3P2xCM6/FpgNNK5EBpFSXXxwZz6ftZZ73vuJA7u2oI3+JCsiNcDO3AIueXEi87O28uz5mWR2ah50JBGJUKQL1lSmyMbdxwIbKnKumbUDjgeeq0wGkbIkJhiPntGfAndufnM68Tojj4jEjtz8Qq54dTJTlm3kL2fuxWDN+S8SUyKddaSemd1vZuPMbKGZLdr9qOT1rzaz6eGhJc1KOOYvwM2Aln2XqOvYogG3H9eLbxes49VxS4OOIyK1WEGhc91/pvL1vGweOLUfx/dvHXQkESmnSGcdeQI4GXgFaAkMB3KAEZW49j+ArsBAYHX4PX/BzE4Astx9ciRvaGaXmdkkM5uUnZ1diWhSm52zXwcO6Z7KAx/NYcm67UHHEZFayN25450ZfDhjNXcc14uzBnUIOpKIVECkhfaJwEnu/ncgP/z8G2BIRS/s7mvdvcDdC4FngUHFHHYQcJKZLQHeAA43s1dLec9n3D3T3TPT0tIqGk1qOTPjkdP7k5Ro3DhqGgWFGkIiItXH3Xnw4zm8MXE5Vw/pxm8P7RJ0JBGpoEgL7YbuvnuYSK6Z1XH3WcC+Fb2wmRX9G9ipwMw9j3H329y9nbt3As4CvnL3cyt6TZFItW5Sjz+f3IdJSzfy3DeVHSElIhK5p8cs5Jmxizj/gI7ccHSPoOOISCVEWmgvNrNe4ddzgIvN7CxgcyQnm9nrwA9AhpmtMLNLgEfMbIaZTSfUM35d+Ng2ZvZRub4LkSg4ZWBbjunTkuGfzWPumq1BxxGRWuDlH5bw6KdzOXWvttx9Yh/N6S8S4yySmRXM7Exgk7t/amZHAe8AdYDfufvzUc5YIZmZmT5p0qSgY0iMW7cth2OeGEurJnX571UHkZwY6e+mIiLl898fV/KH/0zlyF7p/OPcffT/G5EYYWaT3T2zuH2RTu/3H3f/NPz6c6AZ0KymFtkiVSW1YQr3n9qPn1Zt4a9fLQg6jojEqS9mreWGUdM4oEsL/vZ/e6vIFokTFWrJ7p7n7pqOQWqFoX1bcdpebfn76AVMX7Ep6DgiEme+X7iOK/89hb5tGvPsBZnUTU4MOpKIVJESV4Y0s/lAmeNK3F13akjc+9OJffh+4XquHzmND645WP8QikiVmLZ8E799aRIdm9fnxYsG0TCl1AWbRSTGlNai76u2FCI1XJP6yTx8en8uGDGB4Z/N5Y7jewcdSURi3Ly1W7nghQk0b1iHVy7Zj2YN6gQdSUSqWImFtru/VJ1BRGq6w3qkcc5+HXju28X0b9eUE/q31owAIlIhy9bv4NznxpOcmMCrl+xHqyZ1g44kIlEQ0d+ozOzAkva5+/dVF0ekZrv9uF78uGwT17z+I/8ev4w7ju9F37ZNgo4lIjEka8suzn1+PDn5hYy8/AA6tmgQdCQRiZJIp/crLGazA7h7jRysqun9JFryCgp5Y8IyHv98Hpt25vGbvdtx0zEZtGysHikRKd2mHbmc+a9xLN+4g9cu3Y+9OjQLOpKIVFJVTO+XUPQBtANeAs6owpwiMSE5MYHzDujEmJuGcNkhXXhv6ioGPzqGv3wxjx25+UHHE5EaaltOPhe8MJHF67bz7PmZKrJFaoGIerSLPdGsETDF3btXbaSqoR5tqS7L1u/g4U/m8OGM1bRsnMJNx/TktL3akpCg8dsiErIrr4CLX5zI+MUbePqcvTmmT6ugI4lIFal0j3YJUoD0SpwvEhc6tKjP38/ZmzevOIBWTepx46hpnPT3b/lh4fqgo4lIDZBfUMg1r//I9wvX88hv+qvIFqlFIh2jffsemxoApwCz3f30KOSqNPVoSxAKC533p6/i4Y/nsGrzLo7u3ZLbjutF51Td7CRSGxUWOjeOmsbbP67k7hN7c+FBnYOOJCJVrLQe7UgL7dF7bNoKTAaecPctlY9Y9VRoS5B25RXw/LeLeXr0AnLyCzn/gE78/ohuNK2veXJFagt35573Z/Hi90u4/qge/P6IGjnSUkQqqdKFdixSoS01QfbWHB7/fB7/mbiMRnWT+f0R3Tlv/47USarMqC0RiQWPfz6Pp76czyUHd+bO43tp3n2ROBWtMdoiUoa0Rik8eFo/Prr2EPq3a8K9H8zi6Ce+5tOf1hCvv+SKCDz3zSKe+nI+wzLbqcgWqcUiKrTNLMPMPjWz9WaWW/QR7YAi8aBnq8a8fPEgXrhoX5ISE7j8lcmc9cw4Zq7cHHQ0EaliIycu574PZ3Ns31Y8eFp/FdkitVhEK0MCrwJzgHOBHdGLIxK/zIwhGekc0i2V1ycu54nP53Hi377ltL1CC95oCWaR2PfRjNXc+vZ0Dumeyl/OGkiipvkUqdUivRlyC9DM3QuiH6lqaIy21HRbduXx99ELeOHbJSQkwGWHduXyQ7vQICXS339FpCYZOy+bS16aSP92TXnlkkHUr6O2LFIbVMUY7YlA16qLJCKN6yZz27G9+PKGwziyV0ue+nI+Qx4bw8hJyyko1PhtkVgyeekGLn9lMt3SGzHiwn1VZIsIEHmPdgfgOeBTYHXRfe7+7+hEqxz1aEusmbx0A/d+MJupyzfRu3Vj7jy+Fwd2Sw06loiUYdaqLZz5zA+kNkxh5OUHkNYoJehIIlKNqmIe7euAR4H1/HKMtrt7lypJWcVUaEsscnfen76ahz+ew8pNOzmyV0tuO64nXdMaBh1NRIqxeN12zvjn9yQnJjDqigNo16x+0JFEpJpVRaGdDZzn7p9UdbhoUaEtsWxXXgEjvlvM06MXsiuvgHP378i1R3SnWQMteCNSU6zatJMz/vkDO/MKGHn5AXRL1y/EIrVRVYzRdkLDRkSkGtRNTuTKwd0Yc9Ngzty3PS//sITDHh3Nc98sIje/MOh4IrXe+m05nPv8eLbszOPliwepyBaRYkVaaI8ALoxiDhEpRmrDFO4/tR8fX3soe3Voxn0fzuaoJ77mk5mrteCNSEC27Mrj/BETWLlxJ89fuC992zYJOpKI1FCRDh35AjgEmMevb4Y8OjrRKkdDRyQejZmbxQMfzWbe2m0M6tScO0/oRf92TYOOJVJr7Mwt4IIRE5iybCPPnp/JkJ7pQUcSkYCVNnQk0vmHvgk/RCRAgzPSObhbKv+ZtJzHP5vHSX/7jtP2astNQzNo3aRe0PFE4lpufiG/e20yE5du4Kmz9lKRLSJliqhHOxapR1vi3dZdeTw9ZiHPf7uYBIPLDunC5Yd11YI3IlFQUOhc+8aPfDB9NQ+c2o//269D0JFEpIaoillHDixpn7t/X4lsUaNCW2qL5Rt28Minc3l/2irSGqVw09EZ/Gafdlr6WaSKuDu3vzOT1ycs49Zje3LFYVq/TUT+pyoK7eKmOXAAd0+sXLzoUKEttc2UZRu574NZTFm2iZ6tGnHXCb05SAveiFTagx/P5l9fL+LKwV25eWjPoOOISA1T6en93D2h6ANoB7wEnFGFOUWkEvbu0Iy3fncgf/u/vdiWk885z43nkhcnsiBrW9DRRGLW02MW8K+vF3Hu/h246ZiMoOOISIyJdHq/X3D3VcC1wMORHG9mI8wsy8xmFtl2t5mtNLOp4cdxxZxX18wmmNk0M/vJzO6pSF6R2sLMOKF/G764/jBuPbYnExZv4Ji/jOVP785kw/bcoOOJxJRXxy3lkU/mctKANvz5pL6YaTiWiJRPhQrtsBQg0luuXwSGFrP9CXcfGH58VMz+HOBwdx8ADASGmtn+FQkrUpvUTU7kisO6MvqmwZw9qD2vjFvKYY+O5tmxi8jJLwg6nkiN9+7Uldz17kwO75nO8GEDSNA9DyJSARFNT2Bmt++xqQFwCvB5JOe7+1gz61SuZKHzHNj9d+/k8CM+p0kRiYLUhincd0o/zj+gEw98NJv7P5rNK+OWctuxPRnat5V66ESK8eXstdwwchqDOjXn6XP2JjmxMn1SIlKbRfp/j6P2ePQDRgIXV/L6V5vZ9PDQkmbFHWBmiWY2FcgCPnf38ZW8pkit06NlI168aBAvXzyIesmJ/O61KQz71w9MW74p6GgiNcq4Reu58rUp9GrdmOcuyKRuco28319EYkS1zaMd7tH+wN37hr9uCawj1EN9L9Da3Uss3M2sKfAOcI27zyzhmMuAywA6dOiwz9KlS6vyWxCJC/kFhYyavILhn81l3bZcThnYhpuG9qRtUy14I7Xb9BWb+L9nx9OqSV1GXn4AzRvUCTqSiMSACs86YmYtzWxYCfuGmVmFl8Vy97XuXuDuhcCzwKAyjt8EjKb4sd67j3nG3TPdPTMtLa2i0UTiWlJiAmcP6sCYm4Zw1ZCufDRzDYc/NobHPp3Ltpz8oOOJBGL+2q1cMGICTeol88olg1Rki0iVKGvoyC1A9xL2dQnvrxAza13ky1OBX/VSm1lauCcbM6tHaNjKnIpeU0T+p2FKEjcd05OvbjiMoX1b8bfRCxj86BjemLCMgkLdCiG1x/INOzjv+QkkJiTw2qX70bqJ/rojIlWjrEL7OOC5Eva9AJwQyUXM7HXgByDDzFaY2SXAI2Y2w8ymA0OA68LHtjGz3TOQtAZGh4+ZSGiM9geRXFNEItOuWX2ePGsv/nvVQXRqUZ9b357B8U99wzfzs4OOJhJ1WVt2ce7z49mRm88rlwyiU2qDoCOJSBwpdYy2mW1y96al7N/s7k2iEayytDKkSPm5Ox/PXMODH89m+YadDMlI447je9EtvVHQ0USq3IqNO7j0pUks27CDVy/dj707FHtPvohIqUobo13W9H65Ztba3VcX86atgbyqCCgiNYOZcVy/1hzRK52Xvl/CX79cwDF/+Yb/G9SBPxzZnRYNU4KOKFJheQWFTFm6kdFzsxkzN4s5a7ZSJzGBERfuqyJbRKKirB7td4DZ7r7nPNqY2X1AH3c/NYr5Kkw92iKVt35bDk9+OZ/Xxi+jfnIiVx/ejQsP6kRKkqY8k9iQtWUXY+aFCutv5q9j6658khKMzE7NGJKRzjF9Wmm4iIhUSmk92mUV2pnAN8CrwOvASqAtcDZwDnCwu0+p8sRVQIW2SNVZkLWVBz6aw1dzsmjfvB63Du3Fcf204I3UPAWFztTlmxgzN4vRc7OYuXILAOmNUhiSkc7gjDQO6p5K47rJAScVkXhR4UI7fPJRwN+BboTmvDZgAXClu39RxVmrjAptkar3zfxs7v9wNnPWbGWfjs248/he7KU/uUvA1m/LYez8bEbPyWbs/Gw27cgjwWDvDs0Y0jNUXPdu3Vi/GIpIVFSq0C7yJt2BNCDb3edXYb6oUKEtEh0Fhc6oSct57LN5rNuWw8kD23CzFryRalRY6MxYuZkxc7MZPTeLaSs24Q4tGtThsIw0hmSkc0j3VJrW11zYIhJ9VVJoxxoV2iLRtS0nn399vZBnxi4C4JKDO/O7wV1ppD/JSxRs3pEX6rWem8XYedms25aLGfRv15Qh4eK6X9smJCSo11pEqpcKbRGJmlWbdvLop3N558eVpDasw/VHZTAssx1JiWVN0y9SMndn1uotjAnPEDJ56UYKHZrUS+awHmkM6ZnGod3TNBOOiAROhbaIRN205Zu478NZTFyykYyWjbjj+F4c2iMt6FgSQ7buyuO7BesYPSebMfOyWLslB4C+bRv/fCPjwPbNSFSvtYjUICq0RaRauDufzFzDgx/PYdmGHQzOSOOO43rRvaUWvJFfc3fmZ21j9JzQDCGTlmwkv9BplJLEIT1SGZyRzuAeaaQ3rht0VBGREqnQFpFqlZNfwCs/LOXJL+ezI7eAs/Ztz3VH9SBVf+av9bbn5PP9wvWMmZvFmLnZrNy0E4CerRoxOCOdIRlp7N2xGckaeiQiMUKFtogEYsP2XJ76cj6vjFtKveRErhrSjYsO6kTdZC14U1u4O4vXbf95NcbxizaQW1BI/TqJHNwt3GudkUYbzVojIjFKhbaIBGpB1jYe+ng2X8zOom3Tetx6bE9O6N9a8xrHqV15BfywaD1fh6ffW7p+BwBd0xowJCOdIT3TyezUTCuMikhcUKEtIjXCdwvWcd+Hs5m9egt7dWjKXSf0Zm8teBMXlq3fwZh5WYyek8X3C9eTk19I3eQEDuyayuCMNAb3SKdDi/pBxxQRqXIqtEWkxigodN6asoJHP51L9tYcThzQhpuPyaB9cxVhsSQnv4CJizcyOrzU+aLs7QB0bFH/5xlC9u/SQsOERCTuqdAWkRpne04+/xq7iGfGLqTQ4eKDOnPVEC14U5Ot2rTz59UYv1uwjh25BdRJTGC/Ls1/HhLSObVB0DFFRKqVCm0RqbFWbw4tePP2lJW0aFCH64/uwZmZ7bXgTQ2QV1DI5KWhXusxc7KZu3YrAG2b1mNweDXGA7u1oH6dpICTiogER4W2iNR401ds4r4PZzNh8Qa6pzfkjuN7MTgjPehYtU7Wll0/91p/O38dW3PySUow9u3UnCE9Q8V1t/SGupFVRCRMhbaIxAR359Of1vLgx7NZun4Hh/YILXiT0UoL3kRLQaEzdflGRs8JFdc/rdoCQMvGKeGx1ukc1K2FhvSIiJRAhbaIxJTc/EJe/mEJT305n205+Zw1qAPXHdmDtEZa8KYqrN+Ww9fzshk9N5tv5mezaUceCQb7dGwWXjQmnV6tG6nXWkQkAiq0RSQmbdyey1NfzeeVH5ZSNzmRK4d05eKDOmsmi3IqLHSmr9zMmLlZjJ6bzfQVm3CH1IZ1OKxHOkN6pnFItzSa1FevtYhIeanQFpGYtih7Gw9+PIfPZ62lbdN63Dw0g5MGtFGPayk27chl7Px1jJmTxdfzslm/PRczGNi+KYPDxXXfNk1ISNBnKCJSGSq0RSQufL9wHfd9MJtZq7cwsH1T7jqhF/t0bB50rBrB3flp1RbGzM1izNxspizbSKFD0/rJHNYjdBPjoT3SaN6gTtBRRUTiigptEYkbBYXO2+EFb7K25nB8/9bcOrRnrVzwZsuuPL6bvy40/d7cbLK25gDQr20ThmSkcVhGOgPbNyVRvdYiIlGjQltE4s6O3Hz+9fUinhm7iIJC56KDO3HVkG40juPZMdydeWu3hVZjnJPF5KUbyS90GtVN4tDuaQzOSOOwjDTSG9UNOqqISK2hQltE4taazbt47LO5vDVlBc3q1+G6o3pw9r7xs+DN9px8vl+4PrxoTBarNu8CoGerRgzpGZohZK8OTUmOk+9XRCTWqNAWkbg3c+Vm7v1gFuMXb6BbekPuOK4XgzPSYu6GSXdn0brtjJ4TGg4yYfEGcgsKaVAnkYO7pzI4I53BGWm0blIv6KgiIoIKbRGpJdydz2et5cGP57B43XYO6Z7KHcf3omerxkFHK9XO3ALGLVr/8/R7yzbsAKBbekOGhJc6z+zUnDpJ6rUWEalpVGiLSK2Sm1/Iq+OW8uSX89m6K48z923PdUf1qFFjl5et3xEaaz03ix8Wricnv5C6yQkc1DWVwT3TGdwjrVbe4CkiEmtUaItIrbRpRy5PfbmAl39YQkpSAlcO6cYlBwez4E1OfgETFm9g9JxsxszNYtG67QB0alE/tBpjz3T269xci/GIiMQYFdoiUqstXredhz6ezac/raVNk7rccmxPTuzfJuqLtazctPPnsdbfL1zHjtwC6iQlsH+XFgzJSGNwRjqdUxtENYOIiERX4IW2mY0ATgCy3L1veNvdwG+B7PBht7v7R3uc1x54GWgJOPCMuz8ZyTVVaIvInsYtWs99H85i5sotDGjflLuO70Vmp6pb8CavoJBJSzaGx1pnMW/tNgDaNq3HkJ6hsdYHdG1B/TpJVXZNEREJVk0otA8FtgEv71Fob3P3x0o5rzXQ2t2nmFkjYDJwirvPKuuaKrRFpDiFhc47P67kkU/nsHZLDsf1a8WtQ3vRoUXFxkOv3bLr59UYv52/jq05+SQnGvt2as6QjNBS513TGsbc7CciIhKZ0grtaulWcfexZtapAuetBlaHX281s9lAW6DMQltEpDgJCcZv9mnHsf1a8ezYxfzz64V8MSuLCw8KLXjTpF7pC97kFxQydfmm8KIx2cxavQWAVo3rcsKA1gzOSOegbqk0TFGvtYhIbRf0vwRXm9n5wCTgBnffWNKB4UJ9L2B8NWUTkThWv04S1x7ZnbMGteexT+fy7DeLGDVpeWjBm0EdfrEAzLptOXw9N5vRc7MYOy+bLbvySUww9unQjJuHZjAkI52erRqp11pERH6h2m6GDBfKHxQZOtISWEdo7PW9hIaIXFzCuQ2Br4H73f3tUq5xGXAZQIcOHfZZunRplX4PIhK/flq1mfs+mM0Pi9bTNa0BVw3pxtL1OxgzN4vpKzfjDqkNUxgcntf64O6pZfZ+i4hI/At8jHY4RCeKFNrl2JcMfAB86u6PR3o9jdEWkfJyd76cncUDH81m0brtmMFe7ZuGpt/LSKdPm8ZRn6lERERiS+BjtItjZq3DY7ABTgVmFnOMAc8Ds8tTZIuIVISZcWTvlhyWkcbkpRvp0bIRzRvUCTqWiIjEqGoptM3sdWAwkGpmK4A/AYPNbCChoSNLgMvDx7YBnnP344CDgPOAGWY2Nfx2v5oGUESkKiUnhua6FhERqYzqmnXk7GI2P1/CsauA48KvvwX0d1oRERERiTkJZR8iIiIiIiLlpUJbRERERCQKVGiLiIiIiESBCm0RERERkShQoS0iIiIiEgUqtEVEREREokCFtoiIiIhIFKjQFhERERGJAhXaIiIiIiJRYO4edIaoMLNsYGkAl04F1gVwXale+jnXDvo5xz/9jGsH/Zxrh6B+zh3dPa24HXFbaAfFzCa5e2bQOSS69HOuHfRzjn/6GdcO+jnXDjXx56yhIyIiIiIiUaBCW0REREQkClRoV71ngg4g1UI/59pBP+f4p59x7aCfc+1Q437OGqMtIiIiIhIF6tEWEREREYkCFdoiIiIiIlGgQltEREREJApUaItEwMySirxuaGaZZtY8yEwiUnFm1lxtWESiTYW2SBnM7EJgrZnNM7NjgenAw8A0Mzs70HAiEjEz62Bmb4RXDh4PTDCzrPC2TgHHE5EqYGYzgs5QVFLZh4jUejcAGUAjYBqwl7svNLOWwOfA60GGE5GI/Qf4C3COuxcAmFkicAbwBrB/cNFEJFJmdlpJu4BW1ZmlLJreT6QMZjbV3QeGX69y9zZF9k139/6BhRORiJnZfHfvXt59IlKzmFke8BpQXBF7urs3quZIJVKPtkjZlpnZg4R6tOeY2XDgbeBIYHWgyUSkPCab2dPAS8Dy8Lb2wAXAj4GlEpHymg485u4z99xhZkcGkKdE6tEWKYOZNQauIvSb89+AY4CLgKXAfe6uYlskBphZHeAS4GSgbXjzSuA94Hl3zwkqm4hEzswOAZa6+7Ji9mW6+6QAYhVLhbaIiIiISBRo1hGRMphZopldbmb3mtlBe+y7M6hcIlI+ZlbfzG42s5vMrK6ZXWBm75nZI2bWMOh8IhIZM0sK/7v8iZlNDz8+NrMrzCw56HxFqUdbpAxm9hxQH5gAnAd87e7Xh/dNcfe9g8wnIpExs5GExmbXIzST0GxCM5GcBLRy9/MCjCciETKz14FNhO63WBHe3I7Q/RbN3f3MgKL9igptkTIUnVkkvHDN00AqcDYwzt33CjKfiERm9wxCZmaEbmRu7e4e/nqaZhASiQ1mNs/de5R3XxA0dESkbHV2v3D3fHe/DJgKfAXoz80iMcZDPUwfhZ93f61eJ5HYscHMzjCzn+tYM0swszOBjQHm+hUV2iJlm2RmQ4tucPc/Ay8AnQJJJCIVMWn3WGx3v3j3RjPrCmwNLJWIlNdZwOn8b9XmecBa4LTwvhpDQ0dERKTWMzNz/YMoEnPMrAWAu68POktx1KMtUgFm9kzQGUSk8na3ZRXZIrHJ3de7+/qa+u+yCm2RiskMOoCIVAm1ZZH4UCPbsgptkYrJCjqAiFQJtWWR+FAj27LGaIuIiIiIRIF6tEUqoaaOCROR8lFbFokdsbRis3q0RcpgZs1L2kVokYt21ZlHRCpGbVkkPsTSis0qtEXKYGYFwFJC/xjv5uGv27p7nWJPFJEaRW1ZJD7E0orNSUEHEIkBi4Aj3H3ZnjvMbHkAeUSkYtSWReLDL1ZsBi4zsz9SA1ds1hhtkbL9BWhWwr5HqjGHiFTOX1BbFokHMbNis4aOiIiIiIhEgYaOiETAzHoCJwNtw5tWAu+5++zgUolIeakti8SHWGnLGjoiUgYzuwV4g9ANUxPCDwNeN7Nbg8wmIpFTWxaJD7HUljV0RKQMZjYP6OPueXtsrwP85O7dg0kmIuWhtiwSH2KpLatHW6RshUCbYra3Du8TkdigtiwSH2KmLWuMtkjZ/gB8aWbzgd1TgHUAugFXBxVKRMrtD6gti8SDPxAjbVlDR0QiYGYJwCB+edPFRHcvCC6ViJSX2rJIfIiVtqxCW6QCzOwyd38m6BwiUjlqyyLxoaa2ZY3RFqmYK4IOICJVQm1ZJD7UyLasQlukYizoACJSJdSWReJDjWzLGjoiUgFm1s7dVwSdQ0QqR21ZJD7U1LasHm2RCtjdmM3soqCziEjFqS2LxIea2pbVoy1SCWa2zN07BJ1DRCpHbVkkPtS0tqx5tEXKYGbTS9oFtKzOLCJScWrLIvEhltqyCm2RsrUEjgE27rHdgO+rP46IVJDaskh8iJm2rEJbpGwfAA3dfeqeO8xsTLWnEZGKUlsWiQ8x05Y1RltEREREJAo064iIiIiISBSo0BYpg5n1N7NxZrbczJ4xs2ZF9k0IMpuIRE5tWSQ+xFJbVqEtUrangbuBfsA84Fsz6xrelxxUKBEpN7VlkfgQM21ZN0OKlK2Ru38Sfv2YmU0GPjGz8wDd5CASO9SWReJDzLRlFdoiETCzJu6+GcDdR5vZb4C3gObBJhOR8lBbFokPsdKWNXREpGwPA72KbnD36cARwNuBJBKRilBbFokPMdOWNb2fiIiIiEgUqEdbpAxm1sTMHjKzOWa2wczWm9ns8LamQecTkcioLYvEh1hqyyq0Rco2ktAyr4Pdvbm7twCGhLeNDDSZiJSH2rJIfIiZtqyhIyJlMLO57p5R3n0iUrOoLYvEh1hqy+rRFinbUjO72cxa7t5gZi3N7BZgeYC5RKR81JZF4kPMtGUV2iJlOxNoAXxtZhvNbAMwhtAUQsOCDCYi5aK2LBIfYqYta+iISATMrCfQDhjn7tuKbB9aZNJ8Eanh1JZF4kOstGX1aIuUwcx+D7wLXA3MNLOTi+x+IJhUIlJeassi8SGW2rJWhhQp22+Bfdx9m5l1At40s07u/iRgwUYTkXJQWxaJDzHTllVoi5QtYfefpdx9iZkNJtSoO1LDGrSIlEptWSQ+xExb1tARkbKtNbOBu78IN+4TgFSgX1ChRKTc1JZF4kPMtGXdDClSBjNrB+S7+5pi9h3k7t8FEEtEykltWSQ+xFJbVqEtIiIiIhIFGjoiIiIiIhIFKrRFRERERKJAhbaIiIiISBSo0BYRiTNmNsbMcsxsq5ltNrNFZvaKme1TjvdYYmbnRjOniEi8U6EtIhKf7nX3Ru7eBBgCLAXGmdmpAecSEak1VGiLiMQ5d1/q7ncCLwN/tZBrzWxOuNd7mZk9aGaJAGb2PtABeM7MtpnZZ+HtSWZ2u5nNM7NNZvadmWUG952JiNRsKrRFRGqPN4C2QAawAjgWaAycDFwMXArg7icCy4BL3b2hux8dPv+e8LFDgRbACOATM2tWnd+EiEisUKEtIlJ7rAg/t3D3t9x9sYf8CLwCHFHSiWZmwO+Bm9x9kbsXuPvzwGrg+KgnFxGJQUlBBxARkWrTLvy83szOBq4HuhD6t6AOMK6Uc1OBhsD7ZlZ0pbPkIu8rIiJFqNAWEak9zgRWAtuBV4HTgI/dPdfMHgOKjrcu3OPcdeHzjnT3idURVkQk1mnoiIhInDOz9mZ2D3AhcC2hnukEIBvIM7P9gfP2OG0N0H33F+7uwJPAY2bWPfy+Dc3sGDNrE/3vQkQk9ljo/50iIhIvzGwMcACQCziwHvgeeNLdJ4SP+SNwDaEhI6OBJcBAdx8c3n8c8FegOTDO3Y81syRC47QvJTRcZDuh4SbXuPvu8d8iIhKmQltEREREJAo0dEREREREJApUaIuIiIiIRIEKbRERERGRKFChLSIiIiISBSq0RURERESiQIW2iIiIiEgUqNAWEREREYkCFdoiIiIiIlGgQltEREREJAr+H8Gb2/0wEhN2AAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plot_daily_cumulative_accept_rate(df[df['pyName']=='UPlusPersonal'], 'Accepted', 'Rejected', \n", - " **{'allTime':True, 'shrinkTicks':True, 'showOutlier':True,\n", - " 'title':'Proposition: UPlusPersonal'})" + "plot_daily_cumulative_accept_rate(\n", + " df[df[\"pyName\"] == \"UPlusPersonal\"],\n", + " \"Accepted\",\n", + " \"Rejected\",\n", + " **{\n", + " \"allTime\": True,\n", + " \"shrinkTicks\": True,\n", + " \"showOutlier\": True,\n", + " \"title\": \"Proposition: UPlusPersonal\",\n", + " },\n", + ")\n", + "# Not sure how useful that really is" ] }, { @@ -652,23 +170,14 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuAAAAFwCAYAAAD5UM7XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABAH0lEQVR4nO3dd5idZZ3/8fd3Jr33kB5K6C0QIWBBRBSQLigKrAUX9GfXFcu6llXXta/r2lh11QRBFEIRpVgQRRJIJxA6TJJJJSG9Tfn+/pgTGIaUk8mcOVPer+uaa865n+c55zMMd+Y799z3/URmIkmSJKl1VJQ7gCRJktSZWIBLkiRJrcgCXJIkSWpFFuCSJElSK7IAlyRJklqRBbgkSZLUirqUO0BrGzJkSI4fP77cMSRJktTBzZo167nMHNq0vdMV4OPHj2fmzJnljiFJkqQOLiKqdtbuFBRJkiSpFVmAS5IkSa3IAlySJElqRRbgkiRJUiuyAJckSZJakQW4JEmS1IoswCVJkqRW1CoFeET8LCJWRsSCRm3fiIhHI2J+REyLiAG7uPaMiHgsIp6MiE81av95RDwTEXMLH8eW/iuRJEmS9k1rjYD/HDijSdvdwJGZeTTwOPDpphdFRCXwfeBM4HDgbRFxeKNTPpGZxxY+5pYiuCRJktSSWuVOmJl5b0SMb9J2V6On04GLdnLpCcCTmfk0QERcD5wHPFKiqJKkTq6+PpmzeC1bttdREVBREVREUBEQhc+VhbYICseCyoodxxvO2XF8p+dGEBW8/Nx48dyIKPd/Ckkl0lZuRf9u4Nc7aR8FLG70fAlwYqPnX4mIzwF/Aj6VmdtKF1GS1JGt2bSd38xczLUzFrFozeZyx3mhYK98SfFe+FzRuHB/8XFlRfHnNm7f8ctFZQQVFTs59yXXNTq3yes2vP9OXncnGfb13BdyNclb+bKvs/HX8NJfpLpUVtCnexf69mj46N2tCxUV/uKj0it7AR4R/wrUAtfu5aWfBpYD3YBrgE8C/76L97gSuBJg7Nixzc4qSepYMhtGu6feX8XvHlrG9tp6Tth/EB87/WBGDexJfX1Sl0km1GdSv+Nz/YuPs1F7XX1x59bVZ+F5Eedmo3MbHX/xo+HrqK9vcu6O161/+bl1L3mfF1+vrj6pqWtybuG1G59b1+S6+vpG5+7Is9P/Jk0zlfv/gJeKgD7dutCnUJA3FOddXyjQ+/bo2qhgb3jcr8eO81881qNrZbm/FLVxZS3AI+KdwNnAaZm5s25YDYxp9Hx0oY3MXFZo2xYR/wf8y67eJzOvoaFIZ9KkSW2su0uSWtvm7bXcOncpU6ZX8fDS9fTp3oVLXjGGS08cxyH79S13vE5lb4r1l/xSsrNzG/1CsOMXhhd/KXnx9Xb8AlFTV8/GbbVs3FrLhq21bNhWy4atNWzYWmjbVsPazdtZvGYz67fWsnFbDVtr6vf4NXWrrGhSxBcK+cLjHQX7juP9enR9WdHfp3sXKh2N77DKVoBHxBnA1cApmbmrv/U9CEyIiP1pKLwvAd5euH5EZi6Lhkly5wMLdvEakiQB8OTKjVw7o4rfzlrChq21HLpfX758/pGcP3EUfbqX/Y/CnVLDdBaoJGgPA8fba+vZtK2hYF+/tYaNhccbtzUU7i9+NDq2tZbFazYXzms4Vszof+9ulQ3FeI+dFPE7CvXCsX49utCne9dGBX5DYd+9S4XrCdqgVvnXJiKuA14LDImIJcDnaZhC0h24u/A/xvTMfG9EjAR+kplnZWZtRHwAuBOoBH6WmQ8XXvbaiBgKBDAXeG9rfC2SpPalpq6ePz6yginTq/jHU6vpWhmceeQILj9pHJPGDbQ40V7p1qWCbl26MbB3t2a/RmaypabuhUJ9R9G+cdtLnzcu7Dduq2XdlhqWPL/5hRH7LTV1e3yvLhXxYlHefefTaXaMyPdrMgK/o4jv3b2SLpXeOqYlxc5nfnRckyZNypkzZ5Y7hiSpxFas38p1DyziugcWsWL9NkYN6MnbTxzLWyaNYWjf7uWOJ+2z2sIUmp2Num/YWlOYUrNjik1No2k2Lx2xrytiOL5Xt8qXTqfZyZSZHQV70+k0O+bJ9+xa2el+4Y2IWZk5qWm7f2+TJHUYmcn9T69m6vQq7nx4BXX1ySkHD+Ur54/j1EOHOadWHUqXygoG9OrGgF77Nhq/taZ+DwV7TaN58i8W7svWbX3hvM3b9zwaX1kRL466F+a+N54ys2MKTb9GI/aNp9P06d7Q3rUDjMZbgEuS2r11W2q4afYSpk6v4qlVmxjQqytXvGp/3n7CWMYP6V3ueFKbFRH07FZJz26VDNuH16mrzxcWru5pOk3jEfvl67eyYeWL59YWMRrfo2vFS+bD72k6zaDe3TjpwMH78NW1PAtwSVK79fDSdUydXsXNc5aypaaOY8YM4JsXH8PZR49wKzipFVVWBP17daV/r67Nfo3MZFtt/Uvmxjct5F/y/IUR+xpWrN/6wvGN22pf8rpjB/Xi3qtP3dcvsUVZgEuS2pWtNXX8YcEyptxfxexFa+nRtYLzjhnFZZPHcdTo/uWOJ6mZIoIeXSvp0bVyn9Zp1NUnm7a/OMpezBz31mYBLklqFxav2czUGVXc8OBint9cw/5DevNvZx/ORceN3qdRN0kdS2VF0K9HV/r16Ar0LHecnbIAlyS1WXX1yV8fX8mU+6u45/FVBHD64cO5fPJ4Tj5wsLcNl9QuWYBLktqc1Ru3ccPMJVw7o4olz29haN/ufPB1E3jbCWMY0b9tjmhJUrEswCVJbUJmMnvR80y5v4rfP7Sc7XX1TD5gEJ8+8zDecMTwDrH1mCSBBbgkqcw2bavllrlLmTK9ioXL1tOnexfedsIYLps8jgnD+5Y7niS1OAtwSVJZPLlyA1OnL+LGWUvYsK2WQ/fry1cuOJLzjx1F7+7+eJLUcfkvnCSp1dTU1XPXwyuYOr2K+59eTbfKCs46aj8uP2kcx40d2OluUy2pc7IAlySV3LJ1W7jugcVc/8AiVm7YxqgBPbn6jEN4y6QxDOnT/P1+Jak9sgCXJJVEZvKPp1Yz5f4q7l64gvpMXnvwUP7zpHGccvAwKt1CUFInZQEuSWpR67bUcOOsJUydUcXTqzYxsFdX3vPq/bn0hHGMHdyr3PEkqewswCVJLWJB9Tqm3F/FLfOq2VpTz3FjB/DttxzDWUeNoEfXynLHk6Q2wwJcktRsW2vquH3+MqZMr2Lu4rX07FrJBRNHcemJ4zhyVP9yx5OkNskCXJK016pWb+JXMxZxw8zFPL+5hgOG9ubz5xzOhceNpn/PruWOJ0ltmgW4JKkodfXJXx5dyZTpVdz7xCoqInjD4cO5fPI4TjpwsFsISlKRLMAlSbv13MZt/PrBxfxqxiKq125heL/ufPi0CVzyirHs179HueNJUrtjAS5JepnMZFbV80yZXsXvH1pGTV1y8oGD+eybDuP1hw+na2VFuSNKUrtlAS5JesHGbbXcPKeaqdOreHT5Bvp278KlJ47jssljOWhY33LHk6QOwQJcksTjKzYwdXoVN82uZuO2Wg4f0Y//vPAozj12JL26+aNCklqS/6pKUie1vbaeOx9eztTpVcx4Zg3dKis4++gRXHbSOCaOGeCiSkkqEQtwSepklq7dwnUPLOK6Bxbz3MZtjBnUk0+deSgXHz+awX26lzueJHV4FuCS1AnU1yf3PfUcU+6v4o8LV5DA6w4ZxmWTx/Gag4dSWeFotyS1FgtwSerA1m2u4TezFnPtjEU889wmBvXuxlWnHMjbTxjLmEG9yh1PkjolC3BJ6oDmL1nLlPuruHXeUrbV1nP8uIF8+LQJnHnUfnTvUlnueJLUqRVdgEfEAcAgYA3wTGZmyVJJkvba1po6bpu3lKnTq5i3ZB29ulVy4XGjuWzyWI4Y2b/c8SRJBbstwCPiOODDwNnAwEaH1kbEbcB3M3N2CfNJkvbg2ec2ce2MKm6YuYR1W2o4aFgfvnjuEVxw3Cj69eha7niSpCZ2WYBHxC+BU4ApwIXAQ8B6oB9wJHA6cFNE3JuZ/9QKWSVJBbV19fz50ZVMmV7F3554ji4VwRuP2I/LJo9j8gGD3EJQktqw3Y2A3we8KzPrmrSvAe4F7o2ILwBXlCibJKmJVRu28esHF/GrGYtYum4rw/t156OvP5hLThjD8H49yh1PklSEXRbgmfnjPV1cKM6vadFEkqSXyEwefPZ5pkyv4o4Fy6ipS1510BA+d84RvP6wYXSprCh3REnSXtjrXVAi4jRgIvBkZt7c4okkSQBs2FrDzXOqmTp9EY+t2EDfHl24fPJ4Lp08lgOH9il3PElSM+1VAR4RnwHeCPwDuCgizsrMK0uSTJI6qUeXr2fq9Cqmza5m0/Y6jhjZj6+9+SjOOWYkvbq5e6wktXd73AWlyS4nbwBem5kZET2BasACXJL20fbaeu54eDlT76/igWfX0K1LBeccPZLLJo/l2DEDXFQpSR3InoZS/jci7gS+kJnbgVXA5RFxP3Am8GyJ80lSh1a9dgvXzVjE9Q8u4rmN2xk7qBefOetQLj5+DAN7dyt3PElSCeypAD8R+FfgwYh4L/Ah4FvAJ4EngUtLG0+SOp76+uRvTz7HlPur+POjKwB43aHDuGzyOF4zYSgVFY52S1JHttsCPDNrgS9GxI3A/wIPAO/JzM2tEU6SOpLnN23nt7OWMHVGFVWrNzO4dzfe99oDedsJYxk9sFe540mSWklRq3kyc0FEvBL4GA2j4R/KzD+VNpokdQzzFq9lyvQqbpu3lG219bxi/EA+dvrBnHHkfnTvUlnueJKkVranRZgnAT8CDgTmA+8GbqZhbvjbgI9l5vpSh5Sk9mbL9jpum7eUKdOreKh6Hb27VXLR8aO5bPI4DhvRr9zxJElltKcR8P8DPg38Hjgb+G5mvhE4NSLeB8wADittRElqP55etZFrZyziNzMXs35rLQcP78OXzjuC8yeOom+PruWOJ0lqA/ZUgA8B/piZ2yLizzQsyAQgM38YEbcV+0YR8TMaiviVmXlkoe0bwDnAduAp4F2ZuXYn154BfBeoBH6Smf9ZaN8fuB4YDMwCLi/s1iJJraa2rp4/PbqSqdOr+NsTz9GlIjjjyP24fPI4Tth/kFsISpJeYk8F+I+AWRHxIHAC8I3GBzNzyV6818+B/wF+2ajtbuDTmVkbEV+jYbT9k40viohK4PvA6cASGuag35qZjwBfA76TmddHxI+AK4Af7kUmSWq2leu3cv2Di7nugUUsW7eVEf178PHTD+atJ4xhWN8e5Y4nSWqj9rQLymcLO6AcBHw5Mxc2940y896IGN+k7a5GT6cDF+3k0hNouO390wARcT1wXkQsBF4HvL1w3i+AL2ABLqmEMpMZz6xhyvQq7lywnNr65NUThvDFc4/gdYcOo0tlRbkjSpLauD3ugpKZc4A5rZDl3cCvd9I+Cljc6PkSGvYnHwysLWyVuKN9VEkTSurUttbU8Y6fPcCMZ9bQv2dX3nnyeC6dPI79h/QudzRJUjuyywI8Ir4IfD0zN+3mnD7AJzLz8/sSIiL+FagFrt2X19nN618JXAkwduzYUryFpA4uM/nXaQuY8cwaPn/O4VzyirH07OYWgpKkvbe7v5V2B56JiP+NiLdExJERMbbw+S0RcQ3wNLBPy/oj4p00LM68NDNzJ6dUA2MaPR9daFsNDIiILk3aXyYzr8nMSZk5aejQofsSV1In9cv7q7hx9hI+fNoE3vXK/S2+JUnNtssCPDM/BUwCVtEwt3o+8Ezh8xdpKIAnZeZnmvvmhd1NrgbO3c3dNR8EJkTE/hHRDbgEuLVQrP+FF+eNvwO4pblZJGlXZjy9mi/97hFOO3QYHz5tQrnjSJLaud2uFsrMRZn5mcw8HOhFwyhzr8w8LDM/nZmLin2jiLgOuB84JCKWRMQVNOyK0he4OyLmFnYyISJGRsTvCxlqgQ8AdwILgRsy8+HCy34S+FhEPEnDnPCfFv+lS9KeLVu3hff/ajZjB/XiO5ccS0WFWwpKkvZN7HzWR8c1adKknDlzZrljSGoHttbU8dZrpvPkig3c/P5XMmF433JHkiS1IxExKzMnNW3f4y4oktQZZSafu2UB8xav5UeXHW/xLUlqMW5YK0k7ce2MRdwwcwkffN1BnHHkfuWOI0nqQCzAJamJmc+u4Yu3PcyphwzlI68/uNxxJEkdjAW4JDWyfN1W3jt1NqMG9OS/LplIpYsuJUktrOgCPCIui4i7I2J+4flrIuLC0kWTpNa1rbaO9107i83ba/nx5ZPo33OfbnMgSdJOFVWAR8THaNj7+w/AjltJrqJhD29J6hC+cOsjzFm0lm9efAyH7OeiS0lSaRQ7Av4+4MzM/DawY9/Cx4GDSpJKklrZr2Ys4roHFvH/XnsgZx01otxxJEkdWLEF+KDMfLzweEcBHo0eS1K7NavqeT5/6wJec/BQPv6GQ8odR5LUwRVbgD8SEWc3aTsDmNfCeSSpVa1cv5X3TZ3FiP49+e9LjnXRpSSp5Iq9Ec9ngNsj4gage0R8D7gEaFqUS1K7sb22nvddO5sNW2v5xbtPYECvbuWOJEnqBIoaAc/MvwGTgS3AXwrXvTYzZ5QwmySV1L//7mFmVT3P1y86msNG9Ct3HElSJ1HUCHhEjM/MR4APNmkfl5lVJUkmSSX06wcXMXX6Iq56zQGcc8zIcseRJHUixc4Bn7+L9jktFUSSWsvcxWv5t5sf5tUThnD1GYeWO44kqZMptgB/2aqkiOiKu6BIamdWbtjKe6fMYli/7vy3d7qUJJXBbqegRMTdNBTZ3SPiriaHxwKzSxVMklra9tp63n/tbNZu2c6N7zuZgb1ddClJan17mgP+98LnU4D7GrXXA8uB35QilCSVwlduf4QHn32e715yLEeM7F/uOJKkTmq3BXhmfhEgIhZm5g2tE0mSWt5vZi7mF/dX8Z5X7c95x44qdxxJUidW1C4oO4rviOgBDKXRnPDMXFSaaJLUMuYvWcu/3ryAkw8czKfOdNGlJKm8it2G8ABgKnDiTg5XtmgiSWpBz23cxnunzGJon+78z9uPo0tlsWvPJUkqjWJ/Ev0PsBg4BtgAHA3cDFxRmliStO9q6hoWXa7etJ0fX348g1x0KUlqA4q9Ff2JwPjM3BARZObDEXEV8Ffg5yVLJ0n74D9+v5AZz6zh2285hiNHuehSktQ2FDsCXk/DbegBNkbEAGANDVsRSlKbc9PsJfzffc/yrleO58LjRpc7jiRJLyh2BPxh4JU0jHjPAL4DbAKeKVEuSWq2BdXr+PRND3Hi/oP4zFmHlTuOJEkvUewI+IdoGPEG+AQwCpgEXFWKUJLUXGs2beeqKbMY3Lsb37/0OLq66FKS1MYUuw3h/EaPnwbeABARTqqU1GbU1tXzgV/NZtXGbfz2vScxpE/3ckeSJOllmjU0FBE9IuJq4OkWziNJzfaff3iUfzy1mq+cfyRHjx5Q7jiSJO3UbgvwiBgfEX+OiPUR8UBEHBwRbwSeoGELwg+1SkpJ2oNb5lbzk78/wztOGsfFk8aUO44kSbu0pyko3y58/hTwVuA3wCDg34BfZmZ9CbNJUlEeXrqOT944nxPGD+KzZx9e7jiSJO3Wngrwk4EjMnN1RNwELAUmZebs0keTpD17vrDockBPF11KktqHPf2k6pWZqwEyczmw0eJbUltRW1fPB6+bw8r12/jR5ccztK+LLiVJbd+eRsAjIkYAUXhe1+Q5mbm0VOEkaXe+cddj/P3J5/j6m4/m2DEDyh1HkqSi7KkA7w0safQ8Gj0PIIHKEuSSpN26bd5SfvzXp7ls8lje8goXXUqS2o89FeD7t0oKSdoLC5et5+rfzuf4cQP53NlHlDuOJEl7ZbcFeGZWtVYQSSrG2s0Niy779ujCDy89jm5dXHQpSWpfiroTpiS1BXX1yYeun8uydVu4/sqTGNavR7kjSZK01yzAJbUb37rrMe59fBVfvfAojh83sNxxJElqFv92K6ld+P1Dy/jBPU/xthPG8rYTxpY7jiRJzVZUAR4R43fRPq5F00jSTjy2fAP/8pt5TBw7gC+c650uJUntW7Ej4PN30T6npYJI0s6s21zDlVNm0rt7F3502fF07+LOp5Kk9q3YAjxe1hDRlYZ9wCWpJOrqkw//eg7Vz2/hh5cex3AXXUqSOoDdFuARcXdE3AV0j4i7Gn8ADwFF3ZY+In4WESsjYkGjtosj4uGIqI+ISbu59sMRsaBw7kcatX8hIqojYm7h46xiskhqP/7rj49zz2Or+Py5RzBp/KByx5EkqUXsaReUvxc+nwLc16i9HlgO/KbI9/k58D/ALxu1LQAuBH68q4si4kjgn4ETgO3AHRHxu8x8snDKdzLzm0VmkNSO3LFgOd/785O8ddIYLjvRRZeSpI5jTzfi+SJARCzMzBua+yaZeW/ThZyZubDw2ru79DBgRmZuLpz7VxqK9q83N4uktu+JFRv4+A1zOWbMAL543hF7+ndCkqR2pag54Jl5Q0T0iYhLIuJfCp/7ljocDaPkr46IwRHRCzgLGNPo+AciYn5hioubAksdwPqtNVw5ZRY9u1Xyo8uOo0dXF11KkjqWYrchnAQ8DXwNOK/w+andzd1uCYVR8q8BdwF3AHOBusLhHwIHAscCy4Bv7ep1IuLKiJgZETNXrVpVysiS9kF9ffLR6+eyeM1mvv/24xjRv2e5I0mS1OKK3QXlB8C3MnNcZr46M8cB36ShCC6pzPxpZh6fma8BngceL7SvyMy6zKwH/peGeeK7eo1rMnNSZk4aOnRoqSNLaqbv/ukJ/vToSv7t7MM58YDB5Y4jSVJJFFuAH8bLR5i/DRzasnFeLiKGFT6PpWH+968Kz0c0Ou0CGqarSGqn7n5kBd/90xNcdPxo/ukk7/ElSeq4ii3A5wJHNmk7qtC+RxFxHXA/cEhELImIKyLigohYApwE3B4RdxbOHRkRv290+Y0R8QhwG/D+zFxbaP96RDwUEfOBU4GPFvm1SGpjnly5kY/+ei5Hj+7Pl88/0kWXkqQObU/bEO5wF/C7iPgJUAWMB94NXBMRb99xUmb+amcXZ+bbdvG603Zy7lIaFlvueP7qXbzm5UVml9SGbdhaw1VTZtK9SwU/uux4F11Kkjq8YgvwdwM1wDsatdUW2ndICtNDJKkY9fXJx26Yx7OrNzP1ihMZOcBFl5Kkjq+oAjwz9y91EEmdz//85UnufmQFnzv7cE460EWXkqTOodg54MAL87MnlyqMpM7jTwtX8J0/Ps4FE0fxrleOL3ccSZJaTbH7gA+LiD8CS4A/FtreGhE/KGU4SR3T06s28pHr53L4iH589cKjXHQpSepUih0B/2/gGWAoDXPBAf4MnF6KUJI6ro3barlqyiy6VAY/vtxFl5KkzqfYRZinAuMyc2tEJEBmrtqxR7ckFSMz+Zcb5vHUqo1MveJERg/sVe5IkiS1umJHwLfRpFiPiEHAmhZPJKnD+sE9T3HHw8v5zFmHcfJBQ8odR5Kksii2AL8L+FZEdG3U9kXg9paPJKkj+stjK/nmXY9x7jEjueJVbqwkSeq8ip2CcjVwM/A80CMi1gLzgPNKE0tSR/Lsc5v48HVzOHS/fnztzUe76FKS1KkVuw/4GuA1EXE8sD8Nd8OcmZlZynCS2r9NhUWXFRXBNZcfT89uLrqUJHVuRRXgETEQ2J6Zs4BZhbbeEdE1M9eWMJ+kdiwzufq383li5QZ+8e4TGDPIRZeSJBU7B/xW4IgmbUcCt7RsHEkdyY/++jS3P7SMT55xKK+eMLTccSRJahOKLcCPAGY2aZsJHNWycSR1FH99fBVfv/NR3nT0CK58zQHljiNJUptRbAG+FWj6t+PevHhTHkl6waLVm/nQdXM4ZHhfvnGRiy4lSWqs2AL878B/REQFQDT8NP134L5SBZPUPm3eXsuVUxr+YPbjy4+nV7diN1uSJKlzKPYn4ydouPX8myPiaRp2QtkOvK5UwSS1P5nJJ298iMdWbODn7zqBcYN7lzuSJEltTrHbEFZFxJHA2cB44Fng9szcXLpoktqbn/ztGW6bt5SrzziEUw520aUkSTtT9N+GM3ML8JsSZpHUjv39ief46h8WctZR+/G+Uw4sdxxJktqsouaAR8RdEfG6Jm2nRcQfShNLUnuyeM1mPnDdbA4a1odvXHSMiy4lSdqNYhdhHgfc26Ttb8ArWjaOpPZmy/Y6rpoyi7r65MeXT6J3dxddSpK0O8UW4PVA1yZtlYDDXFInlpl8+qb5LFy+nv++ZCL7D3HRpSRJe1JsAT4L+GCTtg8As1s2jqT25Gf3PcvNc5fy8dMP5tRDh5U7jiRJ7UKxfyv+JHBPRLwZeByYABwCvLZEuSS1cf946jn+4/cLeeMRw/l/rz2o3HEkSWo3ihoBz8z5wOHAb4H1wI3A4Zk5r4TZJLVRS57fzAd+NYf9h/TmW285looKZ6NJklSsvdmGcDnwjR3PI+KIiPh0Zn6oJMkktUlba+p479RZ1NTW8+PLj6ePiy4lSdorxc4BByAiukfEP0XEfcBDNOyOIqmTyEw+M+0hFlSv5ztvPZYDh/YpdyRJktqdooauIuJw4CrgMqAXDYX7GZl5VwmzSWpjfvGPZ7lpdjUfff3BvP7w4eWOI0lSu7TbEfCIuDwi/gYsAE4BvgCMAtYAc0sdTlLbMf3p1Xzp9oW8/rDhfPB1LrqUJKm59jQC/gtgNfCmzHzhrpfe5U7qXJau3cL7r53NuMG9+PZbj3HRpSRJ+2BPc8D/DdgI3BwR0yLinIjYq3njktq3HYsut9XWc83lk+jXo+k9uSRJ0t7YbTGdmV8BDgDOB5KG7QergQHAyBJnk1Rmmclnb17A/CXr+PZbjuGgYS66lCRpX+1xNDsb/CEzLwTGAT8AlgMPRsQNpQ4oqXymTq/it7OW8KHTJvCGI/YrdxxJkjqEvZpOkpnLMvNLNIyKnwd0K0kqSWX34LNr+OJtj3DaocP4yGkTyh1HkqQOo1l30MjMBH5f+JDUwSxft5X3TZ3NmEG9+PZbvdOlJEktyVvYSXqJbbUNiy63bK/lun8+kf49XXQpSVJLsgCX9ILM5HM3P8zcxWv50WXHMWF433JHkiSpw3FLQUkv+NUDi/j1zMW8/9QDOePIEeWOI0lSh7RXBXg08Key1AHNqlrDF259mNceMpSPnX5IueNIktRhFVWAR0SfiPgpsAV4stB2fkR8vpThJLWOFeu38t6psxk5oCfffetEKl10KUlSyRQ7Av4tYDjwSmB7oe1B4K2lCCWp9Wyvred9U2exaVst11w+if69XHQpSVIpFVuAnw1cmpmzaLgjJplZTZF3w4yIn0XEyohY0Kjt4oh4OCLqI2LSbq79cEQsKJz7kUbtgyLi7oh4ovB5YJFfi6RGvnDbw8xetJZvXHQMh+znoktJkkqt2AK8gobpJy+IiD7AxiKv/zlwRpO2BcCFwL27uigijgT+GTgBOAY4OyIOKhz+FPCnzJwA/KnwXNJeuO6BRfxqxiLee8qBvOlol3dIktQaii3A/w58uknbB4G/FHNxZt4LrGnStjAzH9vDpYcBMzJzc2bWAn+loWiHhjtx/qLw+BfA+cVkkdRg9qLn+fwtD/PqCUP4xBtddClJUmspdh/wjwF/jojLgD4R8RANt6F/XcmSNVgAfCUiBtMwAn8WMLNwbHhmLis8Xk7DHHVJRVi5YSvvmzqL4f278723uehSkqTWVFQBnpmLC9NBzgHGA1XA7zJzy24v3EeZuTAivgbcBWwC5gJ1OzkvIyJ39ToRcSVwJcDYsWNLE1ZqJ7bX1vP+a2ezfkstN/2/kxnQq1u5I0mS1KkUVYBHxNjMXAT8tsR5XiYzfwr8tJDjP4AlhUMrImJEZi4r7E2+cjevcQ1wDcCkSZN2WahLncGXb3+EB599nv9+20QOG9Gv3HEkSep0ip0D/nRhp5FLIqJ7SRM1ERHDCp/H0jD/+1eFQ7cC7yg8fgdwS2vmktqjG2Yu5pf3V3Hlaw7g3GOK2sRIkiS1sGIL8AnAP4CvAssi4ge72zqwqYi4DrgfOCQilkTEFRFxQUQsAU4Cbo+IOwvnjoyI3ze6/MaIeAS4DXh/Zq4ttP8ncHpEPAG8vvBc0i7MXbyWz05bwCsPGszVLrqUJKlsInPvZmRExGk0jDhfADydmceUIlipTJo0KWfOnLnnE6UOZNWGbZzzvb/TpTK47QOvYmBv531LklRqETErM182aF3sLiiN3QP0A8YAr9nHXJJKrKaunvf/ajZrt2znxvedbPEtSVKZFTsFhYg4OiK+AywFvgvcB/h3bKmN+8rtC3ngmTV87c1Hc8TI/uWOI0lSp1fsLihzaCi2bwX+Cbg7M+tLGUzSvrtx1hJ+/o9nueJV+3PesaPKHUeSJFH8FJT/BX7VaAGkpDbuoSXr+PS0h5h8wCA+feah5Y4jSZIKir0Rzw9KHURSy1m9cRtXTZnJ0D7d+f7bj6NLZdGzzSRJUontsgCPiFsy87zC47t2dV5mvqEUwSQ1T21h0eXqTQ2LLgf3adWt+yVJ0h7sbgR8eqPH/wC8g6TUDnz1D48y/ek1fPstx3DkKBddSpLU1uyyAM/MrzZ6/IVWSSNpn9w8p5qf/v0Z3nnyeC48bnS540iSpJ0oamJoRCzcRftDLRtHUnMtqF7HJ2+czwn7D+Jf33RYueNIkqRdKHZl1q6G0hxik9qANZu2c9WUWQzq3Y3vv/04urroUpKkNmu3u6BExGd2nNfo8Q4HAYtLkkpS0Wrr6vngdbNZtXEbv7nqJIb2ddGlJElt2Z62ITy98Llro8cA9cBy4N2lCCWpeF+/8zHue3I137joaI4ZM6DccSRJ0h7stgDPzFMBIuJ7mfnB1okkqVi3zlvKNfc+zT+dNI6LJ40pdxxJklSEoiaKWnxLbc8jS9dz9W/n8YrxA/nsmw4vdxxJklSkou6EGRE9gc8CpwFDgdhxLDMPKE00Sbvy/KbtXDV1Jv17duX7lx5Hty4uupQkqb0o9qf2d4DzgCnAcOBbwDbgZyXKJWkX6uqTD10/hxXrtvHDy45nWN8e5Y4kSZL2QrEF+DnAuZn5faC28PnNwKklSyZpp75x52P87Ynn+PfzjuC4sQPLHUeSJO2lYgvwPpn5dOHx9ojolpmPAK8oUS5JO/G7+Uv50V+f4tITx3LJCWPLHUeSJDVDUXPAgWci4rDMXAg8Crw7ItYC60qWTNJLPLp8PZ/4zXyOHzeQz59zRLnjSJKkZiq2AP8qMBZYCHwJmAZ0A95XolySGlm3uYarpsyib48u/NBFl5IktWtFFeCZ+etGj++OiIFAt8zcVLJkkoAXF10uXbuF66+czLB+LrqUJKk9K3YE/CUyswaoaeEsknbi23c/xl8fX8VXLjiS48cNKnccSZK0j3ZZgEfEE0Du6QUy8+AWTSTpBX94aBnf/8tTvO2EMVx64rhyx5EkSS1gdyPgX261FJJe5vEVG/j4b+Zx7JgBfOFcF11KktRR7LIAz8xftGYQSS9at6Vh0WWvbl340WXH071LZbkjSZKkFlLsrehP3tWxzPxHy8WRVF+ffPTXc1m8ZjPXXTmZ/fq76FKSpI6k2EWYf99J24754Q7NSS3ov/74OH9+dCVfOu8IXjHeRZeSJHU0RW0mnJkVjT+A0cAvgItLmk7qZO58eDn//ecnufj40Vw22UWXkiR1RM26m0dmLgU+DHytZeNIndeTKzfw8Rvmcczo/nzp/COJiHJHkiRJJbAvt9PrDgxrqSBSZ7Z+aw1XTplFj64V/PCy4+nR1ZldkiR1VMUuwvxMk6bewPnA3S0dSOps6uuTj/16HotWb+ba95zIyAE9yx1JkiSVULGLME9v8nwDcAPwnZaNI3U+3/vzk/xx4Qq+cM7hnHjA4HLHkSRJJVZUAZ6Zp5Y6iNQZ/fGRFXznj49z4XGjeMfJ48sdR5IktYJ9mQMuaR88tWojH/31XI4a1Z//uOAoF11KktRJFFWAR8QhEXFnRKyOiO2NP0odUOqINmyt4cpfzqRrlwp+dLmLLiVJ6kyKnQM+FXgUuAzYXLo4UsdXX598/IZ5PLt6M1OuOIFRLrqUJKlTKbYAPwSYnJl1pQwjdQY/uOdJ7npkBf929uGcfOCQcseRJEmtrNg54A8CB5YyiNQZ/PnRFXzr7sc5/9iRvPuV48sdR5IklUGxI+DvAn4SEXcCyxofyMxftXgqqQN65rlNfPj6uRy2Xz++euHRLrqUJKmTKrYAfzPwOuAYXjoHPAELcGkPNm6r5cpfzqRLRfDjy4+nZzcXXUqS1FkVW4B/Bjg7M+8oZRipI8pMPvGbeTy1aiNTrjiRMYN6lTuSJEkqo2LngCdwZ3PfJCJ+FhErI2JBo7aLI+LhiKiPiEm7ufajhfMWRMR1EdGj0P7ziHgmIuYWPo5tbj6plH7416f4w4LlfPrMw3jlQS66lCSpsyu2AP8Z8M59eJ+fA2c0aVsAXAjcu6uLImIU8CFgUmYeCVQClzQ65ROZeWzhY+4+5JNK4p7HVvKNOx/jnGNG8p5X71/uOJIkqQ0odgrKJOCjEfExXr4I8w17ujgz742I8U3aFgLFLETrAvSMiBqgF7C0yMxSWVWt3sSHrpvDIcP78rU3e6dLSZLUoNgC/G+Fj1aVmdUR8U1gEbAFuCsz72p0ylci4nPAn4BPZea21s4o7cymbbVc+ctZRATXXD6JXt2K7WqSJKmjK6oqyMwvljrIzkTEQOA8YH9gLfCbiLgsM6cCnwaWA92Aa4BPAv++i9e5ErgSYOzYsaUPrk4tM7n6xvk8sXIDv3j3CYwd7KJLSZL0oqIK8Ig4eVfHMvMfLRfnZV4PPJOZqwo5bgJOBqZm5o6pMNsi4v+Af9lNxmtoKNKZNGlSljCvxDX3Ps3t85fxqTMP5dUThpY7jiRJamOK/bv433fStqOQLeWGxouAyRHRi4YpKKcBMwEiYkRmLouGibXn07CoUyqrvz2xiq/d8ShvOmoEV73mgHLHkSRJbVBRu6BkZkXjD2A08Avg4mKuj4jrgPuBQyJiSURcEREXRMQS4CTg9sJdNomIkRHx+8L7zgB+C8wGHirkvabwstdGxEOF9iHAl4v7kqXSeGrVRj543RwmDOvL1y/yTpeSJGnnIrN5MzIioi8wOzMntGyk0po0aVLOnDmz3DHUQWzcVssdC5Yzbc4S/vHUavp278KtH3gV44f0Lnc0SZJUZhExKzNfdr+bfdmaoTswbB+ul9ql2rp6/vbkc0ybXc1djyxna0094wb34kOvm8DFk0YzeqCLLiVJ0q4VuwjzM02aetMw7/rulg4ktUWZyYLq9dw0Zwm3zVvKcxu3M6BXVy46fjQXTBzNcWMHOOVEkiQVpdgR8NObPN8A3AB8p2XjSG3Lkuc3c8vcpUybU82TKzfSrbKC0w4bxvkTR3HqIcPo1qXYm8lKkiQ1KHYf8FNLHURqK9ZvreEPDy3jptnVzHhmDQAnjB/Ef1xwFG86agT9e3Utc0JJktSe7bYAj4jhwCmZecNOjr0FuCczV5YqnNRattfWc+/jq5g2p5q7F65ge209BwzpzcdPP5jzJ45izCDndUuSpJaxpxHwTwKrd3HsAOBE4OMtmkhqJZnJ3MVrmTanmtvmLeX5zTUM6t2Nt58wlgsmjuLo0f2d1y1Jklrcngrws4BTdnHs/4B7sQBXO7No9Wamzanm5rnVPPPcJrp3qeD0w4dzwcRRvObgoXStdF63JEkqnT0V4Ptl5oqdHcjMFRGxXwkySS1u7ebt3P7QMqbNrmZm1fNEwOT9B/O+Uw7kjKP2o18P53VLkqTWsacCfPuOW743PRARI4Ca0sSS9t222jr+8ugqps1Zwl8eXcX2unomDOvD1WccwvnHjmLkgJ7ljihJkjqhPRXg9wEfBJruAw7wfuBvLZ5I2geZyayq57lpTjW3z1/Gui01DOnTnctPGscFE0dxxMh+zuuWJElltacC/CvA3yJiKHAdUA2MAt4GXAq8qrTxpOI889wmps1ewrS51Sxes4WeXSt54xHDOX/iKF510BC6OK9bkiS1EbstwDNzZkScC3wfuAJIIIAngXMzc3bpI0o7t2bTdm6b13CTnLmL11IR8MqDhvCR0w7mjUfuR5/uxd5nSpIkqfXssULJzLuBgyNiAjAUWJWZT5Q8mbQTW2vq+NPClUybs4R7HltFbX1y2Ih+fOasQznv2FEM79ej3BElSZJ2q+ghwkLRbeGtVldfnzzw7Bqmza7m9w8tY8O2Wob3684Vr9qfC44bxaH79St3REmSpKL5N3q1WU+u3MBNs6u5Ze5SqtduoXe3Ss44cgQXTBzFSQcOprLCxZSSJKn9sQBXm7JqwzZunbeUm+dU81D1OiorgldPGMLVZxzC6YcPp1c3/5eVJEntm9WMym7L9jruemQ50+ZU87cnnqOuPjlqVH/+7ezDOfeYkQzt273cESVJklqMBbjKoq4+mf70am6aXc0dC5axaXsdowb05KrXHMAFE0cxYXjfckeUJEkqCQtwtapHl69nWmFe9/L1W+nbvQtnHz2SC44bxQnjB1HhvG5JktTBWYCr5Fas38otc6uZNmcpC5etp0tF8NpDhvLZsw/j9YcNp0fXynJHlCRJajUW4CqJTdtquWPBcm6eW819Tz5HfcKxYwbwxXOP4OyjRzC4j/O6JUlS52QBrhZTW1fPfU+tZtrsJdz58Aq21NQxZlBPPnDqQZw/cRQHDO1T7oiSJEllZwGufZKZPLx0PdPmVHPrvKWs2rCN/j27csFxo7hw4iiOHzeQCOd1S5Ik7WABrmZZunYLN8+tZtrsap5YuZGulcHrDh3GBRNHceqhw+jexXndkiRJO2MBrqJt2FrDHx5q2K97+jOryYRJ4wby5fOP5OyjRzCgV7dyR5QkSWrzLMC1WzV19fztiVXcNLuaux9ZwbbaesYP7sVHTjuYCyaOYuzgXuWOKEmS1K5YgOtlMpP5S9YxbU41t81byupN2xnYqytvfcUYLpg4imPHDHBetyRJUjNZgOsFi9ds5uY51UybW83TqzbRrUsFpx82nPMnjuKUg4fSrUtFuSNKkiS1exbgndy6zTXc/tAyps1ZwoPPPg/AifsP4spXH8CZR42gf8+uZU4oSZLUsViAd0Lba+u557GVTJtTzZ8WrmR7XT0HDu3NJ954COcdO5LRA53XLUmSVCoW4J1EZjJ70VqmzVnC7+YvY+3mGob06calk8dy4cTRHDmqn/O6JUmSWoEFeAf37HObmDanmpvnVlO1ejM9ulbwhsP344LjRvHqg4bQpdJ53ZIkSa3JArwDen7Tdn43fyk3zalmzqK1RMDJBw7mA6cexBlH7kffHs7rliRJKhcL8A5ia00df360YV73PY+tpKYuOWR4Xz515qGcd+xIRvTvWe6IkiRJwgK8XauvT2ZWPf/CvO4NW2sZ1rc77zx5PBdMHM1hI/o6r1uSJKmNsQBvh55atZFpsxvmdS95fgu9ulVyxhEN87pPPnAIlRUW3ZIkSW2VBXg78dzGbdw2bynT5lQzf8k6KgJeNWEo//KGQ3jDEcPp1c1vpSRJUntg1daGba2p465HVnDznGr++vgq6uqTw0f047NvOoxzjxnJsH49yh1RkiRJe8kCvI2pr0+mP72aaXOq+cOC5WzcVsuI/j3451cfwAUTR3HIfn3LHVGSJEn7wAK8jXh8xQZuml3NLXOrWbZuK326d+HMIxvmdU/efzAVzuuWJEnqEFqtAI+InwFnAysz88hC28XAF4DDgBMyc+Yurv0o8B4ggYeAd2Xm1ojYH7geGAzMAi7PzO2l/lpaysr1W7l13lJuml3NI8vWU1kRnHLwUD5z1mG8/rDh9OxWWe6IkiRJamGtOQL+c+B/gF82alsAXAj8eFcXRcQo4EPA4Zm5JSJuAC4pvN7XgO9k5vUR8SPgCuCHJUnfQjZvr+XOh5czbc5S/v7EKuoTjh7dn8+fczjnHDOSIX26lzuiJEmSSqjVCvDMvDcixjdpWwgUs1d1F6BnRNQAvYCl0XDR64C3F875BQ2j6W2uAK+rT+578jlunlPNHQ8vZ/P2OkYN6Mn/e+1BnD9xFAcN61PuiJIkSWolbX4OeGZWR8Q3gUXAFuCuzLwrIoYAazOztnDqEmBUuXLuzrUzqvjcLQ/Tt0cXzjt2JBdMHM2kcQOd1y1JktQJtfkCPCIGAucB+wNrgd9ExGXAHXvxGlcCVwKMHTu2BCl376yjRjCkT3ded+gwenR1XrckSVJnVlHuAEV4PfBMZq7KzBrgJuBkYDUwICJ2/BIxGqje2Qtk5jWZOSkzJw0dOrRVQjc2pE93zjpqhMW3JEmS2kUBvgiYHBG9CvO+TwMWZmYCfwEuKpz3DuCWMmWUJEmSitJqBXhEXAfcDxwSEUsi4oqIuCAilgAnAbdHxJ2Fc0dGxO8BMnMG8FtgNg1bEFYA1xRe9pPAxyLiSRq2Ivxpa309kiRJUnNEw0By5zFp0qScOXOn241LkiRJLSYiZmXmpKbt7WEKiiRJktRhWIBLkiRJrcgCXJIkSWpFFuCSJElSK7IAlyRJklqRBbgkSZLUiizAJUmSpFbU6fYBj4hVQFUZ3noI8FwZ3lety+9z5+D3uXPw+9zx+T3uHMr5fR6XmUObNna6ArxcImLmzjZiV8fi97lz8PvcOfh97vj8HncObfH77BQUSZIkqRVZgEuSJEmtyAK89VxT7gBqFX6fOwe/z52D3+eOz+9x59Dmvs/OAZckSZJakSPgkiRJUiuyAJckSZJakQW4JEmS1IoswKV9EBFdGj3uExGTImJQOTNJar6IGGQfllRqFuBSM0XEO4EVEfF4RJwJzAe+BsyLiLeVNZykokXE2Ii4vnCn5BnAAxGxstA2vszxJLWQiHio3Bl26LLnUyTtwseBQ4C+wDxgYmY+FRHDgbuB68oZTlLRfg38F3BpZtYBREQlcDFwPTC5fNEk7Y2IuHBXh4D9WjPL7rgNodRMETE3M48tPF6amSMbHZufmUeXLZykokXEE5k5YW+PSWp7IqIGuBbYWYF7UWb2beVIO+UIuNR8iyLiqzSMgD8aEd8CbgJeDywrazJJe2NWRPwA+AWwuNA2BngHMKdsqSQ1x3zgm5m5oOmBiHh9GfLslCPgUjNFRD/g/TT8lv0/wBuBdwFVwJcz0yJcagciohtwBXAeMKrQXA3cCvw0M7eVK5ukvRMRrwaqMnPRTo5NysyZZYj1MhbgkiRJUityFxSpmSKiMiKuiogvRcQrmxz7bLlySdo7EdErIq6OiE9ERI+IeEdE3BoRX4+IPuXOJ6l4EdGl8LP5joiYX/j4Q0S8NyK6ljvfDo6AS80UET8BegEPAJcDf83MjxWOzc7M48qZT1JxIuIGGuZ+96RhZ6OFNOyMci6wX2ZeXsZ4kvZCRFwHrKVhTceSQvNoGtZ0DMrMt5Yp2ktYgEvN1Hink8INeX4ADAHeBkzPzInlzCepODt2NIqIoGEB9YjMzMLzee5oJLUfEfF4Zh68t8dam1NQpObrtuNBZtZm5pXAXODPgH+2ltqZbBiR+n3h847njlJJ7cuaiLg4Il6ocSOiIiLeCjxfxlwvYQEuNd/MiDijcUNm/jvwf8D4siSS1Bwzd8z1zsx372iMiAOBDWVLJak5LgEu4sU7VT8OrAAuLBxrE5yCIknSLkREpD8opXYpIgYDZObqcmdpyhFwqQVFxDXlziBp3+3oyxbfUvuVmaszc3Vb/NlsAS61rEnlDiCpRdiXpY6jzfVnC3CpZa0sdwBJLcK+LHUcba4/OwdckiRJakWOgEsl0Bbnm0nae/ZlqX1pL3epdgRcaqaIGLSrQzTcvGN0a+aR1Dz2ZanjaC93qbYAl5opIuqAKhp+SO+QheejMrPbTi+U1KbYl6WOo73cpbpLuQNI7djTwGmZuajpgYhYXIY8kprHvix1HC+5SzVwZUR8jjZ2l2rngEvN91/AwF0c+3or5pC0b/4L+7LUUbSLu1Q7BUWSJElqRU5BkfZBRBwKnAeMKjRVA7dm5sLypZK0t+zLUsfRHvqzU1CkZoqITwLX07BQ64HCRwDXRcSnyplNUvHsy1LH0V76s1NQpGaKiMeBIzKzpkl7N+DhzJxQnmSS9oZ9Weo42kt/dgRcar56YORO2kcUjklqH+zLUsfRLvqzc8Cl5vsI8KeIeALYsVXZWOAg4APlCiVpr30E+7LUUXyEdtCfnYIi7YOIqABO4KULPR7MzLrypZK0t+zLUsfRHvqzBbjUgiLiysy8ptw5JO0b+7LUcbTF/uwccKllvbfcASS1CPuy1HG0uf5sAS61rCh3AEktwr4sdRxtrj87BUVqQRExOjOXlDuHpH1jX5Y6jrbYnx0Bl1rQjg4eEe8qdxZJzWdfljqOttifHQGXSiAiFmXm2HLnkLRv7MtSx9GW+rP7gEvNFBHzd3UIGN6aWSQ1n31Z6jjaS3+2AJeabzjwRuD5Ju0B/KP140hqJvuy1HG0i/5sAS413++APpk5t+mBiLin1dNIai77stRxtIv+7BxwSZIkqRW5C4okSZLUiizApWaKiKMjYnpELI6IayJiYKNjD5Qzm6Ti2ZeljqO99GcLcKn5fgB8ATgKeBz4e0QcWDjWtVyhJO01+7LUcbSL/uwiTKn5+mbmHYXH34yIWcAdEXE54OIKqf2wL0sdR7vozxbg0j6IiP6ZuQ4gM/8SEW8GbgQGlTeZpL1hX5Y6jvbQn52CIjXf14DDGjdk5nzgNOCmsiSS1Bz2ZanjaBf92W0IJUmSpFbkCLjUTBHRPyL+MyIejYg1EbE6IhYW2gaUO5+k4tiXpY6jvfRnC3Cp+W6g4Va3r83MQZk5GDi10HZDWZNJ2hv2ZanjaBf92SkoUjNFxGOZecjeHpPUttiXpY6jvfRnR8Cl5quKiKsjYviOhogYHhGfBBaXMZekvWNfljqOdtGfLcCl5nsrMBj4a0Q8HxFrgHto2OboLeUMJmmv2JeljqNd9GenoEj7ICIOBUYD0zNzY6P2MxrdCEBSG2dfljqO9tCfHQGXmikiPgTcAnwAWBAR5zU6/B/lSSVpb9mXpY6jvfRn74QpNd8/A8dn5saIGA/8NiLGZ+Z3gShvNEl7wb4sdRztoj9bgEvNV7HjT1uZ+WxEvJaGjj6ONtTJJe2RfVnqONpFf3YKitR8KyLi2B1PCh3+bGAIcFS5Qknaa/ZlqeNoF/3ZRZhSM0XEaKA2M5fv5NgrM/O+MsSStJfsy1LH0V76swW4JEmS1IqcgiJJkiS1IgtwSZIkqRVZgEuSJEmtyAJckjqJiLgnIrZFxIaIWBcRT0fElIg4fi9e49mIuKyUOSWpo7MAl6TO5UuZ2Tcz+wOnAlXA9Ii4oMy5JKnTsACXpE4qM6sy87PAL4HvRYMPR8SjhVHyRRHx1YioBIiI24CxwE8iYmNE3FVo7xIRn4mIxyNibUTcFxGTyveVSVLbZgEuSboeGAUcAiwBzgT6AecB7wbeA5CZ5wCLgPdkZp/MfEPh+i8Wzj0DGAz8DLgjIga25hchSe2FBbgkaUnh8+DMvDEzn8kGc4ApwGm7ujAiAvgQ8InMfDoz6zLzp8Ay4E0lTy5J7VCXcgeQJJXd6MLn1RHxNuBjwAE0/IzoBkzfzbVDgD7AbRHR+M5uXRu9riSpEQtwSdJbgWpgEzAVuBD4Q2Zuj4hvAo3nc9c3ufa5wnWvz8wHWyOsJLV3TkGRpE4qIsZExBeBdwIfpmEkuwJYBdRExGTg8iaXLQcm7HiSmQl8F/hmREwovG6fiHhjRIws/VchSe1PNPzbKUnq6CLiHuAkYDuQwGrgH8B3M/OBwjmfAz5Iw9STvwDPAsdm5msLx88CvgcMAqZn5pkR0YWGeeDvoWHaySYapq18MDN3zC+XJBVYgEuSJEmtyCkokiRJUiuyAJckSZJakQW4JEmS1IoswCVJkqRWZAEuSZIktSILcEmSJKkVWYBLkiRJrcgCXJIkSWpFFuCSJElSK/r/a+YjM5ZUSw4AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plot_daily_cumulative_accept_rate(df, 'Accepted', 'Rejected', \n", - " **{'allTime':True, 'shrinkTicks':True, 'showOutlier':True})" + "plot_daily_cumulative_accept_rate(\n", + " df,\n", + " \"Accepted\",\n", + " \"Rejected\",\n", + " **{\"allTime\": True, \"shrinkTicks\": True, \"showOutlier\": True},\n", + ")" ] }, { @@ -682,24 +191,18 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAFwCAYAAACRo0zvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA+B0lEQVR4nO3deZxcVZ3H/c+vt3QWQgKETcgiopBACNAgYTOBkUXQEJ0x5EFHQBYXYHR0EHhUwFE244yIC08GFxwZQBmIgjoDSmJAREhCiEAgcQRCIEASkpA9ne7z/FHVnepOL1W9VXX35/169atunbv9upoL3zqce26klJAkSZKUn7JiFyBJkiT1JgZoSZIkqQAGaEmSJKkABmhJkiSpAAZoSZIkqQAGaEmSJKkAFcUuoFB77LFHGj16dLHLkCRJUh83f/78VSmlEc3be12AHj16NPPmzSt2GZIkSerjIuLlltodwiFJkiQVwAAtSZIkFcAALUmSJBXAAC1JkiQVwAAtSZIkFcAALUmSJBXAAC1JkiQVwAAtSZIkFcAALUmSJBWg1z2JsBjeeHsLS95Yz/BBVQwfXMXwQZUMrCwnIopdmiRJknqYAToPjyxdxRd/8XSTtqqKMoYPqmT4oCqGNb5W7dQ2fHAlwwZVsdugKoYOrKS8zNAtSZLUmxmg83DSQXvy84sn8tbGbazdtI01m2qzrzuWl765oXFdXX1q8TgRsOvA5qE7G7QHVWYDeM7y4My66sryHv6NJUmS1BoDdB52G1zF0WN2y2vblBLrt25n7cZa1mzaxlubsqF7Y21jwF6zaRtrN9XyxttbeOH19azZtI1N2+paPebAyvImobppT3cLvd6DqtiluoIye7slSZK6nAG6i0UEQ6srGVpdycjdB+W935baOtZtzoTrhrD9VjZor9nYtNd7xdq3MyF8cy2p5c5uysuCYQMrdx5eMrjtXu+qCu8rlSRJaosBukRUV5ZTXVnOXkOr896nvj7x9pbanF7tTPhe02x4yZqNtSxfs4lnXs2s27q9vtVjDq4qz94o2VLQbgjgTXu9hwyo8IZKSZLUbxige7GysmBYtnd5DIPz3m/ztrrGkL12U21j2M70dDdtW/bWJtZs3MbbW7a3erzK8mgM1W0OLxm8Y5thAyupKLe3W5Ik9T49EqAjYn/gp8BeQAJmppRujojdgLuB0cBLwEdTSmt6oqb+bGBVOQOrBrLvsIF577O9rj47xKT5WO5tvLWx6U2VL67ayIJNa1m7aRu1da2MMQF2qa5oMjVgazdV5oZvpw+UJEnF1lM90NuBL6SUFkTELsD8iHgIOBf4fUrphoi4ArgC+FIP1aQCVJSXsfuQAew+ZEDe+6SU2LitjjUbc3u1c5Y37gjiqzds469vbmDtplo2bG29tzt3+sDhbdxUmdu268BKb6iUJEldpkcCdEppBbAiu7w+IhYD7wCmAJOym90OzMEA3WdEBEMGVDBkQAX75zeJCQDbttezdnNLN1Du6Olu6PV+4fX1rN1Uy9rN+U8fuFsrN1XmzmIybFCl0wdKkqQW9fgY6IgYDRwO/BnYKxuuAV4nM8SjpX0uAi4CGDlyZA9UqWKqqihjz12q2XOXwm6oXL91+07DS5pPH7hm0zZWrNvC4hVvs2ZTLZtr254+cLcWA3bTebobQ/fgSnbxhkpJkvq8Hg3QETEE+G/gcymlt3ODRkopRUSLXYgppZnATICamprWB9Wq3yorC3YdWMmuAysZtXv++22prWtxeElD7/dbOW2vrt3Mmk3bWNfG9IEVZcGwZkNIGsJ1S1MHDh/sDZWSJPU2PRagI6KSTHi+I6V0b7b5jYjYJ6W0IiL2Ad7sqXokyEwfuPeu5ey9a/693XX1ibc350wXuLF2p9lLGoaavLx6EwtfWcvaTbVsq2t9+sDGGyqbTRnoDZWSJJWenpqFI4AfAotTSv+Ws+pXwCeAG7Kvv+yJeqTOKC+LTIgdXJX3PiklNmWnD2xv6sA1m7bxt1UbWLOx7RsqB1SUNR1aMnjnnu8dbZn2odXeUClJUmf1VA/0ccDHgb9ExMJs21VkgvPPI+KTwMvAR3uoHqlHRQSDB1QweEAF+w3Pf7+2bqjMnckk3xsqy3JuqMydl7ulp1LulrPeJ1RKkrRDT83C8SjQWrfXyT1Rg9QbdfaGyrc2tjy0pGH51bVbePa1zKPht9S2/YTK3Bsnhw2qYjefUClJ6qd8EqHUxzS9oTL/J1Ruqa1rHNPd0Mv91qZtrN24c/h+5a1NvNXZJ1TuNMY781ruEBNJUokzQEsCMjdU7rPrQPbZtbhPqBxaXZGdPrCFB+Rkh5U0zuWd7RF3zm5JUk8yQEvqsM4+obLJ0JIWxniv3LCVJW9sYO2mbWzc1vqc3dWVZS30bjfr8c654bKqooxEZrgLQH1KpJR9zdaYeQ+JRH195jUlmmzXsF9qeF+/o53c/dOOY+44Xgv759TRtK5m+zerI+XW39r+7DhXm/s3qb+N/XPqaHN/Wqi/voX92/z9M9uS2vjsm31WkLnRdmBVOYOqKjKvleUMqipnYFUFAxuXM68N7YOqyhlYmdNemdnX+wAk5TJAS+pRTZ9QOSjv/bZur2PdptqderkbH5qTE74Xr3g7c0Plpm20cj+lckRAWQRB5pXI3HAaROY1gmwzZWU7touGdTnvy7Lj3svKduy/45it7JuzLZE9Z4vbBRURLZw7s5wZ/dPwClu217NlWx1vrt/Cpm11bN5W1/ja1rSSLakoiybBuiFsNw/cAysrdgrm1ZWZEN+kvbKC6qqyTLivLHfoktTLGKAl9QoDKsrZc2g5ew4t7IbKt7c0Dd1rNtZSW1e/c6jLhrjICWM7AmQrQbHFUNg0eLa5f7P3kQ2uNN+fHcdusj07wm2LAbi1/WkaTvuj7XX1bK7dEao3batjc+32xuUttTnt21ppr93Ohq3bWbl+a/Z9w/G2F/zFbUBFWTZwZ3q8m/SCNwvmDb3nTXrNW2rPHmNARVm//TtL3cUALanPKivL3Mg4bFAVY8j/hkr1fRXlZexSXsYu1ZVdfuyUElu312fCdG0mgG/eVs+mbduz75sG89aC/OZtdby1cRvL1+wI5pu21bF1e2G952VBNpDn9JrnBPOG4S3Nh7O0OMwlN8hn9/NJquqPDNCSJHWhiKC6MjN0o4Bp3/NWV58ae8IzIX170yEqtU17zVtq35zd/62NmzMBP+d42wvsPq8qL8sZ2lKeE7wrcsadN2tva5hLY096BdWV9p6rNBmgJUnqRcrLdjyYqTtsa+w93xHMdwTspsF8p/Zsb/rmbXWs21zL6+s27xTiC9W8Jzx3eMrAynIqyoPysqCiLCgvK8u+Zt+Xt9LeuD7TXlEW2eO0tF1Z5rW8lfaG9+WttGdf/SLQtxigJUlSo6qKMqoqytiVrh/eUl+fGd6yqVlP+KZt25sF8jaGuWR70t94u5YttXXU1Se216emr3X1O7UXW3kLAb6ivLOBvZUvBmVlLey/Y/uKlr5YtPVFpL0vGOU7f2HIrbesD94ka4CWJEk9oqwsGsdP796D500ptRy067NBu66V9sZA3kp7faKuvr6F/bPt9Ym6ukRts/ctbtfwvq7l9s21dTudr9Wac85RW19PKvL3hwiyPf1lzf7PwM5fDFoK6/vsWs3NZx9e3F+iGQO0JEnq0yKyva799JlL9W19OWgM3B34crDTl4+moX97XbMvB43bt9De6vkydZUaA7QkSVIfVlYWVDUOo+in3yK6mHPPSJIkSQUwQEuSJEkFMEBLkiRJBTBAS5IkSQUwQEuSJEkFMEBLkiRJBch7GruIeCewG/AW8GJKxZ6WW5IkSep5bfZAR8QREXF7RKwG/go8kX1dnW0/oieKlCRJkkpFqwE6In4K3Ae8AnwY2AOoyr6eBSwD7s1uJ0mSJPULbQ3h+CNwXkqprln7W8BcYG5EXAN8sptqkyRJkkpOqwE6pfT/tbdzNlzP7NKKJEmSpBKW902EDSLiZOBw4K8ppVldXpEkSZJUwgqaxi4irgK+CuwOXBER9j5LkiSpX2mzBzoijkgpLchpOgWYlFJKETEQeBW4qDsLlCRJkkpJez3Q/xER10VEVfb9SuDjEXEgcCHwUncWJ0mSJJWa9gL0e4GtwJMRMRG4DDgNmAWcDJyTz0ki4kcR8WZEPJPTNiEiHo+IhRExLyKO7tBvIEmSJPWgNodwpJS2A9dGxH8D/0HmQSoXpJQ2FXienwDfBXLnjL4JuDal9NuI+ED2/aQCjytJkiT1qLxuIkwpPQMcR+ahKk9mZ+LIW0ppLpn5o5s0A0Ozy7sCrxVyTEmSJKkY2nuU98SIeDoiNgCPAg8AU4AvR8RtETG0rf3b8TngmxHxCjADuLITx5IkSZJ6RHs90D8GriEzbd23gJtTSn9NKU0G5gN/7sS5Pw18PqW0P/B54IetbRgRF2XHSc9buXJlJ04pSZIkdU6klFpfGbEKGJNSWh8Rw4Hfp5SOyFm/X0ppeV4nihgNPJBSOiT7fh0wLDslXgDrUkrt9mjX1NSkefPm5XNKSZIkqcMiYn5KqaZ5e3s90LcC8yPiDjI3EN6auzLf8NyK14D3ZZdPApZ24liSJElSj2hvFo4vZ2fgeBfw9ZTS4o6cJCLuJDPDxh4RsRy4msw80jdHRAWwBR/IIkmSpF6gzQANkFJ6CniqMydJKU1vZdWRnTmuJEmS1NNaHcIREddGxOC2do6IIRFxbdeXJUmSJJWmtnqgBwAvRsQvgYeA54C3yczdPBb4O+As4LZurlGSJEkqGa0G6JTSFRHxfeBTZKayO4jMw08CeIHM47xrUkrLur9MSZIkqTS0dxPhMuAq4KqIqAaGA2tSSlt6ojhJkiSp1LR7E2GDbGhe0Y21SJIkSSWvvXmgJUmSJOUwQEuSJEkFMEBLkiRJBTBAS5IkSQXIO0BHxMci4qGIWJR9f2JEfLj7SpMkSZJKT14BOiL+GbgW+C0wMtu8Eri8m+qSJEmSSlK+PdCfBk5PKf0bmYepACwB3tUtVUmSJEklKt8AvVtKaUl2uSFAR86yJEmS1C/kG6Cfi4gzm7WdBjzdxfVIkiRJJS3fJxFeBfw6In4ODIiIW4CzgeahWpIkSerT8uqBTik9AhwDbAZmZ/eblFL6czfWJkmSJJWcvHqgI2J0Suk54NJm7aNSSi93S2WSJElSCcp3DPSiVtqf6qpCJEmSpN4g3wAdOzVEVOIsHJIkSepn2hzCEREPkQnJAyLiwWarRwILuqswSZIkqRS1Nwb60ezr+4A/5rTXA68Dv+iOoiRJkqRS1WaATildCxARi1NKP++ZkiRJkqTSldcsHA3hOSKqgRHkjIlOKS3rntIkSZKk0pPvNHbvBH4GvLeF1eVdWpEkSZJUwvKdheO7wCvAYcB6YDwwC/hk95QlSZIklaZ8H+X9XmB0Sml9RJBSejYiLgb+APyk26qTJEmSSky+PdD1ZB7jDbAhIoYBb5GZyk6SJEnqN/IN0M8Cx2WX/wz8O/Ad4MV8do6IH0XEmxHxTLP2SyPi+Yh4NiJuyrdoSZIkqVjyDdCXkelxBvgX4B1ADXBxnvv/BDgttyEiJgNTgMNSSuOAGXkeS5IkSSqafKexW5Sz/DfgFICI2DXP/edGxOhmzZ8Gbkgpbc1u82Y+x5IkSZKKKd8e6CYiojoiLgf+1olzvxs4ISL+HBF/iIijOnEsSZIkqUe0GaAjYnREPBwRb0fEExHx7og4FVhKZgq7yzpx7gpgN+AYMsNCfh4R0dKGEXFRRMyLiHkrV67sxCklSZKkzmmvB/rfsq9XkJmF4xfAbcBXgINTSnd04tzLgXtTxhNkZvrYo6UNU0ozU0o1KaWaESNGdOKUkiRJUue0Nwb6WGBcSml1RNwLvAbUpJQWdMG5ZwGTgdkR8W6gCljVBceVJEmSuk17AXpQSmk1QErp9YjY0JHwHBF3ApOAPSJiOXA18CPgR9mp7bYBn0gppUKPLUmSJPWk9gJ0RMQ+QMPY5Lpm70kpvdbeSVJK01tZ9bG8qpQkSZJKRHsBejCZscoNIud9AAko74a6JEmSpJLUXoAe0yNVSJIkSb1EmwE6pfRyTxUiSZIk9QYdepCKJEmS1F8ZoCVJkqQCGKAlSZKkAuQVoCNidCvto7q0GkmSJKnE5dsDvaiV9qe6qhBJkiSpN8g3QMdODRGVZOaBliRJkvqNNqexi4iHyITkARHxYLPVI4GCH+stSZIk9WbtPUjl0ezr+4A/5rTXA68Dv+iOoiRJkqRS1d6DVK4FiIjFKaWf90xJkiRJUulqrwcagJTSzyNiCHAmsB+wHPh1Sml9dxYnSZIklZq8AnRE1AC/ATYDy8iMf/5ORHwgpTSvG+uTJEmSSkq+s3B8H/hWSmlUSumElNIoYAbwg+4rTZIkSSo9+Qbog4FvNWv7N+Cgri1HkiRJKm35BuiFwCHN2g7NtkuSJEn9Rl5joIEHgQci4jbgZWA0cD4wMyL+n4aNUkr/1eUVSpIkSSUk3wB9PlALfCKnbXu2vUECDNCSJEnq0/Kdxm5MdxciSZIk9Qb59kADEBH7AiNTSo93Uz2SJKnE1NbWsnz5crZs2VLsUqRuUV1dzX777UdlZWVe2+c7D/SeZIZnnARsAoZExDTgfSmlz3S0WEmSVPqWL1/OLrvswujRo4mIYpcjdamUEqtXr2b58uWMGZPfoIt8Z+H4DvAiMILMWGiAh4H3F1ylJEnqVbZs2cLuu+9ueFafFBHsvvvuBf0flnyHcEwGRqWUtkREAkgprcz2TEuSpD7O8Ky+rNB/vvPtgd5Ks7AdEbsBbxV0NkmSJKmXyzdAPwh8KyJyR1ZfC/y660uSJEna2euvv87ZZ5/NAQccwJFHHskHPvABlixZ0qFjnXvuudxzzz0AXHDBBTz33HMAXHfddd12ToBrrrmGGTNmdHj/Bj/5yU+45JJL2txmzpw5nHnmmZ0+VyFyP9d8/PKXv+Sss85qfH/99dfzrne9q/H9/fffz4c+9KFW9++qz7NQ+Qboy8k8znsNMDQi1gLjgS93U12SJEmNUkpMnTqVSZMm8X//93/Mnz+f66+/njfeeKNxm+3bt3fo2Lfddhtjx44FmgbofM7ZXs319fUdqqm/OPbYY3n88R2Tu/3pT39i6NChvPnmmwA89thjHHvsscUqr1X5zgP9FnBiRBwJjCHzNMJ5KaWUz/4R8SPgTODNlNIhzdZ9AZgBjEgprSqkeEmS1LOuvf9Znnvt7S495th9h3L1B8e1uc3s2bOprKzkU5/6VGPbYYcdxpw5czjhhBMYPnw4zz//PIsXL+aKK65gzpw5bN26lc9+9rNcfPHFpJS49NJLeeihh9h///2pqqpqPM6kSZOYMWMG99xzD5s3b2bChAmMGzeOT37yky2eE2DDhg1MmTKFNWvWUFtby9e//nWmTJnCSy+9xKmnnsp73/te5s+fz29+8xt+9rOfcfvtt7Pnnnuy//77c+SRRwLwne98h1tvvZWKigrGjh3LXXfd1aHP79xzz2Xo0KHMmzeP119/nZtuuom///u/B+Dtt9/mjDPO4K9//SuTJ0/m+9//PmVlZdx5551cd911pJQ444wzuPHGGwEYMmQIGzZsAOCee+7hgQce4Cc/+Umr52jrc83HiBEjGDp0KH/9619517vexauvvspHPvIRHnvsMc466ywee+wxvv71r7Ny5Uo+9alPsWzZMgC+/e1vc9xxxwHw9NNPM3HiRFatWsXll1/OhRde2KHPsRD5TmM3HNiWUpoPzM+2DY6IypTS2jwO8RPgu8BPmx13f+AUYFkBNUuSpH7mmWeeaQyezS1YsIBnnnmGMWPGMHPmTHbddVeefPJJtm7dynHHHccpp5zCU089xQsvvMBzzz3HG2+8wdixYzn//PObHOeGG27gu9/9LgsXLgQyAbe1c1ZXV3PfffcxdOhQVq1axTHHHNM41GDp0qXcfvvtHHPMMcyfP5+77rqLhQsXsn37do444ojGY95www28+OKLDBgwgLVr13bq81mxYgWPPvoozz//PB/60IcaA/QTTzzBc889x6hRozjttNO49957OfbYY/nSl77E/PnzGT58OKeccgqzZs1qMpQi33Pcd9997X6u7TnuuON47LHHqKur48ADD+SYY47hf//3fznzzDN5+umnOeqoozj//PP5/Oc/z/HHH8+yZcs49dRTWbx4MQCLFi3i8ccfZ+PGjRx++OGcccYZ7Lvvvh36HPOV7ywcvwK+ADyR03YIcBPwvvZ2TinNjYjRLaz6dzLDQ36ZZx2SJKmI2uspLoajjz66cf7eBx98kEWLFjWOw123bh1Lly5l7ty5TJ8+nfLycvbdd19OOumkTp0zpcRVV13F3LlzKSsr49VXX20c2jFq1CiOOeYYAB555BGmTp3KoEGDAJqM5x0/fjznnHMOZ511VrvhtT1nnXUWZWVljB07tskQk6OPPpp3vvOdAEyfPp1HH32UyspKJk2axIgRIwA455xzmDt3brs1tHSOrvhcjz322MYAPXHiRI4++mi+9rWv8dRTT3HQQQdRXV3N7373u8Zx6pDpWW/oKZ8yZQoDBw5k4MCBTJ48mSeeeKLTn2d78g3Q44B5zdrmAYd29MQRMQV4NaX0tFPjSJKktowbN67Vm9MGDx7cuJxS4pZbbuHUU09tss1vfvObLj3nHXfcwcqVK5k/fz6VlZWMHj26cR7h3Hra8utf/5q5c+dy//33841vfIO//OUvVFQU9JDoRgMGDGhczh1h2zxjtZe5ctc3nxe5tXN01nHHHcctt9xCXV0dF154Ibvssgtbtmxhzpw5jeOf6+vrefzxx6murm6z5pbed4d8byLcAgxq1jaYHQ9VKUhEDAKuAr6a5/YXRcS8iJi3cuXKjpxSkiT1YieddBJbt25l5syZjW2LFi3ikUceabLdqaeeyg9+8ANqazMRZcmSJWzcuJETTzyRu+++m7q6OlasWMHs2bNbPE9lZWXjvm2dc926dey5555UVlYye/ZsXn755RaPd+KJJzJr1iw2b97M+vXruf/++4FMIHzllVeYPHkyN954I+vWrWvsUe1KTzzxBC+++CL19fXcfffdHH/88Rx99NH84Q9/YNWqVdTV1XHnnXfyvvdlBhTstddeLF68mPr6eu677752j5/v59qWgw8+mNdee41HH32Uww8/HIAJEyZw6623No5zPuWUU7jlllsa92kYZgOZmTy2bNnC6tWrmTNnDkcddVTBNRQq3wD9KHBdRJQBRCbafw34YwfPewCZmxGfjoiXgP2ABRGxd0sbp5RmppRqUko1Df+7QZIk9R8RwX333cfvfvc7DjjgAMaNG8eVV17J3ns3jQ4XXHABY8eO5YgjjuCQQw7h4osvZvv27UydOpUDDzyQsWPH8o//+I9MnDixxfNcdNFFjUMr2jrnOeecw7x58zj00EP56U9/ykEHHdTi8Y444gimTZvGYYcdxumnn94Y7urq6vjYxz7GoYceyuGHH85ll13GsGHDuvQzAzjqqKO45JJLOPjggxkzZgxTp05ln3324YYbbmDy5MkcdthhHHnkkUyZMgXIjMs+88wzOfbYY9lnn33aPX6+n2tbIoL3vve97L777lRWZmZMnjhxIn/7298ae6C/853vMG/ePMaPH8/YsWO59dZbG/cfP348kydP5phjjuErX/lKt49/Boh8uuAjYhSZR3dXA38jE363ASellF7K60SZMdAPNJ+FI7vuJaAmn1k4ampq0rx5zUeTSJKk7rJ48WIOPvjgYpchdauW/jmPiPkppZrm2+Y7jd3LEXEImanoRgMvAb9OKW3KZ/+IuBOYBOwREcuBq1NKP8xnX0mSJKmU5D1SPaW0GfhFR06SUprezvrRHTmuJElSX/HjH/+Ym2++uUnb0qVLOfDAA5u0HXfccXzve9/rydIKNnXqVF588cUmbS+//DKjRo1q0nbjjTfudMNnb5DvPNAPAjeklB7OaTsZ+GJK6fTuKk6SJKm/OO+88zjvvPOKXUaXyOcGxN4s35sIjwDmNmt7BOj+2xwlSZKkEpJvgK4HKpu1lQNO4CxJkqR+Jd8APR+4tFnbJcCCri1HkiRJKm353kT4JWBORHwEWAIcCLyHzMwakiRJUr+RVw90SmkRMBa4B3gb+G9gbErp6W6sTZIkqdHrr7/O2WefzQEHHMCRRx7JBz7wAZYsWdKhY5177rmNj+m+4IILeO655wC47rrruu2cANdccw0zZszo8P65Zs2axfjx4zn44IM59NBDmTVrVrv7LFy4sMljzbuynnyNHj2aVavaffRHE9/4xjcYN24c48ePZ8KECfz5z38GYNKkSYwcObLJo8XPOusshgwZAmSe+HjZZZdxyCGHcOihh3LUUUftNDtIRxQyjd3rwDcb3kfEuIi4MqV0WaerkCRJakNKialTp/KJT3yCu+66C4Cnn36aN954g3e/+90AbN++nYqKvKNNo9tuu61x+brrruOqq67K+5zt1ZxSoqws3xGz+Xv66af54he/yEMPPcSYMWN48cUXef/738873/lOxo8f3+p+CxcuZN68eXzgAx/o8pq6y5/+9CceeOABFixYwIABA1i1ahXbtm1rXD9s2DD++Mc/cvzxx7N27VpWrFjRuO7uu+/mtddeY9GiRZSVlbF8+XIGDx7c6ZoK+qcsIgYA04CLgYnAY52uQJIk9R6/vQJe/0vXHnPvQ+H0G9rcZPbs2VRWVvKpT32qse2www5jzpw5nHDCCQwfPpznn3+exYsXc8UVVzBnzhy2bt3KZz/7WS6++GJSSlx66aU89NBD7L///lRVVTUeZ9KkScyYMYN77rmHzZs3M2HCBMaNG8cnP/nJFs8JsGHDBqZMmcKaNWuora3l61//OlOmTOGll17i1FNP5b3vfS/z58/nN7/5DT/72c+4/fbb2XPPPdl///058sgjgczjqW+99VYqKioYO3ZsY0jPx4wZM7jqqqsYM2YMAGPGjOHKK6/km9/8Jv/5n//Z+DvV1NSwatUqampqWLJkCV/96lfZvHkzjz76KFdeeSWQCeMTJ05k1apVXH755Vx44YWklLj88sv57W9/S0Tw5S9/mWnTpjFnzhxmzJjBAw88AMAll1xCTU0N5557LqNHj+YTn/gE999/P7W1tfziF7/goIMOYvXq1UyfPp1XX32ViRMnks9TsHOtWLGCPfbYgwEDBgCwxx57NFl/9tlnc9ddd3H88cdz77338uEPf5hnn322cd999tmn8UvMfvvtV9C5W5PXV6KIGBsRNwOvAf8fUAOcllI6vkuqkCRJasMzzzzTGDybW7BgATfffDNLlizhhz/8IbvuuitPPvkkTz75JP/xH//Biy++yH333ccLL7zAc889x09/+lMee2znPsAbbriBgQMHsnDhQu644442z1ldXc19993HggULmD17Nl/4whcag+HSpUv5zGc+w7PPPsuqVau46667GodOPPnkk03O99RTT7Fo0SJuvfXWgj6PZ599dqfaampqGoNjS6qqqvja177GtGnTWLhwIdOmTQNg0aJFPPzww/zpT3/ia1/7Gq+99hr33nsvCxcu5Omnn+Z3v/sd//Iv/9KkZ7c1e+yxBwsWLODTn/5049CQa6+9luOPP55nn32WqVOnsmzZsoJ+11NOOYVXXnmFd7/73XzmM5/hD3/4Q5P1J598MnPnzqWuro677rqr8fcC+OhHP8r999/PhAkT+MIXvsBTTz1V0Llb02YPdER8HLgIOA5YBFwD3AE8CyzskgokSVLv0U5PcTEcffTRjT2xDz74IIsWLWoc37xu3TqWLl3K3LlzmT59OuXl5ey7776cdNJJnTpnSomrrrqKuXPnUlZWxquvvsobb7wBwKhRozjmmGMAeOSRR5g6dSqDBg0C4EMf+lDjMcaPH88555zDWWedxVlnndWpejpjypQpDBw4kIEDBzJ58mSeeOIJHn300cbPa6+99uJ973sfTz75JEOHDm3zWB/+8IcBOPLII7n33nsBmDt3buPyGWecwfDhwwuqb8iQIcyfP59HHnmE2bNnM23aNG644QbOPfdcAMrLyzn++OO566672Lx5M6NHj27cd7/99uOFF17g4Ycf5uGHH+bkk0/mF7/4BSeffHJBNTTX3hCO24HVwBkppd82NEY4/bMkSeo548aNawzFzeWOaU0pccstt+z0eOjcG+e64px33HEHK1euZP78+VRWVjJ69Gi2bNmyUz1t+fWvf83cuXO5//77+cY3vsFf/vKXvMdwjx07lvnz5zcOKQGYP38+48aNA6CiooL6+nqAxrpa0zzXtZXzco/b0rEbhlmUl5ezffv2PH6T/JSXlzNp0iQmTZrEoYceyu23394YoCEzjGPq1Klcc801O+07YMAATj/9dE4//XT22msvZs2a1ekA3d4Qjq8AG4BZEXFfRHwwIrp+JLwkSVIbTjrpJLZu3crMmTMb2xYtWsQjjzzSZLtTTz2VH/zgB9TW1gKwZMkSNm7cyIknnsjdd99NXV0dK1asYPbs2S2ep7KysnHfts65bt069txzTyorK5k9ezYvv/xyi8c78cQTmTVrFps3b2b9+vXcf//9QGZ2iFdeeYXJkydz4403sm7dOjZs2JD35/HFL36R66+/npdeegmAl156ieuuu44vfOELQGami/nz5wM0+RKwyy67sH79+ibH+uUvf8mWLVtYvXo1c+bM4aijjuKEE05o/LxWrlzJ3LlzOfrooxk1ahTPPfccW7duZe3atfz+979vt9YTTzyR//qv/wLgt7/9LWvWrMn79wR44YUXWLp0aeP7hQsXMmrUqCbbnHDCCVx55ZVMnz69SfuCBQt47bXXgMxnvmjRop327Yg2v+aklL4REdcBpwEXkpm+bjUwDNgXeLPTFUiSJLUjIrjvvvv43Oc+x4033kh1dTWjR4/eaejDBRdcwEsvvcQRRxxBSokRI0Ywa9Yspk6dysMPP8zYsWMZOXIkEydObPE8F110EePHj+eII47gjjvuaPGc3/72tznnnHP44Ac/yKGHHkpNTQ0HHXRQi8c74ogjmDZtGocddhh77rknRx11FAB1dXV87GMfY926daSUuOyyyxg2bFjen8eECRO48cYb+eAHP0htbS2VlZXcdNNNTJgwAcgE7I9+9KPMnDmTM844o3G/yZMnc8MNNzBhwoTGmwjHjx/P5MmTWbVqFV/5ylfYd999mTp1Kn/605847LDDiAhuuukm9t57byAzrviQQw5hzJgxHH744e3WevXVVzN9+nTGjRvHsccey8iRI/P+PSFzw+all17K2rVrqaio4F3veleTLzWQ+efji1/84k77vvnmm1x44YVs3boVyAz3ueSSSwo6f0uikDshI2If4ALgfGA/4L6U0kc7XUUBampq0rx583rylJIk9WuLFy/m4IMPLnYZUrdq6Z/ziJifUqppvm1BwzFSSitSSv8KvBOYAlS1s4skSZLUpxQ+2ziQMt3Wv8n+SJIkqZN+/OMfc/PNNzdpW7p0KQceeGCTtuOOO47vfe97PVlal1u9evVON/LV1dUBmRsGc/3+979n991377Ha8tGhAC1JkvqXlJKzcHWz8847j/POO6/YZfSI3XffnYULFxa7jEaFPtzFGTUkSVKbqqurWb16dcEhQ+oNUkqsXr2a6urqvPexB1qSJLVpv/32Y/ny5axcubLYpUjdorq6uqDHfBcUoCPz/272Tim1/yxHSZLUJ1RWVjY+6U9SnkM4ImJIRPwQ2Az8Ndt2VkRc3Z3FSZIkSaUm3zHQ3wL2Ao4DtmXbngSmdUdRkiRJUqnKdwjHmcDYlNK6iEgAKaVXI2Lf7itNkiRJKj359kCXkRm+0SgihgD5P7RdkiRJ6gPyDdCPAlc2a7sUmN215UiSJEmlLd8hHP8MPBwRHwOGRMRfyDzG+6Ruq0ySJEkqQXkF6JTSKxFxCPBBYDTwMvBASmlzmztKkiRJfUxeAToiRqaUlgH3dOQkEfEjMjcivplSOiTb9k0ygXwb8H/AeSmltR05viRJktRT8h0D/beIeCgizo6IAR04z0+A05q1PQQcklIaDyxh5zHWkiRJUsnJN0AfCDwGXA+siIjvR0RNvidJKc0F3mrW9mBKaXv27eNA/s9PlCRJkookrwCdUnoxpXR1SmkM8A/AEGB2RDzdRXWcD/y2i44lSZIkdZt8Z+HINQcYCuwPnNjZAiLi/wW2A3e0sc1FwEUAI0eO7OwpJUmSpA7LdwgHETE+Iv4deA24Gfgj8J7OnDwiziVzc+E5KaXU2nYppZkppZqUUs2IESM6c0pJkiSpU/KdheMpMmH5V8A/Ag+llOo7c+KIOA24HHhfSmlTZ44lSZIk9ZR8h3D8B/BfHZ1mLiLuBCYBe0TEcuBqMrNuDAAeigiAx1NKn+rI8SVJkqSeku+DVL7fmZOklKa30PzDzhxTkiRJKoZWA3RE/DKlNCW7/GBr26WUTumOwiRJkqRS1FYP9OM5y48Brd7kJ0mSJPUXrQbolNL1OcvX9Eg1kiRJUonLaxq7iFjcSvtfurYcSZIkqbTlOw90a4/Z9vHbkiRJ6lfanIUjIq5q2C5nucG7gFe6pSpJkiSpRLU3jd37s6+VOcsA9cDrwPndUZQkSZJUqtoM0CmlyQARcUtK6dKeKUmSJEkqXXmNgTY8S5IkSRl5PYkwIgYCXwZOBkYA0bAupfTO7ilNkiRJKj35zsLx78AU4D+BvYBvAVuBH3VTXZIkSVJJyjdAfxD4UErpe8D27OtHgMndVpkkSZJUgvIN0ENSSn/LLm+LiKqU0nPAUd1UlyRJklSS8hoDDbwYEQenlBYDzwPnR8RaYF23VSZJkiSVoHwD9PXASGAx8K/AfUAV8OluqkuSJEkqSXkF6JTS3TnLD0XEcKAqpbSx2yqTJEmSSlC+PdBNpJRqgdourkWSJEkqea0G6IhYCqT2DpBSeneXViRJkiSVsLZ6oL/eY1VIkiRJvUSrATqldHtPFiJJkiT1Bvk+yvvY1tallB7runIkSZKk0pbvTYSPttDWMD66vItqkSRJkkpeXk8iTCmV5f4A+wG3A//QrdVJkiRJJSbfR3k3kVJ6Dfgn4MauLUeSJEkqbR0K0FkDgD27qhBJkiSpN8j3JsKrmjUNBs4CHurqgiRJkqRSlu9NhO9v9n498HPg37u2HEmSJKm05RWgU0qTu7sQSZIkqTfozBjovEXEjyLizYh4Jqdtt4h4KCKWZl+H90QtkiRJUmfkFaAj4j0R8b8RsToituX+5HmenwCnNWu7Avh9SulA4PfZ95IkSVJJy3cM9M+A54GPAZsKPUlKaW5EjG7WPAWYlF2+HZgDfKnQY0uSJEk9Kd8A/R7gmJRSXReee6+U0ors8uvAXq1tGBEXARcBjBw5sgtLkCRJkgqT7xjoJ4EDuquIlFJix6PBW1o/M6VUk1KqGTFiRHeVIUmSJLUr3x7o84DbIuJ/gRW5K1JK/9XBc78REfuklFZExD7Amx08jiRJktRj8g3QHwFOAg6j6RjoBHQ0QP8K+ARwQ/b1lx08jiRJktRj8g3QVwFnppT+pyMniYg7ydwwuEdELAeuJhOcfx4RnwReBj7akWNLkiRJPSnfAJ2A/+3oSVJK01tZdXJHjylJkiQVQ743Ef4IOLcb65AkSZJ6hXx7oGuAz0fEP7PzTYSndHlVkiRJUonKN0A/kv2RJEmS+rW8AnRK6druLkSSJEnqDfIK0BFxbGvrUkqPdV05kiRJUmnLdwjHoy20NTw5sLyLapEkSZJKXl6zcKSUynJ/gP2A24F/6NbqJEmSpBKT7zR2TaSUXgP+Cbixa8uRJEmSSluHAnTWAGDPripEkiRJ6g3yvYnwqmZNg4GzgIe6uiBJkiSplOV7E+H7m71fD/wc+PeuLUeSJEkqbfnOAz25uwuRJEmSeoM2x0BHxF4R8dFW1n00IhwDLUmSpH6lvZsIvwQc2Mq6d2bXS5IkSf1GewH6A8Btraz7MXBm15YjSZIklbb2AvTeKaU3WlqRbd+760uSJEmSSld7AXpbROzT0opse23XlyRJkiSVrvYC9B+BS1tZ91ngka4tR5IkSSpt7U1j9w3gkYgYAdwJvAq8A5gOnAMc373lSZIkSaWlzQCdUpoXER8Cvgd8EkhAAH8FPpRSWtD9JUqSJEmlo90HqaSUHgLeHREHAiOAlSmlpd1emSRJklSC8n2UN9nQbHCWJElSv9beTYSSJEmSchigJUmSpAIYoCVJkqQCGKAlSZKkAhigJUmSpAIUPUBHxOcj4tmIeCYi7oyI6mLXJEmSJLWmqAE6It4BXAbUpJQOAcqBs4tZkyRJktSWovdAk5mLemBEVACDgNeKXI8kSZLUqqIG6JTSq8AMYBmwAliXUnqwmDVJkiRJbSn2EI7hwBRgDLAvMDgiPtbCdhdFxLyImLdy5cqeLlOSJElqVOwhHH8HvJhSWplSqgXuBY5tvlFKaWZKqSalVDNixIgeL1KSJElqUOwAvQw4JiIGRUQAJwOLi1yTJEmS1Kpij4H+M3APsAD4S7aemcWsSZIkSWpLRbELSCldDVxd7DokSZKkfBR7CIckSZLUqxigJUmSpAIYoCVJkqQCGKAlSZKkAhT9JkJJkiSVsJQg1ee81gOttaUdbeSsa7Uttb9deRXsNa6Yn8BODNCSSkd9PdRty/7UNl2ur225Pe/l2h3LqT5zvgggdrzu1EbT9Y1ttLFPIcdpvk9LbQUeZ6c2WjhOZ2ps47PocI3Nj00LbR04DgAp+5J21LtTWyvv89mmxX166jz57pOjoH3yrKXT++RZW2vbtBTmGoNcW8Gt0O3qM6dtN0DmBMPObNeRoNnh7doIxrmff7EMHw3/9HSxq2jCAC31dfV1OwfIjgTR+u0FhtU2Amxr26a67vkMojzTg1FeBeUVmffk/AcEcv5jkttGO9s1W99SW+4+koonyjI/RHY5WmhrYztytm/YrklbJ7YrK8/5YthabTmvHd2u1Try3a7hfUe36+DvVTWop/4pyZsBWuqIxlBaSBDtSA9qFwTYht7WrlZWkQ2klTnhNGe5cX0VVA1pe9tWlysK3L7Zclll9rW8ez6DjkhdEMTbDPRFPHZex2nexs7H6VSNuetze6TZ8b6ltlbf57NNIfu0tr6rz1PI79zK+oJq6a59mq9vYZt2g3HsfDypkwzQ6ntSgu1bYOv67M/b2dcNzd7n/GzbkGnftim/ANttobQy/+A4YJfW1zcEx86Ez8b3rRy/zHuQOyRaCQiSpF7DAK3SUV+XE2abh9/1OSG4hQC8dT1sy1mu397++aIcqodmgmjVLpnXAbtAxYDOhc+y9sJvG8cwVEmSVPIM0OqclGD71pbD7rY2wm5LP7Ub8ztn5aAdYbfhZ/CYbBAe0mzd0JzlIU3fV1QbWCVJUsEM0P1VfV1OT29bQxtaC7xv79ivvrb980X5zoF20G4wfFQLQbchCA/dOShXDckMb5AkSSoSk0hv01pvb6s/b7cwLCL7Ph+5vb0NvbvDRu0cbFv8Gbpjv8qB9vZKkqQ+wQDdE+rrmwbYVoc2NL/ZrYVxwHn39jbrwR04HIaN3Hm8705DHXKGQFTtYm+vJElSM6ajfGxeC2uX7Rxqm/fqNgnFuYF5fX7nqRi4c7AdNrJpqG3es9vSWF97eyVJkrqNATofL/wGZn265XVRtnOorR4Gu+7fyk1sLYz3tbdXkiSp1zCx5WP0CTDtjpbH+VYOsrdXkiSpHzFA52PY/pkfSZIk9Xs+SkySJEkqgAFakiRJKoABWpIkSSqAAVqSJEkqgAFakiRJKoABWpIkSSqAAVqSJEkqgAFakiRJKoABWpIkSSqAAVqSJEkqQKSUil1DQSJiJfByEU69B7CqCOdVz/Lv3D/4d+4f/Dv3ff6N+4di/p1HpZRGNG/sdQG6WCJiXkqppth1qHv5d+4f/Dv3D/6d+z7/xv1DKf6dHcIhSZIkFcAALUmSJBXAAJ2/mcUuQD3Cv3P/4N+5f/Dv3Pf5N+4fSu7v7BhoSZIkqQD2QEuSJEkFMEBLkiRJBTBAS5IkSQUwQKtfi4iKnOUhEVETEbsVsyZJHRcRu3kNS+puBmj1WxFxLvBGRCyJiNOBRcCNwNMRMb2oxUnKW0SMjIi7sk+q/TPwRES8mW0bXeTyJHWRiPhLsWtoUNH+JlKf9QXgPcAuwNPA4Sml/4uIvYCHgDuLWZykvN0NfBs4J6VUBxAR5cA/AHcBxxSvNEmFiIgPt7YK2Lsna2mL09ip34qIhSmlCdnl11JK++asW5RSGl+04iTlLSKWppQOLHSdpNITEbXAHUBLAfXvU0q79HBJLbIHWv3Zsoi4nkwP9PMR8S3gXuDvgBVFrUxSIeZHxPeB24FXsm37A58AnipaVZI6YhEwI6X0TPMVEfF3RainRfZAq9+KiKHAZ8l8y/0ucCpwHvAy8PWUkiFa6gUiogr4JDAFeEe2+VXgV8APU0pbi1WbpMJExAnAyymlZS2sq0kpzStCWTsxQEuSJEkFcBYO9VsRUR4RF0fEv0bEcc3WfblYdUkqTEQMiojLI+JfIqI6Ij4REb+KiJsiYkix65OUv4ioyP63+X8iYlH257cR8amIqCx2fQ3sgVa/FRG3AYOAJ4CPA39IKf1zdt2ClNIRxaxPUn4i4udkxj4PJDOzzmIyM3N8CNg7pfTxIpYnqQARcSewlsw9DcuzzfuRuadht5TStCKV1oQBWv1W7kwb2QeqfB/YA5gOPJ5SOryY9UnKT8OMOhERZG4A3iellLLvn3ZGHan3iIglKaV3F7qupzmEQ/1ZVcNCSml7SukiYCHwMOD/9pV6mZTpEfpN9rXhvb1EUu/yVkT8Q0Q0ZtSIKIuIacCaItbVhAFa/dm8iDgttyGl9DXgx8DoolQkqSPmNYx1Timd39AYEQcA64tWlaSOOBv4e3Y8KXgJ8Abw4ey6kuAQDklSnxURkfwPndQrRcTuACml1cWupTl7oKUcETGz2DVI6ryGa9nwLPVeKaXVKaXVpfjfZgO01FRNsQuQ1CW8lqW+o+SuZwO01NSbxS5AUpfwWpb6jpK7nh0DLUmSJBXAHmipBaU43kpS4byWpd6ltzwl2B5o9VsRsVtrq8g8fGG/nqxHUsd4LUt9R295SrABWv1WRNQBL5P5j2yDlH3/jpRSVYs7SiopXstS39FbnhJcUewCpCL6G3BySmlZ8xUR8UoR6pHUMV7LUt/R5CnBwEUR8VVK7CnBjoFWf/ZtYHgr627qwTokdc638VqW+ope8ZRgh3BIkiRJBXAIh/q1iDgImAK8I9v0KvCrlNLi4lUlqVBey1Lf0RuuZ4dwqN+KiC8Bd5G50eiJ7E8Ad0bEFcWsTVL+vJalvqO3XM8O4VC/FRFLgHEppdpm7VXAsymlA4tTmaRCeC1LfUdvuZ7tgVZ/Vg/s20L7Ptl1knoHr2Wp7+gV17NjoNWffQ74fUQsBRqmuhoJvAu4pFhFSSrY5/BalvqKz9ELrmeHcKhfi4gy4Gia3qjwZEqprnhVSSqU17LUd/SG69kALeWIiItSSjOLXYekzvFalvqOUryeHQMtNfWpYhcgqUt4LUt9R8ldzwZoqakodgGSuoTXstR3lNz17BAOKUdE7JdSWl7sOiR1jtey1HeU4vVsD7SUo+ECjYjzil2LpI7zWpb6jlK8nu2BlloQEctSSiOLXYekzvFalvqOUrqenQda/VZELGptFbBXT9YiqeO8lqW+o7dczwZo9Wd7AacCa5q1B/BYz5cjqYO8lqW+o1dczwZo9WcPAENSSgubr4iIOT1ejaSO8lqW+o5ecT07BlqSJEkqgLNwSJIkSQUwQKvfiojxEfF4RLwSETMjYnjOuieKWZuk/HktS31Hb7meDdDqz74PXAMcCiwBHo2IA7LrKotVlKSCeS1LfUevuJ69iVD92S4ppf/JLs+IiPnA/0TExwFvDpB6D69lqe/oFdezAVr9WkTsmlJaB5BSmh0RHwH+G9ituJVJKoTXstR39Ibr2SEc6s9uBA7ObUgpLQJOBu4tSkWSOsJrWeo7esX17DR2kiRJUgHsgVa/FRG7RsQNEfF8RLwVEasjYnG2bVix65OUH69lqe/oLdezAVr92c/JPCp0Ukppt5TS7sDkbNvPi1qZpEJ4LUt9R6+4nh3CoX4rIl5IKb2n0HWSSovXstR39Jbr2R5o9WcvR8TlEbFXQ0NE7BURXwJeKWJdkgrjtSz1Hb3iejZAqz+bBuwO/CEi1kTEW8AcMtPkfLSYhUkqiNey1Hf0iuvZIRzq1yLiIGA/4PGU0oac9tNyJnKXVOK8lqW+ozdcz/ZAq9+KiMuAXwKXAM9ExJSc1dcVpypJhfJalvqO3nI9+yRC9WcXAkemlDZExGjgnogYnVK6GYjiliapAF7LUt/RK65nA7T6s7KG/zWUUnopIiaRuVBHUUIXqaR2eS1LfUevuJ4dwqH+7I2ImNDwJnvBngnsARxarKIkFcxrWeo7esX17E2E6rciYj9ge0rp9RbWHZdS+mMRypJUIK9lqe/oLdezAVqSJEkqgEM4JEmSpAIYoCVJkqQCGKAlSZKkAhigJamXiIg5EbE1ItZHxLqI+FtE/GdEHFnAMV6KiI91Z52S1NcZoCWpd/nXlNIuKaVdgcnAy8DjETG1yHVJUr9hgJakXiql9HJK6cvAT4FbIuOfIuL5bC/1soi4PiLKASLifmAkcFtEbIiIB7PtFRFxVUQsiYi1EfHHiKgp3m8mSaXNAC1Jvd9dwDuA9wDLgdOBocAU4HzgAoCU0geBZcAFKaUhKaVTsvtfm932NGB34EfA/0TE8J78JSSptzBAS1Lvtzz7untK6b9TSi+mjKeA/wRObm3HiAjgMuBfUkp/SynVpZR+CKwAzuj2yiWpF6oodgGSpE7bL/u6OiKmA/8MvJPMv+OrgMfb2HcPYAhwf0TkPlmrMue4kqQcBmhJ6v2mAa8CG4GfAR8GfptS2hYRM4Dc8cz1zfZdld3v71JKT/ZEsZLU2zmEQ5J6qYjYPyKuBc4F/olMT3IZsBKojYhjgI832+114MCGNymlBNwMzIiIA7PHHRIRp0bEvt3/W0hS7xOZf3dKkkpdRMwBJgLbgASsBh4Dbk4pPZHd5qvApWSGbswGXgImpJQmZdd/ALgF2A14PKV0ekRUkBkHfQGZYRsbyQz7uDSl1DC+WpKUZYCWJEmSCuAQDkmSJKkABmhJkiSpAAZoSZIkqQAGaEmSJKkABmhJkiSpAAZoSZIkqQAGaEmSJKkABmhJkiSpAAZoSZIkqQD/PzspJ9XgEpNtAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plot_daily_cumulative_accept_rate(df, 'Clicked', 'NoResponse', \n", - " **{'hue':['pyGroup', 'pyDirection', 'pyChannel'], \n", - " 'allTime':True, 'shrinkTicks':True})" + "plot_daily_cumulative_accept_rate(\n", + " df,\n", + " \"Clicked\",\n", + " \"NoResponse\",\n", + " **{\n", + " \"hue\": [\"pyGroup\", \"pyDirection\", \"pyChannel\"],\n", + " \"allTime\": True,\n", + " \"shrinkTicks\": True,\n", + " },\n", + ")" ] }, { @@ -713,22 +216,13 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt0AAAFGCAYAAAC2fiPvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA82UlEQVR4nO3dd3Rc12Hv+++eQSMKAaLXA5Ji7xVDyepdsiTSFiWSGN+4vSv7uii+ybuxHec+O46daPnerBs7cWIreclzrAEpieq0ilUt2RYGhV2iKFJlBh0kQYIFRJ39/sDIhmWSIokZHGDw+6zFpcGZU37weMyft/bZx1hrERERERGR+PG4HUBEREREJNGpdIuIiIiIxJlKt4iIiIhInKl0i4iIiIjEmUq3iIiIiEicqXSLiIiIiMRZktsB4i0/P99Onz7d7RgiIiIikuAaGxsPW2sLzvRewpfu6dOn09DQ4HYMEREREUlwxpjQ2d7T9BIRERERkThT6RYRERERiTOVbhERERGROEv4Od0iIiIiEh8DAwM0NzfT29vrdpQxlZaWRnl5OcnJyed9jEq3iIiIiFyU5uZmsrKymD59OsYYt+OMCWstR44cobm5mRkzZpz3cZpeIiIiIiIXpbe3l7y8vElTuAGMMeTl5V3w6L5Kt4iIiIhctMlUuD9wMb+zSreIiIiITGjf//73WbhwIUuWLGHZsmUEg8Gz7vuZz3yGrVu3jmG6YZrTLSIiIiIT1uuvv862bdvYvn07qampHD58mP7+frdj/RGNdMfJy2910t0z4HYMERERkYTW1tZGfn4+qampAOTn51NaWsp3v/tdVq9ezaJFi7jnnnuw1v7RsY2NjVx11VWsXLmSm266iba2NgB+9KMfsWDBApYsWcLGjRtjklOlOw6O9fRzz88bqPrbF/jzh3axPXz0jB+0iIiIiIzOjTfeSFNTE3PmzOFLX/oSv/rVrwD4yle+Qn19PXv37uX06dNs27btD44bGBjgq1/9Klu3bqWxsZHPfe5zfOtb3wLgvvvuY8eOHezevZuf/OQnMcmp6SVxkJOewhNfvpyauhCPbW/hke3NzC+Zit/nsG55GZmp+o9dREREEstfP/UGb7Yej+k5F5RO5du3LzznPpmZmTQ2NvLaa6/x8ssvs2HDBu677z6ysrL4wQ9+QE9PD11dXSxcuJDbb7/9d8ft37+fvXv3csMNNwAwNDRESUkJAEuWLMHv97Nu3TrWrVsXk99F7S9OFpRO5XvrFvONW+bz5M5WAsEQf/X4Xv7u6X2sXV6G3+ewsDTb7ZgiIiIiE57X6+Xqq6/m6quvZvHixfz0pz9l9+7dNDQ0UFFRwXe+850/WuLPWsvChQt5/fXX/+h8v/jFL3j11Vd56qmn+P73v8+ePXtIShpdbVbpjrPM1CSqfQ6bqirY1dxNoDbEo9ubqQmGWVaRg9/ncNuSUqakeN2OKiIiInLRPmpEOl7279+Px+Nh9uzZAOzcuZO5c+eye/du8vPzOXnyJFu3bmX9+vV/cNzcuXM5dOgQr7/+OpdeeikDAwO8/fbbzJ8/n6amJq655houv/xytmzZwsmTJ8nJyRlVTpXuMWKMYVlFDssqcvirjy/g0R3Dxft/bN3N32x7k0+uKMfvc5hdlOV2VBEREZEJ4+TJk3z1q1/l2LFjJCUlMWvWLO6//35ycnJYtGgRxcXFrF69+o+OS0lJYevWrdx77710d3czODjI1772NebMmcOnPvUpuru7sdZy7733jrpwA5hEv8Fv1apVtqGhwe0YZ2Stpf79owSCIZ7Z007/UISqGbn4fQ43LyomNUmj3yIiIjJ+7du3j/nz57sdwxVn+t2NMY3W2lVn2l8j3S4yxlA1I5eqGbn8P7f1sbWxmZq6MH+6ZSe5GSnctaqc6iqHyrwMt6OKiIiIyCiodI8TeZmpfOGqS/ivV8zkN+8cJlAb5t9ee4+f/updrpidj99XyXXzC0n2apVHERERkYlGpXuc8XgMV8wu4IrZBXQc7+XB+ia21IX54gONFE1NZcOqCjZWOZTmTHE7qoiIiIicJ5Xucaxoahr3XjebL18zi1f2dxIIhvnHlw/yTy8f5Np5hfh9lVw5pwCvx7gdVURERCYpay3GTK4ucjH3RKp0TwBej+G6+UVcN7+I5qM9bKlrYkt9Ey/sq6csZwrVPoe7VpVTmJXmdlQRERGZRNLS0jhy5Ah5eXmTpnhbazly5AhpaRfWu7R6yQQ1MBTh+Tc7CARD/ObgEZI8hpsWFuP3OVx6yeT5L76IiIi4Z2BggObm5j968EyiS0tLo7y8nOTk5D/Yfq7VS1S6E8C7h06yuS7M1sZmjvYMMDM/g2qfw50rypmWkeJ2PBEREZFJQaU7wUv3B3oHhnh2bzuBYIj694+SkuThtsUlVPscVlZO0+i3iIiISBypdE+S0j3S/vYT1ARDPLq9hRN9g8wtysK/xmHd8jKmpiV/9AlERERE5IKodE/C0v2Bnv5BntrVSiAYZndzN1OSvaxdVorfV8ni8my344mIiIgkDJXuSVy6R9rdfIyaYJgndrZyemCIJeXZ+H0Oty8tJT1FC9mIiIiIjMa4KN3GmPeBE8AQMGitXWWMyQUeBKYD7wN3W2uPmuHJxz8EbgV6gM9Ya7dHz/Np4K+ip/2etfZn57quSvcfO947wOM7WgjUhtnfcYKs1CQ+uaKMal8lc4uz3I4nIiIiMiGNp9K9ylp7eMS2HwBd1tr7jDHfAKZZa79ujLkV+CrDpdsH/NBa64uW9AZgFWCBRmCltfbo2a6r0n121lq2h48SqA2zbU8b/YMRVlVOw7/G4ZZFJaQle92OKCIiIjJhjOfSvR+42lrbZowpAV6x1s41xvw0+nrzyP0++GOt/UJ0+x/sdyYq3efn6Kl+HtneTCAY5r3Dp5iWnsz6leVU+yqZkZ/hdjwRERGRce9cpXssJ/Ja4JfGGAv81Fp7P1BkrW2Lvt8OFEVflwFNI45tjm4723YZpWkZKfxfV8zk85fP4PV3jhAIhvmP37zPv772Hh+blYffV8kNC4pI9nrcjioiIiIy4Yxl6b7cWttijCkEnjfGvDXyTWutjRbyUTPG3APcA+A4TixOOWkYY7hsVj6Xzcqn80QvDzc0UxMM86XAdgqyUtmwqoKNVRWUT0t3O6qIiIjIhDFmw5bW2pboPzuBx4AqoCM6rYToPzuju7cAFSMOL49uO9v2D1/rfmvtKmvtqoKCglj/KpNGYVYaX75mFq/+xTX8x2dXs7Q8h39+5SBX/OBlPvf/1fPivg6GIom9+o2IiIhILIzJnG5jTAbgsdaeiL5+HvgucB1wZMSNlLnW2r8wxnwc+Aq/v5HyR9baquiNlI3AiuiptzN8I2XX2a6tOd2x1XrsNFvqm9hSF6bzRB+l2WlsrHLYsLqCoqlpbscTERERcY3rN1IaY2YyPLoNw1Naaqy13zfG5AEPAQ4QYnjJwK7okoH/BNzM8JKBn7XWNkTP9TngL6Pn+r619j/OdW2V7vgYGIrw4r5OAsEQrx04jNdjuGF+Ef41Dh+7JB+PR4+cFxERkcnF9dLtJpXu+AsdOUVNXZiHG5rpOtVPZV461VUO61eWk5eZ6nY8ERERkTGh0q3SPSb6Bod4dm87gWCYuve6SPF6uGVxMX5fJaunT2P4X2CIiIiIJCaVbpXuMXeg4wQ1dWEeaWzmeO8gswsz8fscPrGinOwpyW7HExEREYk5lW6Vbtec7h9i2+5WAsEwO5uOkZbs4fYlpfjXVLK0PFuj3yIiIpIwVLpVuseFvS3d1NSFeXxHCz39QywsnYrfV8naZaVkpI7lkvEiIiIisafSrdI9rpzoHeCJncOj3/vajpOZmsS65aVUV1WyoHSq2/FERERELopKt0r3uGStZUfTMQK1YbbtbqVvMMIKJwe/r5KPLykhLdnrdkQRERGR86bSrdI97h3r6eeR7S3UBEO8c+gU2VOSuXNFOdU+h1mFmW7HExEREflIKt0q3ROGtZbge10EgmGe3dvGwJBlzcxc/L5KblpYTEqSx+2IIiIiImek0q3SPSEdPtnHww3N1NSFaOo6TX5mCnetqqC6yqEiN93teCIiIiJ/QKVbpXtCi0Qsrx08TKA2xItvdRKxlitnF+D3OVw7r5Akr0a/RURExH0q3SrdCaOt+zQP1jexpa6J9uO9FE9NY8PqCjZWVVCSPcXteCIiIjKJqXSrdCecwaEIL+8/RCAY4ldvH8IA180vwu9zuHJ2AR6PHrojIiIiY+tcpVtPJJEJKcnr4YYFRdywoIimrh4214V5qKGJ59/soCJ3CpuqHO5aWUFBVqrbUUVEREQ00i2Jo38wwi/fbCdQG+b1d4+Q7DXctLAYv6+SNTNz9ch5ERERiStNL1HpnnQOdp5kc12YrY3NdJ8eYGZBBn5fJXeuKCMnPcXteCIiIpKAVLpVuiet3oEhfrG7jUAwxPbwMVKTPNy2pJRqn8MKJ0ej3yIiIhIzKt0q3QLsaztOTTDMYztaONk3yLziLPxrKlm3rJSstGS344mIiMgEp9Kt0i0jnOob5MldrTxQG+KN1uOkp3hZu6wMv89hUVm22/FERERkglLpVumWM7DWsru5m0AwxJO7WukdiLC0Ige/z+H2JaVMSfG6HVFEREQmEJVulW75CN2nB3hsezOBYJgDnSfJSkvizhXl+H0Os4uy3I4nIiIiE4BKt0q3nCdrLQ2howRqQzy9p53+oQhV03Pxr3G4eVExqUka/RYREZEzU+lW6ZaL0HWqn62NTQSCYUJHesjNSOGuleVsqnKYnp/hdjwREREZZ1S6VbplFCIRy2/fOUIgGOKXb3YwFLFcMTsfv8/huvlFJHs9bkcUERGRcUClW6VbYqTjeC8P1TexuS5Ma3cvhVmpbFxdwYYqh7KcKW7HExERERepdKt0S4wNRSyv7O8kEAzz8v5ODHDtvEL8vkqunFOA16OH7oiIiEw25yrdSWMdRiQReD2G6+YXcd38IpqP9vBgfRNb6pt4YV89ZTlT2FRVwd2rKyjMSnM7qoiIiIwDGukWiZGBoQgvvNlBIBjm1wcPk+Qx3LiwCL+vkktn5uHR6LeIiEhC00i3yBhI9nq4ZXEJtywu4b3Dp9hcF+bhhiae3tPOjPwMqqsc1q8sZ1pGittRRUREZIxppFskjnoHhnh2bzuBYIj694+SkuTh44tL8PscVlZOwxiNfouIiCQK3Uip0i3jwP72E9QEQzy6vYUTfYPMKcrE76vkEyvKmJqW7HY8ERERGSWVbpVuGUd6+gd5alcrNcEwu5q7mZLs5Y6lpfjXOCwpz3E7noiIiFwklW6Vbhmn9jR3U1MX4vEdrZweGGJxWTZ+n8Mdy0pJT9EtFyIiIhOJSrdKt4xzx3sHeGJHCw/UhtnfcYKs1CQ+saKMap/DvOKpbscTERGR86DSrdItE4S1lu3howRqw2zb00b/YIRVldPwr3G4ZVEJacletyOKiIjIWah0q3TLBHT0VD+PbG+mJhjm3cOnyElPZv2Kcqp9DjMLMt2OJyIiIh+i0q3SLROYtZbX3z1CIBjmub3tDEYsl12Sh99XyQ0LikhJ8rgdUURERNDDcUQmNGMMl12Sz2WX5NN5opeHG4ZHv79cs538zFQ2rC5n42qHitx0t6OKiIjIWWikW2QCGopYXj1wiEBtmJfe6sACV88pwO+r5Jp5hXj1yHkREZExp+klKt2SwFqPnWZLfRMP1ofpON5HaXYaG1Y7bFhdQXF2mtvxREREJg2VbpVumQQGhyK8+FYngWCYV98+hNdjuH5+IX5fJZfPysej0W8REZG40pxukUkgyevhpoXF3LSwmNCRU2yua+Lhhiaee6MDJzedap/DXSvLyctMdTuqiIjIpKORbpEE1jc4xHNvdBCoDRF8r4sUr4ebFxXj9zlUzcjFGI1+i4iIxIqml6h0i3Cw8wSBYJhHGps53jvIrMJM/D6HTy4vJzs92e14IiIiE55Kt0q3yO+c7h9i2+5WAsEwO5uOkZbs4fYlpVT7HJZV5Gj0W0RE5CKpdKt0i5zRG63d1ATDPL6jhVP9QywomYp/jcPaZWVkpuqWDxERkQuh0q3SLXJOJ/sGeWJnCw/UhtnXdpyMFC/rlpfh91WyoHSq2/FEREQmBJVulW6R82KtZWfTMQLBME/taqVvMMJyJwe/r5LblpSQlux1O6KIiMi4NW5KtzHGCzQALdba24wxM4AtQB7QCPwXa22/MSYV+E9gJXAE2GCtfT96jm8CnweGgHuttc+d65oq3SIXp7tngEe2NxMIhnjn0CmmpiWxfmUF1T6HWYWZbscTEREZd8ZT6f4zYBUwNVq6HwIetdZuMcb8BNhlrf0XY8yXgCXW2i8aYzYCn7DWbjDGLAA2A1VAKfACMMdaO3S2a6p0i4yOtZa697p4IBjm2b1tDAxZfDNy8a+p5KaFRaQmafRbREQEzl26PWMYohz4OPBv0Z8NcC2wNbrLz4B10ddroz8Tff+66P5rgS3W2j5r7XvAQYYLuIjEiTEG38w8/nHTcl7/5nV845Z5tHX3cu/mHVz2dy9x3zNvET7S43ZMERGRcW0slyf4B+AvgKzoz3nAMWvtYPTnZqAs+roMaAKw1g4aY7qj+5cBtSPOOfIYEYmz/MxUvnjVJdxzxUx+ffAwgWCIf33tXX7yq3e4ck4Bfp/DdfMKSfKO2f+fFxERmRDGpHQbY24DOq21jcaYq8fgevcA9wA4jhPvy4lMOh6P4co5BVw5p4D27l4erG9iS32YL/y8kaKpqWxc7bCxqoKS7CluRxURERkXxmqk+2PAHcaYW4E0YCrwQyDHGJMUHe0uB1qi+7cAFUCzMSYJyGb4hsoPtn9g5DG/Y629H7gfhud0x+U3EhEAirPT+NPrZ/Play7h5f2HCARD/OilA/zjSwe4bn4Rfp/DlbML8Hj00B0REZm8xnzJwOhI9/8dvZHyYeCRETdS7rbW/rMx5svA4hE3Un7SWnu3MWYhUMPvb6R8EZitGylFxpemrh621Id5sL6Jwyf7KZ82hU1VDnevqqAgK9XteCIiInExblYviYa5mt+X7pkMLxmYC+wAPmWt7TPGpAE/B5YDXcBGa+270eO/BXwOGAS+Zq195lzXU+kWcU//YITn3+wgEAzx23eOkOw13LiwGL/P4dKZeXrkvIiIJJRxVbrHmkq3yPjwzqGTbA6Gebixme7TA8zMz6Da57B+ZTk56SluxxMRERk1lW6VbpFxo3dgiKf3tBEIhmkMHSUlycNtS0rw+ypZ4eRo9FtERCYslW6VbpFxaV/bcWqCYR7b0cLJvkHmFWfh9zmsW15GVlqy2/FEREQuiEq3SrfIuHaqb5And7USCIbY23Kc9BQva5eV4vdVsqgs2+14IiIi50WlW6VbZMLY3XyMQG2YJ3a10DsQYWl5Nn5fJbctLSE9ZSyf5yUiInJhVLpVukUmnO7TAzy+o4VAMMTbHSfJSkvizhXlVPsc5hRlffQJRERExphKt0q3yIRlraUhdJRAbYin97TTPxShanou/jUONy8qJjXJ63ZEERERQKVbpVskQXSd6mdrYxM1wTDvH+khNyOF9SvL2VTlMCM/w+14IiIyyal0q3SLJJRIxPL6u0cIBEP88o0OBiOWy2fl4/c5XL+giGSvx+2IIiIyCal0q3SLJKzO47081NDE5romWo6dpiArlY2rK9hY5VCWM8XteCIiMomodKt0iyS8oYjlV293EqgN89L+TgxwzdxC/GscrppTiNejh+6IiEh8qXSrdItMKs1He3iwvokt9U0cOtFHWc4UNlVVcPeqCgqnprkdT0REEpRKt0q3yKQ0MBThhTc7CATD/PrgYZI8hhsWFOH3VXLZJXl4NPotIiIxdK7SrSdNiEjCSvZ6uGVxCbcsLuH9w6fYXBfmoYYmntnbzvS8dKp9DutXVpCbkeJ2VBERSXAa6RaRSaV3YIjn3mgnUBum7v0uUrwebl1cjH9NJasqp2GMRr9FROTiaHqJSreInMHbHSeoCYZ5ZHszJ3oHmVOUid9XybrlZWRPSXY7noiITDAq3SrdInIOPf2DbNvVRiAYYldzN2nJHu5YWorfV8nSihy344mIyASh0q3SLSLnaW9LN4FgiCd2ttLTP8Sisqn4fZXcsbSUjFTdBiMiImen0q3SLSIX6ETvAI/vbCVQG+Kt9hNkpibxieVlVPsc5pdMdTueiIiMQyrdKt0icpGstWwPHyMQDLFtdxv9gxFWVk7D73O4dXEJacletyOKiMg4odKt0i0iMXCsp5+tjc3UBMO8e/gUOenJrF9RziafwyUFmW7HExERl6l0q3SLSAxZa3n93SMEgmGe29vOYMRy6cw8/GscblxQTEqSx+2IIiLiAj0cR0QkhowxXHZJPpddks+hE3083NhETTDMV2p2kJ+Zwt2rKthU5VCRm+52VBERGSc00i0iEgORiOXVA4cIBMO8uK8DC1w1pwC/r5Jr5haQ5NXot4hIotP0EpVuERlDrcdO82B9E1vqw3Qc76MkO42Nqx02rK6gODvN7XgiIhInKt0q3SLigsGhCC++1UkgGObVtw/h9Riun1+I31fJ5bPy8Xj0yHkRkUQSkzndxphvWWu/f4bt37TW/t1oAoqIJKIkr4ebFhZz08Jiwkd6qKkL83BDE8+90YGTm86mKoe7VpWTn5nqdlQREYmz8x7pNsYct9b+0RMhjDFd1trcmCeLEY10i8h40jc4xHNvdFATDFH7bhfJXsPNi0rw+xx8M3IxRqPfIiIT1ahGuo0xpdGXHmNMCTDyb4TZQN/oI4qITA6pSV7uWFrKHUtLOdh5gppgE1sbm3hqVyuXFGTg91Vy54pystOT3Y4qIiIx9JEj3caYCHCmnQwwBPxPa+19ccgWExrpFpHxrndgiG272wgEQ+wIHyM1ycPtS0vx+xyWVeRo9FtEZIIY1Y2UxphKhgv2TmDpiLciwCFrbW+McsaFSreITCRvtHZTEwzz+I4WTvUPsaBkKtU+h3XLy8hM1aMVRETGM61eotItIhPMyb5BntjZQqA2zJttx8lI8bJ2eRl+n8PC0my344mIyBnErHQbYy4FVgFZI7dba/92VAnjSKVbRCYyay27mrsJ1IZ4clcrfYMRllXk4Pc53LaklCkpXrcjiohIVExKtzHmO8BfMjzN5NSIt6y19tpRZowblW4RSRTdPQM8uqOZQDDMwc6TTE1L4s6V5fh9DrMKsz76BCIiElexKt3twFprbTCW4eJNpVtEEo21lrr3uggEwzyzt42BIYtvRi7VPoebFxWTmqTRbxERN8SqdHcAJdbaSCzDxZtKt4gkssMn+9ja2ExNMEy4q4fcjBTuWlVOdZVDZV6G2/FERCaVWJXu7wPvW2v/NZbh4k2lW0Qmg0jE8uuDhwkEQ7ywr5OhiOWK2fn4fZVcP7+QJK/H7YgiIgkvVqX7BeAK4G2gbeR71tobRxsyXlS6RWSy6Tjey4P1TWyuC9PW3UvR1FQ2rHbYuLqC0pwpbscTEUlYsSrd3z7be9bav77IbHGn0i0ik9XgUIRX9h8iEAzxytuHMMC184rwr3G4cnYBXo8euiMiEktap1ulW0QmuaauHrbUh3mwvpnDJ/sonzaFTVUOd6+qoCAr1e14IiIJIVYj3Zed7T1r7W8vMlvcqXSLiPxe/2CE59/sIBAM8dt3jpDkMdy0sBi/z+HSS/L0yHkRkVGIVek+06olFsBaO27Xp1LpFhE5s3cPnWRzXZiHG5s51jPAzPwMqn0Od64oZ1pGitvxREQmnLhMLzHGlALfA7ZZax8dRb64UukWETm33oEhntnbRqA2TEPoKClJHm5bXIJ/jcMKZ5pGv0VEzlPc5nQbY7KA7dba2Rd9kjhT6RYROX9vtR+nJhjm0e0tnOwbZF5xFn6fw9rlZUxNS3Y7nojIuBbP0p0PvGOtzb7ok8SZSreIyIU71TfIU7taCQTD7GnpZkqyl7XLSvH7KllcPm7/J19ExFWxmtP9lx/alAGsA/ZZa9ePKmEcqXSLiIzO7uZjBGrDPLmrldMDQywpz8bvc7h9aSnpKUluxxMRGTdiVbpf/tCmE0Aj8H+stcdHFzF+VLpFRGLjeO8Aj+9o4YHaEG93nCQrNYlPriij2lfJ3OIst+OJiLjO9XW6jTFpwKtAKpAEbLXWftsYMwPYAuQxXOD/i7W23xiTCvwnsBI4Amyw1r4fPdc3gc8DQ8C91trnznVtlW4Rkdiy1tIYOkogGOYXe9roH4ywevo0/L5Kbl5UTFryuF3QSkQkrmJWus3wLexVQAUQBurteZwgelyGtfakMSYZ+DXwp8CfAY9aa7cYY34C7LLW/osx5kvAEmvtF40xG4FPWGs3GGMWAJujGUqBF4A51tqhs11bpVtEJH66TvXzSGMzgWCI94/0MC09mbtWVbCpymFGfobb8URExlSsppdUAE8B84FOoBDYB9xhrQ1fQJh0hkv3fwN+ARRbaweNMZcC37HW3mSMeS76+nVjTBLQDhQA3wCw1v5d9Fy/2+9s11PpFhGJv0jE8vq7RwgEQ/zyjQ4GI5aPzcrD76vkhgVFJHs9bkcUEYm7c5XuC7kD5odAPfAxa+0pY0wm8PfAjxi+ofKjQngZnkIyC/gx8A5wzFo7GN2lGSiLvi4DmgCihbyb4SkoZUDtiNOOPEZERFzi8Rg+Niufj83Kp/NELw83NFMTDPOlwHYKslLZsKqCjVUVlE9LdzuqiIgrLqR0Xw5UWmtPA0Snivx34P3zOTg6BWSZMSYHeAyYd2FRz58x5h7gHgDHceJ1GREROYPCrDS+fM0svnjVJbz69iECwRD//MpBfvzKQa6ZW4jf53D13EK8Hj10R0Qmjwsp3b1ANnB6xLZsoP9CLmitPRZdCeVSIMcYkxQd7S4HWqK7tTA8b7w5Or0km+EbKj/Y/oGRx4y8xv3A/TA8veRC8omISGx4PYZr5hVyzbxCWo6d5sG6MFvqm/j8zxoozU5jU5XDhtUVFE5NczuqiEjcXcgku8eAx4wx1xpjZhpjrgW2Ao981IHGmILoCDfGmCnADQzPB38Z+GCN708DT0RfPxn9mej7L0Vv2HwS2GiMSY2ufDIbqLuA30FERFxQljOFP7txLr/5xrX85FMruKQwk79//m0uve8lvvjzRl47cIhIRGMkIpK4LuRGyinAPwB/wvDSf33Az4D//sGUk3McuyS6r5fhov+Qtfa7xpiZDC8ZmAvsAD5lre2LLjH4c2A50AVstNa+Gz3Xt4DPAYPA16y1z5zr2rqRUkRkfHr/8Ck214V5qKGJoz0DVOalU13lsH5lOXmZqW7HExG5YKNavcQYUwRcZa19KPqzYXglkUPA3cDL1trO2EaOHZVuEZHxrW9wiGf3thMIhql7r4sUr4dbFhfj91Wyevo0hv/aEREZ/0a7esnXGZ5PDUB0mkdn9MQzGF4z+89jkFNERCah1CQva5eVsXZZGQc6ThAIhnlkezNP7GxldmEmfp/DJ1aUkz0l2e2oIiIX7XxGut9ieKS74wzvFQGvWmvnxinfqGmkW0Rk4jndP8RTu1sJBMPsajpGWrKHO5aW4vdVsqQ8W6PfIjIujXZ6yTFrbc453u+21maPLmL8qHSLiExse1u6CQTDPLGzhZ7+IRaVTaW6qpK1y0rJSL2QRbhEROJrtKW7E1hqrW07w3slwB5rbX5MksaBSreISGI40TvA4ztbCdSGeKv9BJmpSaxbPjz6Pb9kqtvxRERGXbofA/ZZa//yDO99D1horf1ETJLGgUq3iEhisdayo+kYgdow23a30jcYYYWTg99XyceXlJCW7HU7oohMUqMt3auA14AHgM0MP4ymDNgE+IHLrbXbY5o4hlS6RUQS17Gefh7Z3kIgGOLdQ6fInpLM+pXlVPscLinIdDueiEwyoyrd0RPcAPwYmAVYwAAHgS9Za1+IYdaYU+kWEUl81lpq3+0iEAzx3BvtDAxZLp2ZR7XP4aaFxaQkXciz4ERELs6oS/eIE80muka3tfZAjPLFlUq3iMjkcuhEHw83NlETDNN89DT5mSnctaqC6iqHitx0t+OJSAKLWemeiFS6RUQmp0jE8uqBQwSCYV7c14EFrpxdgN/ncO28QpK8Gv0WkdhS6VbpFhGZ1Nq6T/NgfRNb6ppoP95L8dQ0NlZVsHG1Q3F2mtvxRCRBqHSrdIuICDA4FOGltzoJBMO8euAQHmO4bl4h/jWVXDErH49HD90RkYs32sfAi4iIJIQkr4cbFxZz48Jiwkd62Fwf5qH6Jn75ZgcVuVOorqrkrlXl5Gemuh1VRBKMRrpFRGRS6x+M8Nwb7QSCIWrf7SLZa7hpYTF+XyVrZubqkfMict40vUSlW0REzsPBzpPUBMNsbWzieO8gMwsy8PsquXNFGTnpKW7HE5FxTqVbpVtERC5A78AQ23a3URMMsT18jNQkD7ctKcW/xmF5RY5Gv0XkjFS6VbpFROQivdl6nJq6EI9tb+FU/xDzS6bi9zmsW15GZqpujRKR31PpVukWEZFROtk3yJM7W3mgNsSbbcfJSPGydnkZ1VUOi8qy3Y4nIuOASrdKt4iIxIi1ll3N3QRqQzy1u5XegQhLK3Lw+xxuX1LKlBSv2xFFxCUq3SrdIiISB909Azy6o5maYJgDnSfJSkvizhXl+H0Os4uy3I4nImNMpVulW0RE4shaS/37RwkEQzyzp53+oQhVM3Lx+xxuXlRMapJGv0UmA5VulW4RERkjR072sbWxmZq6MKEjPeRmpHDXqnKqqxwq8zLcjicicaTSrdItIiJjLBKx/OadwwRqwzy/r4OhiOWK2fn4fQ7XzS8i2etxO6KIxJhKt0q3iIi4qON4Lw/WN7G5Lkxbdy+FWalsXF3BhiqHspwpbscTkRhR6VbpFhGRcWBwKMIr+w8RCIZ45e1DGODaeYX4fZVcOacAr0cP3RGZyM5VurWqv4iIyBhJ8nq4fkER1y8ooqmrhwfrm9hS38QL++opy5lCtc/hrlXlFGaluR1VRGJMI90iIiIuGhiK8PybHQSCIX5z8AhJHsONC4vw+yq5dGYeHo1+i0wYGukWEREZp5K9Hm5dXMKti0t499BJNteFebixmaf3tDMjP4PqKof1K8uZlpHidlQRGQWNdIuIiIwzvQNDPLO3jUBtmIbQUVKSPHx8cQl+n8PKymkYo9FvkfFIN1KqdIuIyAS1v/0ENcEQj25v4UTfIHOLsvCvcVi3vIypacluxxOREVS6VbpFRGSC6+kf5KldrQSCYXY3dzMl2cvaZaX4fZUsLs92O56IoNKt0i0iIglld/MxaoJhntjZyumBIZaUZ1Nd5XDHslLSU3S7lohbVLpVukVEJAEd7x3g8R0tBGrD7O84QVZqEp9YUUa1z2Fe8VS344lMOirdKt0iIpLArLU0ho4SCIb5xZ42+gcjrKqchn+Nwy2LSkhL9rodUWRSUOlW6RYRkUni6Kl+HtneTCAY5r3Dp8hJT+auleVsqnKYWZDpdjyRhKbSrdItIiKTjLWW1985QiAY5rk32hmMWD42K4/qqkpuWFBESpLH7YgiCUelW6VbREQmsc4TvTzc0ExNMEzLsdPkZ6ayYXU5G1c7VOSmux1PJGGodKt0i4iIMBSxvPr2IQLBEC+91YkFrp5TgN9XyTXzCvHqkfMio6LSrdItIiLyB1qPnWZLfRNb6sJ0nuijNDuNjVUOG1ZXUDQ1ze14IhOSSrdKt4iIyBkNDEV4cV8ngWCI1w4cxusx3DC/CP8ah49dko9Ho98i5+1cpVsr6IuIiExiyV4PNy8q5uZFxbx/+BSb68M83NDMs2+0U5mXTnWVw/qV5eRlprodVWRC00i3iIiI/IG+wSGe3dtOIBim7r0uUqLF3O9zqJqRizEa/RY5E00vUekWERG5KAc6ThAIhnlkezMnegeZVZiJ3+fwyeXlZKcnux1PZFxR6VbpFhERGZXT/UM8tbuVmmCYnU3HSEv2cPuSUvxrKllanq3RbxFUulW6RUREYmhvSzc1dWEe39FCT/8QC0un4vdVcseyUjJTdbuYTF4q3SrdIiIiMXeid4AndrbyQG2It9pPkJHiZd3yMvy+ShaUTnU7nsiYU+lW6RYREYkbay07mo4RqA2zbXcrfYMRljs5+H2V3LakhLRkr9sRRcaESrdKt4iIyJg41tPPI9tbqAmGeOfQKbKnJHPninKqfQ6zCjPdjicSV66XbmNMBfCfQBFggfuttT80xuQCDwLTgfeBu621R83w3Rg/BG4FeoDPWGu3R8/1aeCvoqf+nrX2Z+e6tkq3iIjI2LPWEnyvi0AwzLN72xgYsqyZmYvfV8lNC4tJSfK4HVEk5sZD6S4BSqy1240xWUAjsA74DNBlrb3PGPMNYJq19uvGmFuBrzJcun3AD621vmhJbwBWMVzeG4GV1tqjZ7u2SreIiIi7Dp/s4+GGZmrqQjR1nSYvI4W7V1ewabWDk5fudjyRmHG9dP/RRY15Avin6J+rrbVt0WL+irV2rjHmp9HXm6P77weu/uCPtfYL0e1/sN+ZqHSLiIiMD5GI5bWDhwnUhnjxrU4i1nLF7AL8Pofr5hWS5NXot0xs4+ox8MaY6cByIAgUWWvbom+1Mzz9BKAMaBpxWHN029m2i4iIyDjn8RiumlPAVXMKaOs+zYP1TWypa+ILP2+keGoaG1ZXsLGqgpLsKW5HFYm5MS3dxphM4BHga9ba4yMX0rfWWmNMTIbdjTH3APcAOI4Ti1OKiIhIDJVkT+Fr18/hK9fM4qW3OgkEw/zopQP840sHuG5+EX6fw5WzC/B49NAdSQxjVrqNMckMF+6AtfbR6OYOY0zJiOklndHtLUDFiMPLo9taGJ5iMnL7Kx++lrX2fuB+GJ5eEsNfQ0RERGIoyevhxoXF3LiwmKauHjbXhXmooYnn3+ygIncKm6oc7lpZQUFWqttRRUZlrG6kNMDPGL5p8msjtv8v4MiIGylzrbV/YYz5OPAVfn8j5Y+stVXRGykbgRXRU2xn+EbKrrNdW3O6RUREJpb+wQi/fLOdQG2Y1989QrLXcOPCYvw+h0tn5umR8zJuuX4jpTHmcuA1YA8QiW7+S4bndT8EOECI4SUDu6Il/Z+AmxleMvCz1tqG6Lk+Fz0W4PvW2v8417VVukVERCaug50n2VwXZmtjM92nB5hZkEF1lcP6leXkpKe4HU/kD7heut2k0i0iIjLx9Q4M8YvdbQSCIbaHj5Ga5OHjS0rw+ypZ4eRo9FvGBZVulW4REZGEsa/tODXBMI/taOFk3yDzirPwr6lk3bJSstKS3Y4nk5hKt0q3iIhIwjnVN8iTu1p5oDbEG63HSU/xsnZZGX6fw6KybLfjySSk0q3SLSIikrCstexu7iYQDPHkrlZ6ByIsrcjBX+Vw+9JSpqR43Y4ok4RKt0q3iIjIpNB9eoDHtjcTCIY50HmSrLQk7lxRTrXPYU5RltvxJMGpdKt0i4iITCrWWurfP0ogGOKZPe30D0Womp6Lf43DzYuKSU3S6LfEnkq3SreIiMik1XWqn62NTQSCYUJHesjNSOGuleVsqnKYnp/hdjxJICrdKt0iIiKTXiRi+e07RwgEQ/zyzQ6GIpYrZudTXeVw/YIikr0etyPKBKfSrdItIiIiI3Qc7+Wh+iY214Vp7e6lICuVjasr2FjlUJYzxe14MkGpdKt0i4iIyBkMRSyv7O8kEAzz8v5ODHDN3EL8axyumlOI16OH7sj5U+lW6RYREZGP0Hy0hy11TWypb+LwyT7KcqawqaqCu1dVUDg1ze14MgGodKt0i4iIyHkaGIrw/Jsd1ATD/PrgYZI8hhsXFlFdVclll+Th0ei3nMW5SnfSWIcRERERGc+SvR5uXVzCrYtLeO/wKTbXhXm4oYmn97QzPS+dap/D+pUV5GakuB1VJhCNdIuIiIh8hN6BIZ7d204gGKL+/aOkeD3curgY/5pKVlVOwxiNfouml6h0i4iISMzsbz9BTTDEo9tbONE3yJyiTPy+Sj6xooypacluxxMXqXSrdIuIiEiM9fQP8tSuVmqCYXY1dzMl2csdS0vxr3FYUp7jdjxxgUq3SreIiIjE0Z7mbmrqQjy+o5XTA0MsLsvG73O4fWkpGam6hW6yUOlW6RYREZExcLx3gCd2tPBAbZj9HSfITE3iE8vL8K9xmFc81e14Emcq3SrdIiIiMoastWwPHyVQG2bbnjb6ByOsrJyG3+dw6+IS0pK9bkeUOFDpVukWERERlxw91c8j25sJBMO8d/gUOenJrF9RTrXPYWZBptvxJIZUulW6RURExGXWWl5/5wiBYJjn3mhnMGK57JI8/L5KblhQREqSx+2IMkoq3SrdIiIiMo50nujl4YZmaoJhWo6dJj8zlQ2ry9m42qEiN93teHKRVLpVukVERGQcGopYXj1wiEBtmJfe6sACV80pwO+r5Jq5BSR5Nfo9kah0q3SLiIjIONd67DRb6pvYUhem80QfJdlpbFztsGF1BcXZaW7Hk/Og0q3SLSIiIhPEwFCEF/d1EgiGeO3AYbwew/XzC/H7Krl8Vj4ejx45P16dq3RrtXYRERGRcSTZ6+HmRcXcvKiY0JFTbK5r4qGGJp57owMnN51qn8NdK8vJy0x1O6pcAI10i4iIiIxzfYNDPPdGB4HaEMH3ukj2Gm5eVILf5+CbkYsxGv0eDzS9RKVbREREEsTBzhMEgmEeaWzmeO8glxRk4PdVcueKcrLTk92ON6mpdKt0i4iISII53T/Ett2tBIJhdjYdIzXJw+1LS/H7HJZV5Gj02wUq3SrdIiIiksD2tnRTUxfmiR0tnOofYkHJVPxrHNYuKyMzVbfwjRWVbpVuERERmQRO9g3yxM4WHqgNs6/tOBkpXtYtL6Pa57CwNNvteAlPpVulW0RERCYRay07m44RCIZ5alcrfYMRllXk4Pc53LaklCkpXrcjJiSVbpVuERERmaS6ewZ4ZHszgWCIdw6dYmpaEneuLMfvc5hVmOV2vISi0q3SLSIiIpOctZbge10EgmGe3dvGwJDFNyMX/5pKblpYRGqSRr9HS6VbpVtERETkdw6f7OPhhmZq6kI0dZ0mLyOFu1ZVUF3l4OSlux1vwlLpVukWERER+SORiOW1g4epCYZ4YV8nQxHLlXMK8PscrptXSJLX43bECUWlW6VbRERE5Jzau3t5sL6JLfVh2rp7KZqayobVDhtXV1CaM8XteBOCSrdKt4iIiMh5GRyK8PL+QwSCIX719iEMcO28IvxrHK6cXYDXo4funM25SrdWSxcRERGR30nyerhhQRE3LCiiqauHzXVhHmpo4oV9HZRPm8KmKoe7V1VQkJXqdtQJRSPdIiIiInJO/YMRfvlmOzXBML995whJHsNNi4rx+xwunZmnR85HaXqJSreIiIhITLxz6CSbg2Eebmym+/QAM/MzqPY5rF9ZTk56itvxXKXSrdItIiIiElO9A0M8vaeNQDBMY+goKUkebltcgn+Nwwpn2qQc/VbpVukWERERiZt9bcepCYZ5bEcLJ/sGmVechd/nsG55GVlpyW7HGzMq3SrdIiIiInF3qm+QJ3e1EgiG2NtynPQUL2uXleL3VbKoLNvteHGn0q3SLSIiIjKmdjcfI1Ab5oldLfQORFhano3fV8ltS0tIT0nMBfRUulW6RURERFzRfXqAx3e0EAiGeLvjJFlpSXxyeRnVvkrmFme5HS+mVLpVukVERERcZa2lIXSUQG2Ip/e00z8UYfX0afh9ldy8qJi0ZK/bEUdNpVulW0RERGTc6DrVz9bGJmqCYd4/0sO09GTuWlXBpiqHGfkZbse7aOcq3Z4xCvDvxphOY8zeEdtyjTHPG2MORP85LbrdGGN+ZIw5aIzZbYxZMeKYT0f3P2CM+fRYZBcRERGR2MrNSOGeKy/hpT+/mgc+72PNzDz+/dfvcc3/foVP/VuQZ/a0MTAUcTtmTI3JSLcx5krgJPCf1tpF0W0/ALqstfcZY74BTLPWft0YcyvwVeBWwAf80FrrM8bkAg3AKsACjcBKa+3Rc11bI90iIiIi41/n8V4eamhic10TLcdOU5CVyoZVFWysqqB8Wrrb8c7LuJheYoyZDmwbUbr3A1dba9uMMSXAK9baucaYn0Zfbx653wd/rLVfiG7/g/3ORqVbREREZOIYilh+9XYngdowL+3vBOCauYX4fQ5Xzy3E6xm/D905V+l2c72WImttW/R1O1AUfV0GNI3Yrzm67WzbRURERCRBeD2Ga+cVce28IpqP9vBgfRNb6pv4/M8aKMuZwsbVFWxYXUHh1DS3o16QMZnT/VHs8HB7zIbcjTH3GGMajDENhw4ditVpRURERGQMlU9L589vnMtvv3Et/+JfwYz8DP7++be57L6X+G8PNPLrA4eJRCbGoiBujnR3GGNKRkwv6YxubwEqRuxXHt3WwvAUk5HbXznTia219wP3w/D0ktjGFhEREZGxlOz1cMviEm5ZXML7h0+xuS7MQw1NPLO3nel56VT7HNavrCA3I8XtqGfl5kj3k8AHK5B8GnhixPY/ia5isgbojk5DeQ640RgzLbrSyY3RbSIiIiIySUzPz+Cbt87n9W9exw83LqMwK42/ffot1vzti3xtyw7q3utiPC6JPSYj3caYzQyPUucbY5qBbwP3AQ8ZYz4PhIC7o7s/zfDKJQeBHuCzANbaLmPM3wD10f2+a63tGov8IiIiIjK+pCV7WbusjLXLyni74wQ1wTCPbG/m8Z2t/PtnVnHtvKKPPskY0sNxRERERCQh9PQP8vSedtYuKyXZO/YTOsbr6iUiIiIiIjGTnpLE+pXlbsc4o3GxeomIiIiISCJT6RYRERERiTOVbhERERGROFPpFhERERGJM5VuEREREZE4U+kWEREREYkzlW4RERERkThT6RYRERERiTOVbhERERGROFPpFhERERGJM2OtdTtDXBljDgEhly6fDxx26doydvQ5Jz59xpODPufJQZ9z4nPzM6601hac6Y2EL91uMsY0WGtXuZ1D4kufc+LTZzw56HOeHPQ5J77x+hlreomIiIiISJypdIuIiIiIxJlKd3zd73YAGRP6nBOfPuPJQZ/z5KDPOfGNy89Yc7pFREREROJMI90iIiIiInGm0i0iIiIiEmcq3SIiIiIicabSLSIiIiISZyrdMWaMKTLGrIj+KXI7j4iMnjHmDrcziMjFMcYkjXidaYxZZYzJdTOTTE5JH72LnA9jzDLgJ0A20BLdXG6MOQZ8yVq73aVoInIBjDGf/PAm4Mcf/MVtrX107FOJyMUwxnwG+HtjzBHgT4EfA+8Bc4wxf2Gt3exmPplctGRgjBhjdgJfsNYGP7R9DfBTa+1SV4KJyAUxxgwAzwGdDBdugPXAVsBaaz/nVjYRuTDGmD3ANUAWsAtYbq19J/pvop+31i5xNaBMKhrpjp2MDxduAGttrTEmw41AInJRLgPuA+qttf8CYIy52lr7WXdjichFGLLWHgYOG2NOWmvfAbDWdhhjPuJQkdhS6Y6dZ4wxvwD+E2iKbqsA/gR41rVUInJBrLX1xpgbgK8aY14Gvg7oXwmKTExhY8zfMTzS/ZYx5u+BR4HrgTZXk8mko+klMWSMuQVYC5RFN7UAT1prn3YvlYhcLGNMKfAPwCpr7UyX44jIBTLGTAW+zPD/cf4n4Cbgs0AI+J61VsVbxoxKt4iIiIhInGnJwDFgjLnH7QwiMnr6LoskDn2fZaypdI8N3a0hkhj0XRZJHPo+y5hS6R4b/W4HEJGY0HdZJHHo+yxjSnO6x4AxJmytddzOISKjo++ySOLQ91nGmpYMjBFjzO6zvQXocfAiE4S+yyKJQ99nGU9UumOniOGliI5+aLsBfjv2cUTkIum7LJI49H2WcUOlO3a2AZnW2p0ffsMY88qYpxGRi6Xvskji0PdZxg3N6RYRERERiTOtXiIiIiIiEmcq3SIiIiIicabSLSIio2KM+Y4x5gW3c4iIjGcq3SIiCcYYc68x5p0PbfuqMcYaY24ZsW2KMabXGHPH2KcUEZlcVLpFRBLPi8BMY0zliG3XAW8A147Y9jHAC7wydtFERCYnlW4RkQRjrX0DaGO4aGOM8QJXAd/+YFvUdUA90GOM+UtjzNvGmGPGmN8YY1aNPKcx5r8aY/YaY7qNMTuMMTee7frGmM8aY5qNMb5Y/24iIhOVSreISGJ6id8X7JVAO/AkcIkxJi+6/TrgBeCvgbXAzUAe8O/As8aYaTBcuIGvA35gGvAt4FFjzKwPX9QY8zfRfa+01gbj86uJiEw8Kt0iIonpBX4/leQ64CVr7QDDT+G7xhiTDaxgeCrKvcD/sNa+a60dstb+vwyPlH88evyfAt+11u6y1kastU8DLwMbR1wvxRjzAMMj6pdZa9+N9y8oIjKR6ImUIiKJ6UWg2BizgOHy/ZPo9pejPw8AvcCbQCbwlDFm5NPSkoHy6OsZwI+NMT8a8X4S0Dzi5/nApcCN1tquGP8uIiITnkq3iEgCstY2GWPeZni0+lJgQ/Stl4AAMAi8ChwGTgHXW2vrz3K6EPBta+3D57jkLuCfgUeMMXdba7WEoIjICJpeIiKSuF4E/gw4MGL0eQdQCNwFvGCttcAPgf9tjJkNYIzJNMbcZIwpjR7zf4DvGGOWmWFTjDGXG2PmjbyYtfZRYBPwkDFmbfx/PRGRiUOlW0Qkcb0AFDM8ug2AtXaI4RHu4uj7MLyqyRPAE8aY48AB4ItE/46w1v4r8APgP4CjQBj4nwxPQfkD1trnGL4p89+NMf64/FYiIhOQGR7kEBERERGReNFIt4iIiIhInKl0i4iIiIjEmUq3iIiIiEicqXSLiIiIiMSZSreIiIiISJypdIuIiIiIxJlKt4iIiIhInKl0i4iIiIjEmUq3iIiIiEic/f9a93sf3DO4sAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plot_outcome_count_time(df, 'Accepted', 'weekly', **{'hue':'pyIssue', 'allTime':True, 'shrinkTicks':True})" + "plot_outcome_count_time(\n", + " df, \"Accepted\", \"weekly\", **{\"hue\": \"pyIssue\", \"allTime\": True, \"shrinkTicks\": True}\n", + ")\n", + "\n", + "# A count trend seems useful to, not just the CTR" ] }, { @@ -742,45 +236,26 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Text(0.5, 1.0, 'Offers within Inbound direction')" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxQAAAJxCAYAAADbzfTMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABjpUlEQVR4nO3deZyVddn48c/FJorKroIb4C44oOCeey4pZE+LLZZLltmTtlvWU09W1lNPVr+ysmxRW5/UMhW11Cz3VFBA3FIRFxZlEQEBBeb6/XFu8DALDmfmzJlhPu/Xa15z3997+V7ncBzv63y3yEwkSZIkqRLdah2AJEmSpM7LhEKSJElSxUwoJEmSJFXMhEKSJElSxUwoJEmSJFXMhEKSJElSxUwoJKlCUXJpRLwUEfcVZR+NiBciYmlEDOwAMX4xIn6xnuOnRcSd6zl+Y0ScWp3o1qnnnxHxoWrX04I41vt+NHH+zIh4c7G93ve6LbXXv4sktUSPWgcgSR1VRJwGfAbYCVgMXA18ITMXFae8CTga2C4zX4mInsD3gAMyc2r7R9xYZn5zzXZEDAOeBnpm5qoWXv+WSuuOiJnAhzLzlkrv0ZmUv9dtKSLOB3bOzPeX1VXxv4sktTVbKCSpCRHxGeDbwLlAX+AAYEfg5ojoVZy2IzAzM18p9rcGegMPV1hn91YFrQ4rIvwCT9JGy4RCkhqIiC2BrwLnZOZfM3NlZs4ETgKGAe+PiDOAXwAHFt2b/gA8XtxiUUTcWtxr94i4OSIWRsTjEXFSWT2XRcTFEXFDRLwCHBERx0fEIxGxJCJmRcRnm4nxmYgYW2yfHBEZESOL/TMi4i/F9vkR8dvistvL4lsaEQeW3e/CouvW0xHxlrLytV2R1nQHau7cN3hPW3LtThFxX0QsjohrImJA2fVvjYiHI2JREdMeZccyInZu8L5eUGwfHhHPR8RnIuLFiJgTEaeXnTswIq4t6ryPUmvU+l7HB4r3fkFE/FeDY2vf64gYVsR1RkQ8C6z5PHwwIh4t3oO/RcSOZdePLPusvFB0oToO+CLw7uLfbGoT/y7dIuJLRVwvRsSvI6JvgzhOjYhnI2J+w7glqbVMKCSpsYMotTT8ubwwM5cCNwBHZ+YvgbOAezJz88x8LzCyOLVfZh4ZEX2Am4HfA1sB7wF+EhF7lt32fcA3gC2AO4FfAh/JzC2AURQPok24DTi82D4MmAEcWrZ/WxPXrDner4j5nmJ/f0rJ0CDgf4FfRkQ0U++GnLuh154CfBAYAqwCfggQEbsCfwA+CQym9G9wXVlL0RvZhlIr07bAGcCPI6J/cezHwIqizg8WP00q/t0uBj4ADAUGAtu9Qd2HAXsAx0bEiZSSg7cXr+OO4nUREVsAtwB/Le69M/D3zPwr8E3gj8W/2egm6jit+DkCGAFsDvyowTlvAnYDjgL+uzwhk6TWMqGQpMYGAfObGWcwpzjeEuMpdYm6NDNXZeaDwJ+Ad5Wdc01m3pWZ9Zm5AlgJ7BkRW2bmS5n5QDP3vo3SwyrAIcD/lO03l1A055nM/HlmrgYup/RwvXUbnLuh1/4mM6cXXci+DJwUpW5g7wauz8ybM3MlcCGwKaXEryVWAl8rWppuAJYCuxX3fgfw35n5SmZOL+JqzjuBiZl5e2a+WsRY/wZ1n1/cezmlBPR/MvPR4rP1TWBM0UoxHpibmd/NzBWZuSQz723h6zsZ+F5mziiS3i8A74l1u1l9NTOXF2N7pgJNJSaSVBETCklqbD4wKJru9z6kON4SOwL7F910FkXEIkoPf9uUnfNcg2veARwPPBMRt5V3S2rgNuCQiBgCdAeuAA6O0sDrvsCUFsYIMHfNRmYuKzY3b4NzN/Ta8vfiGaAnpeRtaLG/5tr64txtW1jvggbJ4bKi3sGUJidpWG9zhpafWyQ+C96g7vJ77wj8oOyzsBAISq9je+CpN7jX+uIqj/sZSq+rPFmbW7a95vVLUpswoZCkxu4BXqXUNWWtiNgceAvw9xbe5zngtszsV/azeWZ+tOycLL8gM+/PzBMpdZH6C6VEoZHMfJLSg+E5wO2ZuZjSQ+OZwJ3FQ3ejy1oYd61sX7a9A6WWhfnAbEoP40Bput7i3FlF0TJgs7JryxO29ZlHqWtVw3qbM6f83IjYjFK3p/Upf8+fo9SdrfzzsGlm3l0cG9GCezRlnfeH0mtYBbzwBtdJUpswoZCkBjLzZUqDsi+KiOMiomfxzf8VwPPAb1p4q4nArsVA3p7Fz77N9V+PiF5RGmDdt+jas5j1d6m5DTib17s3/bPBfkPzivs19+Baa++PiD2LB/WvAVcV3aOuAE6IiKOiNDXvZyglfHcX100B3hcR3YtBzIc1ce9Ginv/GTg/IjYrxkisb22Hq4DxEfGmYvzG19iw/4/+FPhCvD54vm9ErOn+NhEYEhGfjIhNImKLiNi/OPYCMCwimqvrD8CnImJ4kfSuGXPRoqmBJam1TCgkqQmZ+b+UBtBeSOnB/l5K3yIfVfSfb8k9lgDHUBqMPZtSC8K3gU3Wc9kHgJkRsZhSn/uT13PubZQGc9/ezH7DeJZRGgB+V9Ht5oCWvI529BvgMkrvU2/g4wCZ+TjwfuAiSi0WE4AJmflacd0nirJFlN6vv2xAnWdT6v4zt6j70uZOzMyHgY9RGmQ/B3iJUoLZIpl5NaV///8r/n2nU2rxWvNZObp4HXOBJygNsga4svi9ICKaGlPzK0rv3e2U1hlZQanlSpLaRWR29BZwSZIkSR2VLRSSJEmSKmZCIUmSJKliJhSSJEmSKmZCIUmSJKliJhSSJEmSKtbUKrAblUGDBuWwYcNqHYYkSZLUqU2ePHl+Zg5uWL7RJxTDhg1j0qRJtQ5DkiRJ6tQi4pmmyu3yJEmSJKliJhSSJEmSKmZCIUmSJKliJhSSJEmSKmZCIUmSJKliJhSSJEmSKmZCIUmSJKliHTKhiIjtI+IfEfFIRDwcEZ8oys+PiFkRMaX4Ob7WsUqSJEldWUdd2G4V8JnMfCAitgAmR8TNxbHvZ+aFNYxNkiRJUqFDJhSZOQeYU2wviYhHgW1rGdPYc39dy+q1kZj8nVNqHYIkSVKb6pBdnspFxDBgb+DeoujsiJgWEb+KiP61i0ySJElSh04oImJz4E/AJzNzMXAxsBMwhlILxnebue7MiJgUEZPmzZvXXuFKkiRJXU6HTSgioielZOJ3mflngMx8ITNXZ2Y98HNgv6auzcxLMnNcZo4bPHhw+wUtSZIkdTEdMqGIiAB+CTyamd8rKx9Sdtp/ANPbOzZJkiRJr+uQg7KBg4EPAA9FxJSi7IvAeyNiDJDATOAjtQhOkiRJUkmHTCgy804gmjh0Q3vHIkmSJKl5HbLLkyRJkqTOwYRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsVMKCRJkiRVzIRCkiRJUsU6ZEIREdtHxD8i4pGIeDgiPlGUD4iImyPiieJ3/1rHKkmSJHVlHTKhAFYBn8nMPYEDgI9FxJ7AecDfM3MX4O/FviRJkqQa6ZAJRWbOycwHiu0lwKPAtsCJwOXFaZcDb6tJgJIkSZKADppQlIuIYcDewL3A1pk5pzg0F9i6VnFJkiRJ6uAJRURsDvwJ+GRmLi4/lpkJZDPXnRkRkyJi0rx589ohUkmSJKlr6rAJRUT0pJRM/C4z/1wUvxARQ4rjQ4AXm7o2My/JzHGZOW7w4MHtE7AkSZLUBXXIhCIiAvgl8Ghmfq/s0LXAqcX2qcA17R2bJEmSpNf1qHUAzTgY+ADwUERMKcq+CHwLuCIizgCeAU6qTXiSJEmSoIMmFJl5JxDNHD6qPWORJEmS1LwO2eVJkiRJUudgQiFJkiSpYiYUkiRJkipmQiFJkiSpYiYUkiRJkipW1VmeImIEMABYCDxdrG4tSZIkaSPR5i0UEbFPRFweEQuAJ4H7it8LivJ92rpOSZIkSbXRpglFRPwauBp4Dng7MAjoVfx+G/As8OfiPEmSJEmdXFt3eboLOD0zVzcoXwjcDtweEecDZ7RxvZIkSZJqoE0Tisz8WQvOWQ1c0pb1SpIkSaqNqg7KXiMijgL2Bp7MzL+0R52SJEmSqq/qCUVEfBE4FrgbeGdEHJ+ZZ1a7Xklv7Nmv7VXrENTJ7fDfD9U6BElSjbV5QhER+2TmA2VFxwCHZ2ZGxKbALMCEQpIkSdoIVGNhu59HxDcjolexPw/4QETsAnwYmFmFOiVJkiTVQDUSiv2BV4H7I+JA4OPAccBfgKOAk6tQpyRJkqQaaPMuT5m5CvhqRPwJ+Dmlhe0+lJnL2rouSZIkSbVVjRYKADJzOnAwpUXu7i9mepIkSZK0EWnzhCIiDoyIqRGxFLgTmAicCHwpIn4REVu2dZ2SJEmSaqMaLRSXAucDA4HvAj/IzCcz8whgMnBvFeqUJEmSVAPVSCgGAbdk5qvArcDgNQcy82Lg6CrUKUmSJKkGqrGw3U+ByRFxP7Af8J3yg5n5fBXqlCRJklQD1Zjl6UvFDE87Axdk5qNtXYckSZKkjqEaLRRk5oPAg9W4tyRJkqSOo03HUETEVyOizxucs3lEfLUt65UkSZJUG23dQrEJ8HREXAPcDDwCLAa2BPYE3gy8DfhFG9crSZIkqQbaNKHIzPMi4ifAWZSmjt0dSCCAx4G/AOMy89m2rFeSJElSbVRjUPazwBeBL0ZEb6A/8FJmrmjruiRJkiTVVlUGZa9RJBFzqlmHJEmSpNqpxsJ2kiRJkroIEwpJkiRJFTOhkCRJklQxEwpJkiRJFatqQhER74+ImyNiWrF/aES8vZp1SpIkSWo/VUsoIuLTwFeBG4EdiuJ5wOeqVackSZKk9lXNFoqPAm/JzO9RWtwO4N/AzlWsU5IkSVI7qmZCMSAz/11sr0koomxbkiRJUidXzYTikYgY36DsOGBqSy6OiF9FxIsRMb2s7PyImBURU4qf49syYEmSJEkbpporZX8RuD4irgA2iYiLgPcADZOM5lwG/Aj4dYPy72fmhW0WpSRJkqSKVa2FIjPvAA4AlgP/KOo6PDPvbeH1twMLqxWfJEmSpNarWgtFRAzLzEeAcxqU75iZz7Ti1mdHxCnAJOAzmflSa+KUJEmSVLlqjqGY1kz5g62458XATsAYYA7w3aZOiogzI2JSREyaN29eK6qTJEmStD7VTCiiUUFET1oxy1NmvpCZqzOzHvg5sF8z512SmeMyc9zgwYMrrU6SJEnSG2jzLk8RcTOlpGGTiLipweEdgAdace8hmTmn2P0PYPr6zpckSZJUXdUYQ3Fn8fsw4K6y8npgLnBlS24SEX8ADgcGRcTzwFeAwyNiDKWEZSbwkTaJWJIkSVJF2jyhyMyvAkTEo5l5RSvu894min9ZcWCSJEmS2lzVZnlak0xERG9gMGVjKjLz2WrVK0mSJKn9VHPa2BHAb4H9mzjcvVr1SpIkSWo/1Zzl6UfAc8BoYAlQB/wFOKOKdUqSJElqR1VroaDUMjEsM5dEBJn5cER8BLgNuKyK9UqSJElqJ9VsoagHlhfbSyOiH7CQ0tSxkiRJkjYC1WyheBg4mFKLxL3A94FXgKerWKckSZKkdlTNFoqPU2qRADgX2BYYh2tHSJIkSRuNak4bO61sewZwDEBE9K1WnZIkSZLaVzVbKNYREb0j4nPAjPaqU5IkSVJ1tXlCERHDIuLWiFgcEfdFxK4RcSzwBKUpYz/e1nVKkiRJqo1qdHn6XvH7PODdwJXAAODLwK8zs74KdUqSJEmqgWokFAcBIzNzQUT8GZgNjMvMB6pQlyRJkqQaqsYYis0ycwFAZs4FlppMSJIkSRunarRQREQMAaLYX91gn8ycXYV6JUmSJLWzaiQUfYDny/ajbD+ABLpXoV5JkiRJ7awaCcXwKtxTkiRJUgfU5glFZj7T1veUJEmS1DG128J2kiRJkjY+JhSSJEmSKmZCIUmSJKliVUsoImJYM+U7VqtOSZIkSe2rmi0U05opf7CKdUqSJElqR9VMKKJRQURPSutQSJIkSdoItPm0sRFxM6WkYZOIuKnB4R2AB9q6TkmSJEm1UY2F7e4sfh8G3FVWXg/MBa6sQp2SJEmSaqAaC9t9FSAiHs3MK9r6/pIkSZI6jmq0UACQmVdExObAeGA74Hng+sxcUq06JUmSJLWvqiUUETEOuAFYDjxLafzEDyPi+MycVK16JUmSJLWfas7y9BPgu5m5Y2Yekpk7AhcCF1exTkmSJEntqJoJxR7AdxuUfQ/YvYp1SpIkSWpH1UwopgCjGpTtVZRLkiRJ2ghUbQwFcBMwMSJ+ATwDDAM+CFwSEe9bc1Jm/r6KMUiSJEmqomomFB8EVgKnlpWtKsrXSMCEQpIkSeqkqjlt7PBq3VuSJElSx1DNMRQARMTQiDig2vVIkiRJan9VSygiYquIuIXSgna3FGXvjoifVKtOSZIkSe2rmi0UPwSeBgZTGksBcCtwdEsujohfRcSLETG9rGxARNwcEU8Uv/u3edSSJEmSWqyaCcURwDmZuYDS4Gsycx6wVQuvvww4rkHZecDfM3MX4O/FviRJkqQaqWZC8SoNBn1HxABgYUsuzszbmzj3RODyYvty4G2tC1GSJElSa1QzobgJ+G5E9Cwr+ypwfSvuuXVmzim25wJbt+JekiRJklqpmgnF54A9gJeALSNiEVAHfKktbp6ZSdGVqqGIODMiJkXEpHnz5rVFdZIkSZKaUM11KBYCh0bEWGA4pdWyJxWJQKVeiIghmTknIoYALzZT9yXAJQDjxo1rTX2SJEmS1qOa08b2j4g+mTk5M6/KzPuBzSKiXytuey2vr7x9KnBNa+OUJEmSVLlqdnm6FhjZoGwULUwCIuIPwD3AbhHxfEScAXwLODoingDeXOxLkiRJqpGqdXmilExMalA2CdirJRdn5nubOXRUa4KSJEmS1Haq2UKxAtisQVkfXl/kTpIkSVInV82E4k7gmxHRDSAiAvgacFcV65QkSZLUjqrZ5elc4FbgHRExg9JMT68BR1axTkmSJEntqJrTxj4TEaOA8cAwYCZwfWYuq1adkiRJktpXNVsoyMzlwJXVrEOSJElS7VRzHYqbIuLIBmVHRcSN1apTkiRJUvuq5qDsfYDbG5TdAexbxTolSZIktaNqJhT1QM8GZd2BqGKdkiRJktpRNROKycA5DcrOBh6oYp2SJEmS2lE1B2V/HvhnRLwD+DewC7AbcHgV65QkSZLUjqrWQpGZ04A9gauAxcCfgD0zc2q16pQkSZLUvqo9bexc4Dtr9iNiZER8ITM/Xs16JUmSJLWPao6hACAiNomIUyLiLuAhSrM/SZIkSdoIVK2FIiL2BD4CvB/YjFLyclxm3lStOiVJkiS1rzZvoYiID0TEHcB04DDgfGBbYCEwpa3rkyRJklQ71WihuBxYAJyQmWtXxY5w+QlJkiRpY1ONMRRfBpYCf4mIqyNiQkRUfayGJEmSpPbX5g/6mfkNYATwNiApTRc7C+gHDG3r+iRJkiTVTlVaDrLkxsx8O7Aj8BNgLnB/RFxRjTolSZIktb+qd0XKzDmZ+XVKrRYnAr2qXackSZKk9lHVhe3KZWYCNxQ/kiRJkjYCDpaWJEmSVDETCkmSJEkVM6GQJEmSVDETCkmSJEkVM6GQJEmSVDETCkmSJEkVM6GQJEmSVDETCkmSJEkVM6GQJEmSVDETCkmSJEkVM6GQJEmSVDETCkmSJEkVM6GQJEmSVDETCkmSJEkVM6GQJEmSVDETCkmSJEkV61HrACoRETOBJcBqYFVmjqttRJKkjuDgiw6udQjq5O46565ahyB1Op0yoSgckZnzax2EJElSNd126GG1DkGd3GG331bV+9vlSZIkSVLFOmtCkcBNETE5Is5seDAizoyISRExad68eTUIT5IkSeoaOmtC8abM3Ad4C/CxiDi0/GBmXpKZ4zJz3ODBg2sToSRJktQFdMqEIjNnFb9fBK4G9qttRJIkSVLX1OkSiojoExFbrNkGjgGm1zYqSZIkqWvqjLM8bQ1cHRFQiv/3mfnX2oYkSZIkdU2dLqHIzBnA6FrHIUmSJKkTdnmSJEmS1HGYUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqWKdMKCLiuIh4PCKejIjzah2PJEmS1FV1uoQiIroDPwbeAuwJvDci9qxtVJIkSVLX1OkSCmA/4MnMnJGZrwH/B5xY45gkSZKkLqkzJhTbAs+V7T9flEmSJElqZz1qHUA1RMSZwJnF7tKIeLyW8XQhg4D5tQ6iI4sLT611CNowfqbfyFei1hFow/m5Xo/4uJ/pTsjP9BuJNvtc79hUYWdMKGYB25ftb1eUrZWZlwCXtGdQgoiYlJnjah2H1Fb8TGtj5OdaGxs/07XXGbs83Q/sEhHDI6IX8B7g2hrHJEmSJHVJna6FIjNXRcTZwN+A7sCvMvPhGoclSZIkdUmdLqEAyMwbgBtqHYcasZuZNjZ+prUx8nOtjY2f6RqLzKx1DJIkSZI6qc44hkKSJElSB2FCIUmSJKliJhSSJEmSKmZCIUmSJKliJhSSJEmSKmZCIUmSJKlinXIdig0xaNCgHDZsWK3DkCRJkjq1yZMnz8/MwQ3LN/qEYtiwYUyaNKnWYUiSJEmdWkQ801S5XZ4kSZIkVcyEQpIkSVLFTCgkSZIkVcyEQpIkSVLFNvpB2aq+qc8t4s4n5wPwpp0HMXr7frUNSJIkSe3GhEIVW7JiJWf//kFu+/e8tWXf+dvjHLbrYH70vr3ZonfPGkYnSZKk9mCXJ1Xs3CunrZNMrHHbv+fxuaum1SAiSZIktTcTClXk6fmv8NeH5zZ7/Mbpc5k5/5V2jEiSJEm1YEKhikyaufANz7nrqfntEIkkSZJqyTEUqkivHm+ci55/7cPc/eQCJowewuG7bUXvnt3bITJJkiS1JxMKVeSQXQbTq0c3XltV3+w5K1cn1z80h+sfmkOfXt05ZuQ2jK8bsvZaSZIkdX4mFKrIgD69+PAhw/nxP55q8vi4Hfuz4JXXeLoYR/HKa6u5+sFZXP3gLLbs3YPjRm3DhNFDOXDEQHp0N7mQJEnqrEwoVLHPHL0bvXt055I7ZrBkxSoAtujdgzMPGcHHjtiZCHh49mKumzabiVPnMGvRcgAWr1jFFZOe54pJzzOwTy/estc2jK8byr7DBtC9W9TyJUmSJGkDRWbWOoaqGjduXE6aNKnWYWzUlr22imnPvwxA3XZ92axX4zw1M3nwuUVMnDqH6x+azQuLX210ztZbbsLxew1hwuih7L19PyJMLiRJkjqKiJicmeMalZtQqL3V1yf3z1zIddNmc+NDc1nwymuNztm236aMHz2ECXVDGTl0S5MLSZKkGqtpQhERuwF/LCsaAfw38OuifBgwEzgpM1+K0tPjD4DjgWXAaZn5QHGvU4EvFfe5IDMvX1/dJhQd26rV9dwzYwETp87hxulzWFx0nSo3fFAfxteVWi523XqLGkQpSZKkDtNCERHdgVnA/sDHgIWZ+a2IOA/on5mfj4jjgXMoJRT7Az/IzP0jYgAwCRgHJDAZGJuZLzVXnwlF5/HaqnrufHIe102dw82PvMDSVxsnF7tuvTkT6oYyfvRQhg/qU4MoJUmSuqaOlFAcA3wlMw+OiMeBwzNzTkQMAf6ZmbtFxM+K7T8U1zwOHL7mJzM/UpSvc15TTCg6pxUrV/PPx1/kumlz+PujL7BiZePpaUdtuyXj64Yyvm4I2/XfrAZRSpKkWlldnzw1bymr65Odt9qcns4aWXXNJRS1mOXpPcCaBGDrzJxTbM8Fti62twWeK7vm+aKsuXJtZHr37M5xo4Zw3KghvPLqKv7+2ItcN3U2tz0+j9dWl5KL6bMWM33WYr5142PsvUM/JtQN5YS6IWy9Ze8aRy9JkqrpyknP8f9ueWLtDJKDNt+Esw4bwRlvGu64yxpo14QiInoBbwW+0PBYZmZEtElzSUScCZwJsMMOO7TFLVVDfTbpwVtHD+Wto4eyeMVKbnr4BSZOm82dT8xnVX3pI/Pgs4t48NlFfP36R9hv2ADGjx7K8aO2YeDmm9Q4eknqGFasXM3sRcvZctOeDPJvozqx39wzky9f8/A6ZfOXvsoF1z/Ky8tX8pljdqtRZF1Xu3Z5iogTgY9l5jHFvl2eVLGXXnmNvz48l+umzuZfMxZQ3+Cj3L1bcNBOA5lQN5RjR25D38161iZQSaqhFStX892bHucP9z23dmzawTsP5IvH78HIoX1rHJ1UkpnUZ6kbU33m2t/19bC6bP+VV1dx4o/uYkkT4ywBenYP7j7vKAZvYdJcDR1iDEVE/B/wt8y8tNj/DrCgbFD2gMz8XEScAJzN64Oyf5iZ+xWDsicD+xS3fIDSoOyFzdVpQtE1vLhkBTc+NJeJ02Zz/8zGY/R7dg8O3WUw40cP4eg9t2HzTVzTUdLGr74+Oe2y+7n93/MaHdusV3f+9NGD2GPIljWIrPOpr8+1D7aZZQ+5RXn58fp6Sg/FTZSvXudhec05vH5ug/vVJ02Xry0rPYyvc7z8fsXv0kM6ax/Wm6qvPteN4/VzWaesufI198oiMWiYHLxeRqPyhl8Ktsb/vqOOk/bdvu1uqLVqPoYiIvoARwMfKSv+FnBFRJwBPAOcVJTfQCmZeJLStLGnA2Tmwoj4OnB/cd7X1pdMqOvYaovenHrQME49aBizFy3nhofmcN3U2UwtFtxbuTr5+2Mv8vfHXmSTHg9xxG5bMWH0UI7cfSs27dW9xtFLUnX84/EXm0wmAJa9tpqvXfcI5791ZONvhYuHz4YPivXreSB8fZviobJxecNzmytv+KC5zkP52nNpIubGD+1r7ptJEw/4a7Yb3KvBg/7qtnzaVdUte63p1gtVjwvbaaP27IJlXDdtNhOnzeHROYsbHd+sV3fevMfWjK8bwmG7DWaTHiYXkjYe//m7ydzw0Nxah6FOrnu3oHsE3bpR+h1Bt25B926l7e5ryouy7hFEUHY81t0u7rVmf93fNCorL1/22mqumTJ7vfFe/Z8HsfcO/dvp3elaat5CIdXCDgM342NH7MzHjtiZJ19cysRps7lu6myemvcKUPqG7tqps7l26my26N2DY/bchgmjh3DwzoOcfk5Sp/PyspX86+kF3PPUAu5+aj7/fmFprUOqum4NH1zLHmYbP6yu2W7iYbZb0D2aeJhtqnzNQ3Gwdru8vPzh+PV7lB9f97qIsjoanVva7hY0Wb7Og34TD+3dG9yvW3PlTSQDa8o7mpWr65tNlMfu2J8x2/dr34BkC4W6nszksblLuG5qqeXi2YXLGp3Tf7OeHDdqCBPqhrD/iIF074B/UCXplVdXcd/MhfzrqQXc/dQCps9+mQ353/rmm/Tg5P13oEf3FjxcNnpYLXtgbvTQTqNzyx+wm/vmuvxBv6lvtNf9xrp0vlOEdj2LV6zkw5dP4t6n1+31vueQLbns9H3Zyunjq6ZDDMquBRMKrU9mMu35l5lYdIua8/KKRucM3mITjh+1DRNGD2WfHfp3yG9rJHUNK1au5oFnXypaIBYw9blFa6fPbqhPr+6M2b4f981cyMrVTZ/zpRP24EOHjKhmyFJVZCZ3P7WA2/49j9X1yYEjBnLE7lv5BWCVmVBIb6C+Pnng2Ze4bupsrn9oLvOXvtronCF9ezO+bgjj64ZSt11fvxmTVFUrV9cz7flF3P3kAu6ZsYBJz7zEa6vqmzx3kx7dGDesPwftNIgDdxrIXtv2pWf3btzxxDzO+u1kXnl19Trnv2ff7fnmf+zllySSWsyEQtoAq+uTe2cs4Lppc7hx+hwWLVvZ6JwdBmzG+LohTBg9lN232cLkQlKrra5PHp2zmLufms/dTy3gvqcXsuy11U2e26NbsPcO/ThwxEAO3GkQe+/Qj949m55YYtGy1/jTA7N4bM5i+m7ak7eOGUrddv2q+EokbYxMKKQKrVxdz11Pzue6qXO46eG5TS6ms9PgPkwYPZTxdUPZeavNaxClpM4oM3nixaXc/WQpgfjXjAUsXtH0lJfdAkZt25cDdxrIQTsNYtyO/enjmjqS2pEJhdQGVqxcze3/nsfEaXO45dEXmvzmcI8hW5ZaLuqGssPAzWoQpaSOKjN5ZsEy7n6q1IXpnqfmM3/pa82ev/s2W3DgTgM5cMRA9h8+kL6b9WzHaCVpXSYUUhtb/tpqbn3sRa6bOptbH3+xyX7No7fry4TRQzmhbghD+m5agygl1drsRcvXDqK+56n5zG5i8oc1RgzqwwE7DeSgnQZywIiBDNp8k3aMVJLWz4RCqqIlK1Zyy6MvMHHqHG5/Yl6TM6rsO6w/4+uG8pa9tmGrLZzSTtpYzV/66toE4l8zFvD0/FeaPXfbfpsWXZgGcuBOA/3iQVKHZkIhtZOXl63kbw/P5bpps7n7qQWsbjClY7eAA0YMZMLooRw3chv69+lVo0gltYXyxeTueWoBj7+wpNlzB22+CQeVJRA7DNjMCR0kdRomFFINzF/6KjdOn8vEqbO5b+bCRgtO9egWvGmXQYyvG8oxI7dmy972j5Y6uldeXcX9MxeubYVY32JyfTftyQEjBnDQToM4aKeB7LzV5iYQkjotEwqpxl5YvILrp83hummzefDZRY2O9+rejcN2G8yE0UN58x5bsVkvZ2+ROoLyxeTueWoBU95gMbn9hg9YuxbEHkO2dKEtSRsNEwqpA3lu4TKuf2gOE6fNZvqsxY2O9+7ZjaP22JoJdUM4fLetmp1bXlLbKy0m9zL3FGtBtGQxuTVrQdRtV1pMTpI2RiYUUgf19PxXmDh1NtdNm82/X1ja6Pjmm/Tg6D23ZsLoIbxp58H06uHDitSWGi4md//TC3llPYvJjdm+XzEGYv2LyUnSxsaEQuoEHp+7hInTZjNx2pwmZ4bpu2lPjhu5DeNHD+HAEQPp4Teh0gYrX0zunhkL+NeMhby8fGWT50bAXi4mJ0mACUWtw5A2SGby8OzFXDdtNhOnzmHWouWNzhnYpxdv2WsbJtQNZd9hA+hmP22pSZnJswtLi8ndXYyDmL/01WbP332bLThgRGkmJheTk6TXmVBInVRm8uBzi7hu6myunzaHF5c0fhDaZsveHL/XECaMHsKY7fs5i4y6vDkvL+fuJ19fC6KppHyN4YP6rF0LwsXkJKl5JhTSRmB1fXL/zIVMnDabGx6ay8JXXmt0znb9N2V83VDG1w1h5NAtTS7UJaxZTO6eGaUWiPUtJje0b28O2nmQi8lJ0gYyoZA2MqtW13PPjAVcN3U2f50+l8UrVjU6Z8SgPoyvG8KE0UPZZestahClVB0vL1/JvTNe78L0RovJrWmBOMjF5CSpYiYU0kbstVX13PHEPCZOm8NND89tcoaa3bbeggmjhzC+bijDBvWpQZRS5coXk7tnxgKmz3qZZpaCcDE5SaoSEwqpi1ixcjX/fPxFrps6h78/9gIrVjaeP3+vbfsyvm4IJ9QNYbv+m9UgSmn9VqxczYPPLlq7FkRLFpNbMxOTi8lJUnWYUEhd0CuvruKWR19g4rQ53Pb4PF5b3Ti52GeHfkwYPZQT9hrCVlv2rkGUUuPF5CY/8xKvNrOYXK8e3Ri3Y/+1a0G4mJwktY+aJxQR0Q/4BTAKSOCDwOPAH4FhwEzgpMx8KUpt0z8AjgeWAadl5gPFfU4FvlTc9oLMvHx99ZpQSCUvL1/JzY+8wHVTZ3PXk/MbfdsbAfsPH8D4uqG8ZdQ2DHSmG1VR+WJy9zy1gPtcTE6SOryOkFBcDtyRmb+IiF7AZsAXgYWZ+a2IOA/on5mfj4jjgXMoJRT7Az/IzP0jYgAwCRhHKSmZDIzNzJeaq9eEQmps4Suv8dfpc5k4bTb/mrGgUV/07t2Cg3YayITRQzl2z22ch1+tlpk8+eLSYi2I+S1eTO7AEQPZd9gAF5OTpA6gpglFRPQFpgAjsqzCiHgcODwz50TEEOCfmblbRPys2P5D+XlrfjLzI0X5Ouc1xYRCWr8Xl6zgxofmct3U2Ux6pnFu3rN7cNiugxlfN5Q377k1m/tgpxbY0MXkdtt6i7UzMbmYnCR1TM0lFO31ZDAcmAdcGhGjKbUsfALYOjPnFOfMBbYutrcFniu7/vmirLlySRXaaovenHrQME49aBizFy3n+mlzmDhtNlOffxmAlauTWx59kVsefZFNenTjyN23YsLooRyx21Zs2stuJ3rdnJeXc09ZAuFicpLUNbRXQtED2Ac4JzPvjYgfAOeVn5CZGRFt0lwSEWcCZwLssMMObXFLqUsY2m9TPnzoCD586AieWfAKE6fN4bqps3lsbmmO/1dX1XPj9LncOH0um/XqztF7bs34uqEcuusgNulhctHVzF/6Kv8qWwvijRaTO3Cn1xeTG9rPxeQkaWPRXl2etgH+lZnDiv1DKCUUO2OXJ6nDe/LFJVw3tdRy8dS8xg+NW/TuwbEjt2F83RAO3nmQM+5spDZsMbleaxMIF5OTpI1DRxiUfQfwocx8PCLOB9asrLWgbFD2gMz8XEScAJzN64Oyf5iZ+xWDsidTau0AeIDSoOyFzdVrQiG1nczk0TlLmDhtNtdNm81zCxt3aem/WU+OGzWECaOHsP/wga4H0Ikte20V9898ae1MTOtbTG7L3j3WrgNx4E4D2cXF5CRpo9MREooxlKaN7QXMAE4HugFXADsAz1CaNnZhMW3sj4DjKE0be3pmTiru80FKs0MBfCMzL11fvSYUUnVkJtOef5nrps7m+ofmMOflFY3OGbzFJpyw1xDG1w1hnx36083kokPb0MXk9h0+oGiBcDE5SeoKap5Q1IoJhVR99fXJ5GdfYuLU2Vz/0NwmZ/MZ2rc3J9QNYcLooey1bV+/ve4AyheTu2fGAibNdDE5SVLzTCgktYvV9cm9MxZw3bTZ3Dh9LouWNV5rYMeBmzG+bgjj64ay+zZbmFy0k/r65JE5i4uZmOa3eDG5A3YayD479HcxOUnq4kwoJLW7lavrufPJ+UycOoebHp7LkldXNTpn5602Z3zRcrHT4M1rEOXGa0MXkxs1tO/aWZhcTE6S1JAJhaSaWrFyNbf/ex7XTZvDLY+8wPKVjb8Z32PIlkwYPYQJdUPZfsBmNYiyc1uzmNyatSDudjE5SVIbMqGQ1GEse20Vtz72IhOnzuHWx1/ktSb67Y/evh8T6oZwQt0QhvR1zYLmVLKY3IEjSovJDd7CxeQkSS1nQiGpQ1qyYiW3PPoC102dwx1PzGPl6sZ/k/Yd1p8Jo4fyllFDuvxDcPlicv96agEzXExOktROTCgkdXiLlr3G3x6ey8Rpc7jryfmN1jzoFnDgTgMZXzeU40ZuQ/8+vWoTaDt6eflK7nt64dq1INasWt6U8sXkDhwxkB0HupicJKntmFBI6lTmL32VG6fP5bqps7l/5kIa/qnq0S140y6DmFA3lKNHbs2WvTeO/v8bupjcASNKYyAO2nmQi8lJkqrKhEJSpzX35RVc/9AcJk6bzYPPLmp0vFePbhy+62DGjx7Km/fYis16dZ7ZidYuJjdjAfc8NZ8pzy1qstsXwGa9urOfi8lJkmrEhELSRuG5hcu4/qE5XDd1Ng/PXtzo+KY9u3PkHlsxoW4oh+82uMOtnbBqdT3TZr28di0IF5OTJHUWJhSSNjoz5i1l4rRScvHEi0sbHd98kx4cs+fWjB89hDftPJhePdr/YbySxeQOLAZRu5icJKkjMaGQtFF7fO4SJk6bzXVTZzNzwbJGx/tu2pO3jNqG8XVDOWDEAHpU6Zv+8sXk7nlqAf96ekGTq4WDi8lJkjoXEwpJXUJm8vDsxVw3dTYTp81pcl2GQZv34i2jSqtzj9uxP90ajEN4cfEKJk6bw0vLXmPnrTbn2JHbNNtSkJk8t3A5dz81f4MWkztwp4Ec4GJykqROxIRCUpeTmTzw7CImTpvN9dPm8OKSxg/622zZmxPqhjC+bghjtu/HJbfP4Dt/e5xVZVMrDdq8Fz99/1jGDRsAlAaJr0kg3mgxuWEDN1s7lauLyUmSOjMTCkld2ur65P6ZC7lu6mxunD6Xha+81uicgX16saCJcoDePbtx/KghTHlu0XoXkxvStzcHuZicJGkjZEIhSYVVq+u5+6kFTJw2m79On8viFasqvteaxeQOLNaDcDE5SdLGyoRCkprw2qp67nhiHtdMmc21U2e/4fkuJidJ6qqaSyicTkRSl9arRzeO2mNrDtt1MBOnzW52VWqA/Yb15w9nHuhicpIklXGFJEkCenTvxpG7b73ec94xdjuTCUmSGjChkKTCJ9+8C5s0s/jdbltvwYljtm3niCRJ6vhMKCSpMGrbvvzmjP3ZfZst1pZ1Czh25Nb87sP7u2q1JElNcAyFJJXZb/gAbvzEITz+whJeemUlwwf1YZu+vWsdliRJHZYJhSQ1EBHsvs2WtQ5DkqROod26PEXEzIh4KCKmRMSkomxARNwcEU8Uv/sX5RERP4yIJyNiWkTsU3afU4vzn4iIU9srfkmSJEmNtfcYiiMyc0zZ/LXnAX/PzF2Avxf7AG8Bdil+zgQuhlICAnwF2B/YD/jKmiREkiRJUvur9aDsE4HLi+3LgbeVlf86S/4F9IuIIcCxwM2ZuTAzXwJuBo5r55glSZIkFdozoUjgpoiYHBFnFmVbZ+acYnsusGYS+G2B58qufb4oa65ckiRJUg2056DsN2XmrIjYCrg5Ih4rP5iZGRHrWaO25YqE5UyAHXbYoS1uKUmSJKkJ7dZCkZmzit8vAldTGgPxQtGVieL3i8Xps4Dtyy7frihrrrxhXZdk5rjMHDd48OC2fimSJEmSCu2SUEREn4jYYs02cAwwHbgWWDNT06nANcX2tcApxWxPBwAvF12j/gYcExH9i8HYxxRlkiRJkmqgvbo8bQ1cHRFr6vx9Zv41Iu4HroiIM4BngJOK828AjgeeBJYBpwNk5sKI+Dpwf3He1zJzYTu9BkmSJEkNRGabDFvosMaNG5eTJk2qdRiSJElSpxYRk8uWf1ir1tPGSpIkSerETCgkSZIkVcyEQpIkSVLFTCgkSZIkVcyEQpIkSVLFTCgkSZIkVcyEQpIkSVLFTCgkSZIkVaxFK2VHxC7AS5k5PyL6AOcCq4HvZOaKagYoSZIkqeNqaQvFH4Btiu1vAm8HTgT+XxVikiRJktRJtKiFAhgBPFxsvxM4GFgCPAScVYW4JEmSJHUCLU0oAugeETsDyzJzJkBEbFGtwCRJkiR1fC1NKO4Ffkyp29MNABExDFhYnbAkSZIkdQYtHUPxEWBzSgnE14qy/YDfVyMoSZIkSZ1Di7s8ZebJ5QWZeUVE3FuFmCRJkiR1Ei1toZjWTPmDbRWIJEmSpM6npQlFNCqI6Alk24YjSZIkqTNZb5eniLiZUtKwSUTc1ODwDsAD1QpMkiRJUsf3RmMo7ix+HwbcVVZeD8wFrqxGUJIkSZI6h/UmFJn5VYCIeDQzr2ifkCRJkiR1Fi2a5amY0WlzYDywHfA8cH1mLqlmcJIkSZI6thYlFBExjtKCdsuBZymNn/hhRByfmZOqGJ8kSZKkDqylszz9BPhuZu6YmYdk5o7AhcDFG1JZRHSPiAcjYmKxPzwi7o2IJyPijxHRqyjfpNh/sjg+rOweXyjKH4+IYzekfkmSJEltq6UJxR7AdxuUfQ/YfQPr+wTwaNn+t4HvZ+bOwEvAGUX5GcBLRfn3i/OIiD2B9wAjgeOAn0RE9w2MQZIkSVIbaWlCMQUY1aBsr6K8RSJiO+AE4BfFfgBHAlcVp1wOvK3YPrHYpzh+VHH+icD/Zearmfk08CSwX0tjkCRJktS2WjSGArgJmBgRvwCeAYYBHwQuiYj3rTkpM3+/nnv8P+BzwBbF/kBgUWauKvafB7YttrcFnivuuSoiXi7O3xb4V9k9y6+RJEmS1M5amlB8EFgJnFpWtqooXyOBJhOKiBgPvJiZkyPi8A0Pc8NExJnAmQA77LBDtauTJEmSuqyWThs7vJX1HAy8NSKOB3oDWwI/APpFRI+ilWI7YFZx/ixge+D5iOgB9AUWlJWvUX5NebyXAJcAjBs3LlsZuyRJkqRmtHQMBQARMTQiDtjQSjLzC5m5XWYOozSo+tbMPBn4B/DO4rRTgWuK7Wt5vTXkncX5WZS/p5gFajiwC3DfhsYjSZIkqW20KKGIiK0i4hZKYxZuKcreHRE/aWX9nwc+HRFPUhoj8cui/JfAwKL808B5AJn5MHAF8AjwV+Bjmbm6lTFIkiRJqlCUvvh/g5Mi/g9YQunB/snM7B8Rg4G7M3OXKsfYKuPGjctJk1x7T5IkSWqNiJicmeMalrd0UPYRwI6ZuSIiEiAz50XEVm0ZpCRJkqTOpaVjKF6lQfIREQOAhW0ekSRJkqROo6UJxU3AdyOiZ1nZV4Hr2z4kSZIkSZ1FS7s8fQ74C/AS0DsiFgFTKa1cLUmSJKmLauk6FAuBQyNiLDCc0mrZk7IlI7olSZIkbbRalFBERH/gtcycDEwuyvpERM/MXFTF+CRJkiR1YC0dQ3EtMLJB2SheX4hOkiRJUhfU0oRiJNBwMYdJwF5tG44kSZKkzqSlCcUKYLMGZX2AlW0bjiRJkqTOpKUJxZ3ANyOiG0BEBPA14K5qBSZJkiSp42vptLHnArcC74iIGZRmenoNOLJagUmSJEnq+Fo6bewzETEKGA8MA2YC12fmsuqFJkmSJKmja2kLBZm5HLiyirFIkiRJ6mRaNIYiIm6KiCMblB0VETdWJyxJkiRJnUFLB2XvA9zeoOwOYN+2DUeSJElSZ9LShKIe6NmgrDsQbRuOJEmSpM6kpQnFZOCcBmVnAw+0bTiSJEmSOpOWDsr+PPDPiHgH8G9g1+Ln8CrFJUmSJKkTaFELRWZOA/YE/gQspjTb04nAGdULTZIkSVJH19IuT2TmXOAHwH3AfwD/oDRYW5IkSVIX1aIuTxGxJ/AR4P3AZpQSkbdk5k1VjE2SJElSB7feFoqI+EBE3AFMBw4Dzge2BRYCU6odnCRJkqSO7Y26PF0O7A6ckJljMvOizFy4oZVERO+IuC8ipkbEwxHx1aJ8eETcGxFPRsQfI6JXUb5Jsf9kcXxY2b2+UJQ/HhHHbmgskiRJktrOGyUUXwaWAn+JiKsjYkJEtHjcRZlXgSMzczQwBjguIg4Avg18PzN3Bl7i9UHeZwAvFeXfL85b0/XqPcBI4DjgJxHRvYJ4JEmSJLWB9SYHmfkNYATwNiApzfI0C+gHDG1pJVmytNjtWfwkcCRwVVF+eVEPlGaQurzYvgo4KiKiKP+/zHw1M58GngT2a2kckiRJktrWG7Y2FMnAjZn5dmBH4CfAXOD+iLiipRVFRPeImAK8CNwMPAUsysxVxSnPUxqfQfH7uaL+VcDLwMDy8iaukSRJktTONqj7UmbOycyvU2q1OBHotQHXrs7MMcB2lFoVdt+QujdERJwZEZMiYtK8efOqVY0kSZLU5VUyHmJNq8UNmfm2Cq5dRGkNiwOBfhGxZura7Sh1p6L4vT1AcbwvsKC8vIlryuu4JDPHZea4wYMHb2iIkiRJklqoooRiQ0XE4IjoV2xvChwNPEopsXhncdqpwDXF9rXFPsXxWzMzi/L3FLNADQd2obTQniRJkqQaaNHCdm1gCHB5MSNTN+CKzJwYEY8A/xcRFwAPAr8szv8l8JuIeJLSmhfvAcjMh4txG48Aq4CPZebqdnoNkiRJkhqI0hf/G69x48blpEmTah2GJEmS1KlFxOTMHNewvF26PEmSJEnaOJlQSJIkSaqYCYUkSZKkiplQSJIkSaqYCYUkSZKkiplQSJIkSaqYCYUkSZKkiplQSJLUhSxYsIAxY8YwZswYttlmG7bddlvGjBlDv3792HPPPSu+7/nnn8+FF17YhpG2zGWXXcbZZ59d1fvPnj177f6wYcOYP39+1eqrhoavQWprJhSSJHUhAwcOZMqUKUyZMoWzzjqLT33qU2v3u3WrzWNBZlJfX1+Tut/IxvAwvjG8BnVsJhSSJAmA1atX8+EPf5iRI0dyzDHHsHz5cgCeeuopjjvuOMaOHcshhxzCY4891uT1U6dO5cADD2SXXXbh5z//OQBLly7lqKOOYp999mGvvfbimmuuAWDmzJnstttunHLKKYwaNYrnnnuO0047jVGjRrHXXnvx/e9/v9H9r7zySkaNGsXo0aM59NBD15bPnj2b4447jl122YXPfe5za8v/8Ic/sNdeezFq1Cg+//nPr73Hpz/9aQB+8IMfMGLECABmzJjBwQcfvE59V111FZMmTeLkk09mzJgxa9+Piy66aO3rWfNevPLKK3zwgx9kv/32Y++99177Oss1914A/PrXv6auro7Ro0fzgQ98AIAXXniB//iP/2D06NGMHj2au+++G4Df/va37LfffowZM4aPfOQjrF69GoDNN9+cT33qU4wcOZKjjjqKefPmNfkazjvvPPbcc0/q6ur47Gc/2+S/pbRBMnOj/hk7dmxKkqTGvvKVr+R3vvOdzMx8+umns3v37vnggw9mZua73vWu/M1vfpOZmUceeWT++9//zszMf/3rX3nEEUc0ea+6urpctmxZzps3L7fbbrucNWtWrly5Ml9++eXMzJw3b17utNNOWV9fn08//XRGRN5zzz2ZmTlp0qR885vfvPZ+L730UqM6Ro0alc8///w6xy+99NIcPnx4Llq0KJcvX5477LBDPvvsszlr1qzcfvvt88UXX8yVK1fmEUcckVdffXXOmTMnx40bl5mZ73jHO3LcuHH5/PPP52WXXZbnnXdeozoPO+ywvP/++9fu77jjjvnDH/4wMzN//OMf5xlnnJGZmV/4whfWvl8vvfRS7rLLLrl06dJ17tXcezF9+vTcZZddct68eZmZuWDBgszMPOmkk/L73/9+ZmauWrUqFy1alI888kiOHz8+X3vttczM/OhHP5qXX355ZmYC+dvf/jYzM7/61a/mxz72sUavYf78+bnrrrtmfX19s++z1BxgUjbxvN2j1gmNJEnqGIYPH86YMWMAGDt2LDNnzmTp0qXcfffdvOtd71p73quvvtrk9SeeeCKbbropm266KUcccQT33XcfJ5xwAl/84he5/fbb6datG7NmzeKFF14AYMcdd+SAAw4AYMSIEcyYMYNzzjmHE044gWOOOabR/Q8++GBOO+00TjrpJN7+9revLT/qqKPo27cvAHvuuSfPPPMMCxYs4PDDD2fw4MEAnHzyydx+++287W1vY+nSpSxZsoTnnnuO973vfdx+++3ccccd69xzfdacN3bsWP785z8DcNNNN3HttdeuHUeyYsUKnn32WfbYY4+112Vmk+/Frbfeyrve9S4GDRoEwIABAwC49dZb+fWvfw1A9+7d6du3L7/5zW+YPHky++67LwDLly9nq622AqBbt268+93vBuD9739/k6+nb9++9O7dmzPOOIPx48czfvz4Fr1maX1MKCRJEgCbbLLJ2u3u3buzfPly6uvr6devH1OmTHnD6yOi0f7vfvc75s2bx+TJk+nZsyfDhg1jxYoVAPTp02ftuf3792fq1Kn87W9/46c//SlXXHEFv/rVr9a5309/+lPuvfderr/+esaOHcvkyZObjHvVqlXrjfOggw7i0ksvZbfdduOQQw7hV7/6Fffccw/f/e533/A1ltdXXldm8qc//Ynddtut2evW9160VGZy6qmn8j//8z9veG7Dfw+AHj16cN999/H3v/+dq666ih/96EfceuutGxSD1JBjKCRJUrO23HJLhg8fzpVXXgmUHminTp3a5LnXXHMNK1asYMGCBfzzn/9k33335eWXX2arrbaiZ8+e/OMf/+CZZ55p8tr58+dTX1/PO97xDi644AIeeOCBRuc89dRT7L///nzta19j8ODBPPfcc83Gvd9++3Hbbbcxf/58Vq9ezR/+8AcOO+wwAA455BAuvPBCDj30UPbee2/+8Y9/sMkmm6xt5Si3xRZbsGTJkjd8n4499lguuugiSr1C4MEHH2x0TnPvxZFHHsmVV17JggULAFi4cCFQanm5+OKLgdL4lpdffpmjjjqKq666ihdffHHtuWvuU19fz1VXXQXA73//e970pjc1eg1Lly7l5Zdf5vjjj+f73/9+s/+W0oawhUKSJK3X7373Oz760Y9ywQUXsHLlSt7znvcwevToRufV1dVxxBFHMH/+fL785S8zdOhQTj75ZCZMmMBee+3FuHHj2H333ZusY9asWZx++ulrZ3tq6hv4c889lyeeeILM5KijjmL06NHNtpwMGTKEb33rWxxxxBFkJieccAInnngiUEoonnvuOQ499FC6d+/O9ttv32xcp512GmeddRabbrop99xzT7Pv0Ze//GU++clPUldXR319PcOHD2fixInrnNPcezFy5Ej+67/+i8MOO4zu3buz9957c9lll/GDH/yAM888k1/+8pd0796diy++mAMPPJALLriAY445hvr6enr27MmPf/xjdtxxR/r06cN9993HBRdcwFZbbcUf//jHRq/hxhtv5MQTT2TFihVkJt/73veafU1SS8WaTHpjNW7cuJw0aVKtw5AkSaqqzTffnKVLl9Y6DG3EImJyZo5rWG6XJ0mSJEkVM6GQJEnaCNg6oVoxoZAkqQtZsGABY8aMYcyYMWyzzTZsu+22jBkzhn79+rHnnntWfN/LLruMs88+GyjNxrRmutNqGDZsGPPnz6/a/TuKf/7zn2sXs2tLs2fP5p3vfGdF17rqtppiQiFJUhcycOBApkyZwpQpUzjrrLP41Kc+tXa/W7e2eSw466yzOOWUU9rkXl3Z+hKKww8/nJkzZzZ77fqmzh06dOja2aA2lAmFmmJCIUmSgNLUpB/+8IcZOXIkxxxzDMuXLwdK07Ued9xxjB07lkMOOYTHHntsvfc5//zz1y7wdvjhh/P5z3+e/fbbj1133ZU77rhjbV3nnnsu++67L3V1dfzsZz9rdJ9XXnmFE044gdGjRzNq1Ki1sxYBXHTRReyzzz7stddea+NZuHAhb3vb26irq+OAAw5g2rRpAOy1114sWrSIzGTgwIFrW09OOeUUbr75Zh5++GH2228/xowZQ11dHU888USjWD760Y8ybtw4Ro4cyVe+8pW15ffffz8HHXQQo0ePZr/99mPJkiWsXr2az372s4waNYq6ujouuugiACZPnsxhhx3G2LFjOfbYY5kzZ87a9+gTn/gEY8aMYdSoUdx3333MnDmTn/70p3z/+99nzJgxa9+39fnnP//JIYccwlvf+lb23HPPZt/jmTNnMmrUqDf8d/j2t7/NXnvtxejRoznvvPO46qqrmDRpEieffDJjxoxZ+/mQnDZWkiQB8MQTT/CHP/yBn//855x00kn86U9/4v3vfz9nnnkmP/3pT9lll1249957+c///M8NWgxt1apV3Hfffdxwww189atf5ZZbbuGXv/wlffv25f777+fVV1/l4IMP5phjjmH48OFrr/vrX//K0KFDuf7664HSOg5rDBo0iAceeICf/OQnXHjhhfziF7/gK1/5CnvvvTd/+ctfuPXWWznllFOYMmUKBx98MHfddRc77rgjI0aM4I477uCUU07hnnvu4eKLL+a8887jE5/4BCeffDKvvfYaq1evbvQavvGNbzBgwABWr17NUUcdxbRp09h9991597vfzR//+Ef23XdfFi9ezKabbsoll1zCzJkzmTJlCj169GDhwoWsXLmSc845h2uuuYbBgwfzxz/+kf/6r/9au3jfsmXLmDJlCrfffjsf/OAHmT59OmeddRabb745n/3sZ1v8Xj/wwANMnz6d4cOHc8kllzT5HpcveNfcv8Njjz3GNddcw7333stmm23GwoULGTBgAD/60Y+48MILGTeu0UQ/6sLaJaGIiO2BXwNbAwlckpk/iIgBwB+BYcBM4KTMfClKn/QfAMcDy4DTMvOB4l6nAl8qbn1BZl7eHq9BkqSN3fDhwxkzZgwAY8eOZebMmSxdupS7776bd73rXWvPe/XVVzfovm9/+9vXuSfATTfdxLRp09Z2vXn55Zd54okn1kko9tprLz7zmc/w+c9/nvHjx3PIIYc0ec8///nPANx555386U9/AkqLxS1YsIDFixdzyCGHcPvtt7Pjjjvy0Y9+lEsuuYRZs2bRv39/+vTpw4EHHsg3vvENnn/+ed7+9rezyy67NHoNV1xxBZdccgmrVq1izpw5PPLII0QEQ4YMYd999wVKiwAC3HLLLZx11ln06FF6zBowYADTp09n+vTpHH300UCpZWDIkCFr7//e974XgEMPPZTFixezaNGiRjFceuml/OAHPwDgySef5Pjjj6dXr14MHz6cq6++Gigt6LfmPWzuPd51113X3rO5c2655RZOP/10Nttss7WvQWpOe7VQrAI+k5kPRMQWwOSIuBk4Dfh7Zn4rIs4DzgM+D7wF2KX42R+4GNi/SEC+AoyjlJhMjohrM/OldnodkiRttDbZZJO12927d2f58uXU19fTr1+/ZheQ25D7du/efW3f/szkoosu4thjj232ul133ZUHHniAG264gS996UscddRR/Pd//3ez92zOoYceyo9//GOeffZZvvGNb3D11Vdz1VVXrU1Q3ve+97H//vtz/fXXc/zxx/Ozn/2MI488cu31Tz/9NBdeeCH3338//fv357TTTmPFihUb9B5kJiNHjmx2cbzyVoOm9gFOP/10Tj/9dKDUTeqyyy5j2LBh65zTp0+fdeps6j0uH3vR3Dl/+9vf3vA1SWu0yxiKzJyzpoUhM5cAjwLbAicCa1oYLgfeVmyfCPw6S/4F9IuIIcCxwM2ZubBIIm4GjmuP1yBJUle05ZZbMnz4cK688kqg9AA6derUVt/32GOP5eKLL2blypUA/Pvf/+aVV15Z55zZs2ez2Wab8f73v59zzz2XBx54YL33POSQQ/jd734HlMYTDBo0iC233JLtt9+e+fPn88QTTzBixAje9KY3ceGFF3LooYcCMGPGDEaMGMHHP/5xTjzxxLVjL9ZYvHgxffr0oW/fvrzwwgvceOONAOy2227MmTOH+++/H4AlS5awatUqjj76aH72s5+tTXQWLlzIbrvtxrx589YmFCtXruThhx9eW8ea8SF33nknffv2pW/fvmyxxRYsWbJkw9/cQkve4+bOOfroo7n00ktZtmzZ2tcAtDombZzafQxFRAwD9gbuBbbOzDnFobmUukRBKdl4ruyy54uy5solSVKV/O53v+OjH/0oF1xwAStXruQ973kPo0ePbtU9P/ShDzFz5kz22WcfMpPBgwfzl7/8ZZ1zHnroIc4991y6detGz549ufjii9d7z/PPP58PfvCD1NXVsdlmm3H55a/3it5///3Xjo045JBD+MIXvsCb3vQmoNSd6Te/+Q09e/Zkm2224Ytf/OI69x09ejR77703u+++O9tvvz0HH3wwAL169eKPf/wj55xzDsuXL2fTTTfllltu4UMf+hD//ve/qauro2fPnnz4wx/m7LPP5qqrruLjH/84L7/8MqtWreKTn/wkI0eOBKB3797svfferFy5cu24igkTJvDOd76Ta665hosuumidLl+tfY/XtIA0d85xxx3HlClTGDduHL169eL444/nm9/8JqeddhpnnXUWm266Kffccw+bbrrpBsWkjVNkZvtVFrE5cBvwjcz8c0Qsysx+Zcdfysz+ETER+FZm3lmU/51SV6jDgd6ZeUFR/mVgeWZe2KCeM4EzAXbYYYexzzzzTPVfnCRJUgUOP/zwdh3oPHnyZD796U9z2223tUt92nhExOTMbPRBbbdpYyOiJ/An4HeZ+eei+IWiKxPF7xeL8lnA9mWXb1eUNVe+jsy8JDPHZea4wYMHt+0LkSRJ6qQmTZrEe9/7Xj7xiU/UOhRtRNqlhaKYtelyYGFmfrKs/DvAgrJB2QMy83MRcQJwNqVZnvYHfpiZ+xWDsicD+xS3eAAYm5kLm6t73LhxOWnSpKq8LkmSJKmraK6For3GUBwMfAB4KCKmFGVfBL4FXBERZwDPACcVx26glEw8SWna2NMBMnNhRHwduL8472vrSyYkSZIkVVe7jqGoBVsoJEmSpNardQtFpzf23F/XOgRtBCZ/55RahyBJktSm2m1QtiRJkqSNjwmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqmAmFJEmSpIqZUEiSJEmqWLskFBHxq4h4MSKml5UNiIibI+KJ4nf/ojwi4ocR8WRETIuIfcquObU4/4mIOLU9YpckSZLUvPZqobgMOK5B2XnA3zNzF+DvxT7AW4Bdip8zgYuhlIAAXwH2B/YDvrImCZEkSZJUG+2SUGTm7cDCBsUnApcX25cDbysr/3WW/AvoFxFDgGOBmzNzYWa+BNxM4yRFkiRJUjuq5RiKrTNzTrE9F9i62N4WeK7svOeLsubKJUmSJNVIhxiUnZkJZFvdLyLOjIhJETFp3rx5bXVbSZIkSQ30qGHdL0TEkMycU3RperEonwVsX3bedkXZLODwBuX/bOrGmXkJcAnAuHHj2ixRkTY2z35tr1qHoE5uh/9+qNYhSJJqrJYtFNcCa2ZqOhW4pqz8lGK2pwOAl4uuUX8DjomI/sVg7GOKMkmSJEk10i4tFBHxB0qtC4Mi4nlKszV9C7giIs4AngFOKk6/ATgeeBJYBpwOkJkLI+LrwP3FeV/LzIYDvSVJkiS1o3ZJKDLzvc0cOqqJcxP4WDP3+RXwqzYMTZIkSVIrdIhB2ZIkSZI6JxMKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUsR61DkCSpLZy8EUH1zoEdXJ3nXNXrUNo5LZDD6t1COrkDrv9tqre3xYKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUsU6ZUETEcRHxeEQ8GRHn1ToeSZIkqavqdAlFRHQHfgy8BdgTeG9E7FnbqCRJkqSuqdMlFMB+wJOZOSMzXwP+DzixxjFJkiRJXVJnTCi2BZ4r23++KJMkSZLUznrUOoBqiIgzgTOL3aUR8Xgt4+lCBgHzax1ERxYXnlrrELRh/Ey/ka9ErSPQhvNzvR7xcT/TnZCf6TcSbfa53rGpws6YUMwCti/b364oWyszLwEuac+gBBExKTPH1ToOqa34mdbGyM+1NjZ+pmuvM3Z5uh/YJSKGR0Qv4D3AtTWOSZIkSeqSOl0LRWauioizgb8B3YFfZebDNQ5LkiRJ6pI6XUIBkJk3ADfUOg41YjczbWz8TGtj5OdaGxs/0zUWmVnrGCRJkiR1Up1xDIUkSZKkDsKEQpIkSVLFTCgkSZIkVcyEQpIkSVLFOuUsT6q9iFgCNDuiPzO3bMdwpFaLiE+v73hmfq+9YpHaSkTss77jmflAe8UitYWIuI71P3+8tR3DUcGEQhXJzC0AIuLrwBzgN0AAJwNDahiaVKktah2AVAXfLX73BsYBUyn9ra4DJgEH1iguqVIX1joANea0sWqViJiamaPfqEySVDsR8WfgK5n5ULE/Cjg/M99Z28gkbQxsoVBrvRIRJwP/R6kJ8r3AK7UNSapcRPQGzgBGUvpWF4DM/GDNgpJab7c1yQRAZk6PiD1qGZDUGhGxC/A/wJ6s+7d6RM2C6sIclK3Weh9wEvBC8fOuokzqrH4DbAMcC9wGbAcsqWlEUutNi4hfRMThxc/PgWm1DkpqhUuBi4FVwBHAr4Hf1jSiLswuT5JUJiIezMy9I2JaZtZFRE/gjsw8oNaxSZUqWt4+ChxaFN0OXJyZK2oXlVS5iJicmWMj4qHM3Ku8rNaxdUV2eVJFIuIi1j/LwsfbMRypLa0sfi8q+pnPBbaqYTxSqxWJw/eLH2lj8GpEdAOeiIizgVnA5jWOqcsyoVClJtU6AKlKLomI/sCXgWsp/Q/qv2sbklSZiHiI9X/5U9eO4Uht6RPAZsDHga8DRwKn1jSiLswuT2oTEbE5QGYurXUskqSSiNhxfccz85n2ikXSxsuEQq1SdAn5DTCA0tzm84BTMvPhmgYmVSgiNgHeAQyjrBU3M79Wq5ikthARWwP7Frv3ZeaLtYxHao2I2BU4F9iRdf9WH1mzoLowuzyptS4BPp2Z/wCIiMOBnwMH1TAmqTWuAV4GJgOv1jgWqU1ExEnAd4B/Uvry56KIODczr6ppYFLlrgR+SumZY3WNY+nybKFQq7iwnTY2ETE9M0fVOg6pLUXEVODoNa0SETEYuMW/1eqsnNGpY3EdCrXWjIj4ckQMK36+BMyodVBSK9wdEXvVOgipjXVr0MVpAT4DqHO7LiL+MyKGRMSANT+1DqqrsoVCrVLMhvNV4E1F0R3A+Zn5Uu2ikioXEY8AOwNPU+ryFEA6G446s4j4DlAH/KEoejcwLTM/X7uopMpFxNNNFKcrZdeGCYUklWluVhxnw1FnFxFvp+zLn8y8upbxSNp4mFCoIhHxJmBEZv662L+K0kxPABdk5q01C05qpYgYDRxS7N6RmVNrGY8kaV0R0ZN1V3//J/CzzFzZ7EWqGvtPqlJfZd3F7XajNH3b+cDnahGQ1BYi4hPA7yitjr0V8NuIOKe2UUmViYgzIuLcsv3nI2JxRCyJiLNqGZvUShcDY4GfFD9jizLVgC0UqkhE3J+Z+5bt/zkz315s35WZB9cuOqlyETENODAzXyn2+wD3OIZCnVFE3A8cl5kLiv0HM3PviOgN/C0zD6tthFJlnGWyY7GFQpXqV76zJpkobN2+oUhtKlh3TvPVRZnUGcWaZKJwJUBmrgA2rU1IUptYHRE7rdmJiBG4HkXNuLCdKvVYRJyQmdeXF0bEeODxGsUktYVLgXsj4mpKicSJwC9rG5JUsX7lO5n5TYCI6AYMqkVAUhs5F/hHRMyg9Ld6R+D02obUddnlSRWJiJ2B64G7gQeK4rGUVsgen5n/rlVsUmtFxD6sOxvOg7WMR6pURPwEWJiZX2pQfgEwKDMdR6FOKyI2oTSGE+DxzHy1lvF0ZSYUqljxH/LJwMii6GHg90VTutQpFU3oz2fmqxFxBLAX8OvMXFTbyKQNV4wB+gWwL7BmtrLRlCbV+FBmLq1VbFJrRMS7gL9m5pJiUd19KM0y+cAbXKoqMKFQqxT/s1qRmasjYldgd+BGp21TZxURU4BxwDBKrXDXAiMz8/gahiW1StG/fM2XP49k5lO1jEdqrYiYlpl1xTT2XwcuBP47M/evcWhdkoOy1Vq3A5tExLbATcAHgMtqGpHUOvWZuQp4O/CjzDwXGFLjmKRWycwZlLqnLgC2jYhDI+LQN7hM6sjWDMA+Afh5MaazVw3j6dIclK3WisxcFhFnAD/JzP8tvuGVOquVEfFe4BRgQlHWs4bxSK0WEd8G3k2pa2p9UZyUvhSSOqNZEfEz4Gjg20U3bL8orxETCrVWRMSBlMZSnFGUda9hPFJrnQ6cBXwjM5+OiOHAb2ock9RabwN2c9CqNiInAccBF2bmoogYQmnmJ9WACYVa65PAF4CrM/Phop/uP2obklSZiOgO/FdmnrymLDOfBr5du6ikNjGDUkubCYU6veJv9QOZufuassycA8ypXVRdm4OyJalMRNwJHJmZr9U6FqmtRMSfKM3u9HfKkorM/HjNgpJaISKuAc7JzGdrHYtsoVArRcQ/KPXDXUdmHlmDcKS2MAO4KyKuBV5ZU5iZ36tdSFKrXVv8SBuL/sDDEXEf6/6tfmvtQuq6TCjUWp8t2+4NvANYVaNYpLbwVPHTDdiixrFIbSIzL691DFIb+3KtA9Dr7PKkNhcR92XmfrWOQ2qNiNgsM5fVOg6pNSLiisw8KSIeounW5LoahCW1iYjYEdglM2+JiM2A7pm5pNZxdUW2UKhVImJA2W43YCzQt0bhSK1WzFr2S2BzYIeIGA18JDP/s7aRSRX5TPF7fE2jkNpYRHwYOBMYAOwEbAv8FDiqlnF1VSYUaq3JlL71CkpdnZ7m9eljpc7o/wHHUvQ3z8ypLgCmTuwaYJ/MfCYiLsrMc2odkNRGPgbsB9wLkJlPRMRWtQ2p6zKhUKtk5vBaxyC1tcx8LiLKi1Y3d67UwZV/kA+uWRRS23s1M19b87c6InrQRLc+tQ8TClUkIt6+vuOZ+ef2ikVqY89FxEFARkRP4BPAozWOSaqUD1jaWN0WEf8FbBoRRwP/CVxX45i6LAdlqyIRcel6DmdmfrDdgpHaUEQMAn4AvJnSt7s3AZ/IzAU1DUyqQEQsA56k9Fneqdim2E8HZauziohulLpYH0Pp8/w34Bfpg21NmFBIUiEi3gbsDDyUmX+rcThSqxWz4DQrM59pr1iktlCMk/gixd9q4H8yc3Fto1K3Wgegziki9o+IqRGxNCLuiYg9ah2T1BoR8RPgU8BA4OsR4Rzn6vQy85kiaZgPPFdsb0Jp1ezZNQ1OqsyvKS1kdxGl2fh+WNtwBLZQqEIRMQn4AnA78FbgQ5l5bG2jkioXEdOB0Zm5upjP/I7MHFvruKS2EBGTgUMorS58F3A/8FpmnlzTwKQNFBFTM3N02f4DmblPLWOSLRSqXLfMvDkzX83MK4HBtQ5IaqXXMnM1QLGgXbzB+VJnEsXn+u3ATzLzXcDIGsckVSQi+kfEgGItrO4N9lUDzvKkSvVrMNPTOvvO8qROaPeImFZsB7BTse/gVW0Moli08WReXyuoew3jkSrVl9IaWOVf+jxQ/E5gRLtHJLs8qTLFLE/lH57y/7Cd5UmdjoNXtTGLiMMorZp9V2Z+OyJGAJ/MzI/XODRJGwETClUkIj7ToKie0qC/OzPz6RqEJLWJiOgDLM/M+ojYFdgduDEzV9Y4NElSmYjYFtiRsh43mXl77SLqukwoVJGI+EoTxQOAY4HzM/P/2jkkqU04eFUbo4j4B00scpeZR9YgHKnVIuLbwLuBR4DVRXFm5ltrF1XXZUKhNlUMiLrFGRfUWa2ZMSQizgE2zcz/jYgpmTmm1rFJlYqI8hnLegPvAFZl5udqFJLUKhHxOFCXma/WOhY5KFttLDMXRoSz46gzc/CqNjqZOblB0V0RcV9NgpHaxgygJ2BC0QGYUKhNRcQRwEu1jkNqhU9SWmPl6sx8uBi8+o/ahiS1ToPpNLsBYynNliN1VsuAKRHxd8qSCicaqA27PKkiEfEQjfvjDqC08uopmflY+0clSWpKRDxN6W92AKuAp4GvZeadNQ1MqlBEnNpUeWZe3t6xyIRCFWpiis0EFmTmK7WIR2orDl6VJGnD2OVJFXFOfm3EPlu2vXbwao1ikVqlwQKkjbgIqTqbiLgiM09qpqcELkJaG7ZQSNIbiIj7MnO/WschbahiEdLmuAipOp2I2D4zn2tuMVK/8KwNEwpJKtPM4NUfZuZuNQpJklRYM7V3sX1RZp5T65hklydJamgyjQevnrHeK6QOKiL2By4BdgIeAj6YmY/WNiqpVcqnpj+4ZlFoHSYUklQmM4fXOgapDf2Y0rig24G3Av8POLaWAUmtZNeaDsguT5KEg1e1cSrvHtLUvtTZRMQy4ElKLRU7FdsU++mg7NqwhUKSSias51gCJhTqjPo1SJbX2TdRVie0R60DUGO2UEiStJEqZnkq/x99ef9zZ3lSpxURfYDlmVkfEbsCuwM3ZubKGofWJZlQSBIOXtXGKSI+06CoHpgP3JmZT9cgJKlNRMRk4BCgP3AXcD/wWmaeXNPAuqhutQ5AkjqINYNXBwLfozR4VersNm/wsyUwDrgxIt5Ty8CkVorMXAa8HfhJZr4LGFnjmLosx1BIUkm3zLy52L4yIr5Q02ikNpCZX22qvFhv5Rbg/9o3IqnNREQcCJzM61N7d69hPF2aCYUklTh4VV1GZi6MiHjjM6UO65PAF4CrM/PhiBgB/KO2IXVdjqGQJBy8qq4lIo4AvpyZR9Y6Fkmdny0UklQyvcG+g1fV6UXEQzReCGwAMBs4pf0jktpGRPyDJha5M0muDRMKSSrZvImyYcB/RcT5mWlfc3VG4xvsJ7AgM1+pRTBSG/ps2XZv4B3AqhrF0uXZ5UmS1mPN4FVXF5akji0i7svM/WodR1dkC4UkrYeDVyWp4ym+7FmjGzAW6FujcLo8EwpJWo9i8OpLtY5DkrSOyZS68AWlrk5P8/r0sWpnJhSShINXJakzyczhtY5Br3MMhSQBEbFjgyIHr0pSB9NgvaBGXDOoNmyhkCQgM5+pdQySpDc0YT3HEjChqAFbKCRJkiRVrFutA5AkSZJaIiL2j4ipEbE0Iu6JiD1qHZNMKCRJktR5/JjSonYDge8B/6+m0QgwoZAkSVLn0S0zb87MVzPzSmBwrQOSg7IlSZLUefRrMNPTOvvO8lQbDsqWJElSpxARl7LumkFRtp2Z+cF2DknYQiFJkqTOY3qD/XpgPnBnZj5dg3iEYygkSZLUeWze4GdLYBxwY0S8p5aBdWV2eZIkSVKnFhEDgFsyc59ax9IV2UIhSZKkTi0zF7LueAq1IxMKSZIkdWoRcQTwUq3j6KoclC1JkqROISIeYt1ZngAGALOBU9o/IoFjKCRJktRJRMSODYoSWJCZr9QiHpWYUEiSJEmqmGMoJEmSJFXMhEKSJElSxUwoJElVFxEzI+JPZfvvjIjLahiSJKmNmFBIktrL2IjYs9ZBSJLalgmFJGmDRcSwiHgsIn4XEY9GxFURcXxE/KXsnKMj4uqyy74L/FcT99ovIu6JiAcj4u6I2K0oPy0i/hIRNxctHGdHxKeL8/5VrIxLROwUEX+NiMkRcUdE7F7lly9JKmNCIUmq1G7ATzJzD2AxMBLYPSIGF8dPB35Vdv4VwD4RsXOD+zwGHJKZewP/DXyz7Ngo4O3AvsA3gGXFeffw+pzzlwDnZOZY4LPAT9ro9UmSWsCF7SRJlXouM+8qtn8LfBz4DfD+iLgUOJB1F5paDXwH+AJwY1l5X+DyiNiF0pzyPcuO/SMzlwBLIuJl4Lqi/CGgLiI2Bw4CroyINdds0kavT5LUAiYUkqRKNVzIKIFLKT30rwCuzMxVDc75DaWEYnpZ2dcpJQ7/ERHDgH+WHXu1bLu+bL+e0v/DugGLMnNMxa9CktQqdnmSJFVqh4g4sNh+H3BnZs4GZgNfopRcrCMzVwLfBz5VVtwXmFVsn7YhAWTmYuDpiHgXQJSM3pB7SJJax4RCklSpx4GPRcSjQH/g4qL8d5S6Qz3azHW/ZN0W8v8F/iciHqSylvOTgTMiYirwMHBiBfeQJFUoMhu2WEuStH5F16SJmTmqiWM/Ah7MzF+2e2CSpHbnGApJUpuJiMnAK8Bnah2LJKl92EIhSZIkqWKOoZAkSZJUMRMKSZIkSRUzoZAkSZJUMRMKSZIkSRUzoZAkSZJUMRMKSZIkSRX7/1HO42OdCctaAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "plot_df = get_accept_rate(df[df['pyDirection']=='Inbound'], 'Accepted', 'Rejected', 'pyName')\n", + "outputs": [], + "source": [ + "plot_df = get_accept_rate(\n", + " df[df[\"pyDirection\"] == \"Inbound\"], \"Accepted\", \"Rejected\", \"pyName\"\n", + ")\n", "\n", - "fig, ax = plt.subplots(2,1,figsize=(13,9), sharex=True, gridspec_kw = {'hspace':0.05})\n", - "sort = plot_df.sort_values('Accept Rate (%)', ascending=False)['pyName'].tolist()\n", - "sns.barplot(x='pyName', y='Accept Rate (%)', data=plot_df, ax=ax[0], order=sort)\n", - "sns.barplot(x='pyName', y='Accepted', data=plot_df, ax=ax[1], order=sort)\n", - "sns.pointplot(x='pyName', y='Total', data=plot_df, ax=ax[1], order=sort)\n", + "fig, ax = plt.subplots(2, 1, figsize=(13, 9), sharex=True, gridspec_kw={\"hspace\": 0.05})\n", + "sort = plot_df.sort_values(\"Accept Rate (%)\", ascending=False)[\"pyName\"].tolist()\n", + "sns.barplot(x=\"pyName\", y=\"Accept Rate (%)\", data=plot_df, ax=ax[0], order=sort)\n", + "sns.barplot(x=\"pyName\", y=\"Accepted\", data=plot_df, ax=ax[1], order=sort)\n", + "sns.pointplot(x=\"pyName\", y=\"Total\", data=plot_df, ax=ax[1], order=sort)\n", "for x in ax[1].get_xmajorticklabels():\n", " x.set_rotation(90)\n", - "ax[0].set_xlabel('')\n", - "ax[1].text(2,2000,'The bars show the accepts\\nThe line shows accept+reject')\n", - "ax[0].set_ylabel('Accept Rate (%)', fontsize=13)\n", - "ax[1].set_ylabel('Accepts', fontsize=13)\n", - "ax[0].set_title('Offers within Inbound direction')" + "ax[0].set_xlabel(\"\")\n", + "ax[1].text(2, 2000, \"The bars show the accepts\\nThe line shows accept+reject\")\n", + "ax[0].set_ylabel(\"Accept Rate (%)\", fontsize=13)\n", + "ax[1].set_ylabel(\"Accepts\", fontsize=13)\n", + "ax[0].set_title(\"Offers within Inbound direction\")\n", + "\n", + "# Not sure about this" ] }, { @@ -794,22 +269,11 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0AAAAFUCAYAAAANu1zmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAu70lEQVR4nO3debiVdb3//+cbIUnUHEJ/GspGTQQVDbamKeaQZuWATdrBNIdMO2bWySytjp7Uo1mehmMDWaZHy8xyKLO0xBzym7KRSU00QcUcEIdERQXevz/WDW42e1gb1lr3hvV8XNe69ro/973u+7X3xboW7/UZ7shMJEmSJKkZ9Cs7gCRJkiQ1igWQJEmSpKZhASRJkiSpaVgASZIkSWoaFkCSJEmSmoYFkCRJkqSm0b/sAL311re+NVtaWsqOIUmSJKmPamtreyYzB3e2b5UrgFpaWpg0aVLZMSRJkiT1URHxSFf7HAInSZIkqWlYAEmSJElqGhZAkiRJkprGKjcHSJIkSeprXn/9debMmcOCBQvKjtJUBg4cyJAhQxgwYEDVr7EAkiRJklbSnDlzWGeddWhpaSEiyo7TFDKTefPmMWfOHIYNG1b16xwCJ0mSJK2kBQsWsOGGG1r8NFBEsOGGG/a6180CSJIkSaoBi5/GW5G/uQWQJEmStBqbP38+J5xwAltuuSWjR49mzJgx/PjHPy47Vmmaeg7QmFMuLTuCmkzb+UeUHUGSJDWZY489li222IIHH3yQfv36MXfuXH76058ud9zChQvp33/1Lw/sAZIkSZJWIbNnz2abbbZh/PjxjBgxgg9/+MP8/ve/Z9y4cUuPuemmmzjkkEP4xz/+wV133cVZZ51Fv36V//oPHjyYU089FYBbbrmFsWPHctBBBzFy5EgWLFjAUUcdxfbbb8873vEOJk6cCMDPfvYzTjzxxKXnP+CAA7jlllsAWHvttfnc5z7Htttuyz777MPcuXMb84dYQRZAkiRJ0irmgQce4NOf/jT3338/6667Lvfeey9///vflxYfF198MUcffTT33nsvO+yww9LipzOTJ0/mO9/5DjNnzuTCCy8kIpg+fTq/+MUvOPLII3tcZOCll16itbWVe++9l3e/+92ceeaZNf1da80CSJIkSVrFbLbZZuy2224AHH744dxxxx18/OMf57LLLuP555/nzjvv5H3ve99yrzv77LPZcccd2XTTTZe27bzzzkuXkb799ts5/PDDAdhmm20YOnQoM2fO7DZLv379OPTQQ5dmuf3222vyO9ZLwwb5RcRs4EVgEbAwM1sjYgPgl0ALMBv4aGY+16hMkiRJ0qqo4+pnEcFRRx3FgQceyMCBA/nIRz5C//79GTlyJFOnTmXx4sX069eP008/ndNPP52111576WsHDRrU4/X69+/P4sWLl2531yvU11fDa3QP0F6ZuWNmthbbXwL+nJlvB/5cbEuSJEnqxqOPPsqdd94JwM9//nN23313Nt10UzbddFPOOussjjrqKAC22morWltb+cpXvsKiRYuASvGSmZ2ed+zYsVx++eUAzJw5k0cffZThw4fT0tLClClTWLx4MY899hh33XXX0tcsXryYq666apksfVnZQ+AOBi4pnl8CjCsviiRJkrRqGD58OBdeeCEjRozgueee44QTTgBg/PjxbLbZZowYMWLpsRdddBHz5s1bWgztu+++fOMb3+j0vJ/+9KdZvHgx22+/PYceeig/+9nPWHPNNdltt90YNmwYI0eO5KSTTmL06NFLXzNo0CDuuusutttuO26++Wa+9rWv1feXX0mNXOcugRsjIoEfZeYEYOPMfKLY/ySwcQPzSJIkSauk/v37c9llly3Xfvvtt/PJT35ymbZ1112XH/3oR52eZ88992TPPfdcuj1w4EAuvvji5Y6LiKU9Q5254IILqkxevkYWQLtn5uMRsRFwU0T8vf3OzMyiOFpORBwHHAew+eab1z+pJEmStIoZM2YMgwYN4lvf+lbZUfq0hhVAmfl48fPpiLga2Bl4KiI2ycwnImIT4OkuXjsBmADQ2tra+YBFSZIkqQm0tLQwY8aM5drb2tpKSAPz588v5borqiFzgCJiUESss+Q5sB8wA7gOOLI47Ejg2kbkkSRJktScGtUDtDFwdbEkXn/g55n5h4i4G7gyIo4BHgE+2qA8kiRJkppQQwqgzHwY2KGT9nnAPo3IIEmSJEllL4MtSZIkSQ1jASRJkiStJp588kkOO+wwttxyS8aMGcP73/9+Zs6cuULn+sQnPrH0BqfHHnss9913HwDnnHNO3a4JcMYZZ/DNb35zhV/fk0Yugy1JkiQ1hTGnXFrT87Wdf0SPx2QmhxxyCEceeSRXXHEFAFOnTuWpp55i6623BmDhwoX079/7EuCiiy5a+vycc87htNNOq/qaPWXOTPr1a1y/jD1AkiRJ0mpg4sSJDBgwgOOPP35p2w477MCiRYsYO3YsBx10ECNHjmTRokWccsop7LTTTowaNWrpTVIzkxNPPJHhw4fznve8h6effuMONXvuuSeTJk3iS1/6Eq+88go77rgj48eP7/KaY8eOZf78+eyzzz6MHj2a7bffnmuvrSz4PHv2bIYPH84RRxzBdtttx2OPPcbZZ5/N1ltvze67784DDzyw9Fzf/e53GTlyJKNGjeKwww6ryd/JHiBJkiRpNTBjxgzGjBnT6b7JkyczY8YMhg0bxoQJE3jLW97C3Xffzauvvspuu+3Gfvvtxz333MMDDzzAfffdx1NPPcXIkSM5+uijlznPueeey//+7/8yZcoUoFKgdHXNgQMHcvXVV7PuuuvyzDPPsMsuu3DQQQcB8OCDD3LJJZewyy670NbWxhVXXMGUKVNYuHAho0ePXnrOc889l1mzZrHmmmvy/PPP1+TvZAEkSZIkreZ23nlnhg0bBsCNN97ItGnTls7veeGFF3jwwQe59dZb+djHPsYaa6zBpptuyt57771S18xMTjvtNG699Vb69evH448/zlNPPQXA0KFD2WWXXQC47bbbOOSQQ1hrrbUAlhZJAKNGjWL8+PGMGzeOcePGrVSeJRwCJ0mSJK0Gtt12W9ra2jrdN2jQoKXPM5Pvfe97TJkyhSlTpjBr1iz222+/ml/z8ssvZ+7cubS1tTFlyhQ23nhjFixYsFye7lx//fX8+7//O5MnT2annXZi4cKFK5SzPQsgSZIkaTWw99578+qrrzJhwoSlbdOmTeO2225b5rj3vve9/OAHP+D1118HYObMmbz00kvsscce/PKXv2TRokU88cQTTJw4sdPrDBgwYOlru7vmCy+8wEYbbcSAAQOYOHEijzzySKfn22OPPbjmmmt45ZVXePHFF/ntb38LwOLFi3nsscfYa6+9OO+883jhhReYP3/+iv+BCg6BkyRJklYDEcHVV1/NySefzHnnncfAgQNpaWlZbujYsccey+zZsxk9ejSZyeDBg7nmmms45JBDuPnmmxk5ciSbb745u+66a6fXOe644xg1ahSjR4/m8ssv7/Sa3/72txk/fjwHHngg22+/Pa2trWyzzTadnm/06NEceuih7LDDDmy00UbstNNOACxatIjDDz+cF154gczkpJNOYr311lv5v1NmrvRJGqm1tTUnTZpUk3PVenlCqSfVLGEpSZJWPffffz8jRowoO0ZT6uxvHxFtmdna2fEOgZMkSZLUNCyAJEmSJDUNCyBJkiRJTcMCSJIkSaqBVW1u/epgRf7mFkCSJEnSSho4cCDz5s2zCGqgzGTevHkMHDiwV69zGWxJkiRpJQ0ZMoQ5c+Ywd+7csqM0lYEDBzJkyJBevcYCSJIkSVpJAwYMYNiwYWXHUBUcAidJkiSpaVgASZIkSWoaFkCSJEmSmoYFkCRJkqSmYQEkSZIkqWlYAEmSJElqGlUtgx0R6wPvADYAngWmZOaz9QwmSZIkSbXWbQEUEYcAJwO7Ay8B/wLWBdaKiDuAb2fm1fUOKUmSJEm10OUQuIj4C/AF4P+ALTJz3cwckpnrAlsAlwJfKI6TJEmSpD6vux6gczLzj53tyMxHgZ8AP4mI/eqSTJIkSZJqrMseoK6Kn06Ou7F2cSRJkiSpfqpaBGGJiOgHHEVlQYSHgB9l5iv1CCZJkiRJtdbbZbDPB/YCpgNjgctqnkiSJEmS6qSnVeAOyszr2jWNycw9i30/AebWMZskSZIk1VRPPUDHR8TPI2KDYntWRPxnROwLnAdM683FImKNiLgnIn5XbA+LiL9FxEMR8cuIeFPvfwVJkiRJqk63BVBmvh/4I/DXiDgUOAlYk8q9gQYAh/Xyep8F7m+3fR7wP5m5FfAccEwvzydJkiRJVetxDlBmXgLsCXyEypyf72XmBzLzpMx8otoLRcQQ4APARcV2AHsDVxWHXAKM6014SZIkSeqNHgugolBZmJkfpnJT1Jsj4ugVuNa3gS8Ci4vtDYHnM3NhsT0HeNsKnFeSJEmSqtJtARQRBwNPA9Mj4lHgn8C7gHdHxE0RMbSai0TEAcDTmdm2IiEj4riImBQRk+bOdd0FSZIkSSumpx6g7wL7Z+YmwIeBczPzucw8ErgA+H2V19kNOCgiZgNXUBn69h1gvYhYshLdEODxzl6cmRMyszUzWwcPHlzlJSVJkiRpWT0VQGtS6fUBeKLYBiAzbwDeWc1FMvPLmTkkM1uoLJxwc2aOByZSKawAjgSurT66JEmSJPVOTwXQmcC0iLgD+Bvw3+13Zub8lbz+qcDnI+IhKnOCfrKS55MkSZKkLnV7I9TM/EFE/BoYCjyUmc+t7AUz8xbgluL5w8DOK3tOSZIkSapGtwUQQGY+TWUhBEmSJElapXU5BC4iLomIlu5eHBEtEXFJzVNJkiRJUh101wN0J/C3iJgG3ATcB/wLWBcYCewLjAK+Wu+QkiRJklQLXRZAmfnDiLgUOBwYB3weWB94DrgHuAo4ODNfbkBOSZIkSVppPS2C8DIwoXhIkiRJ0iqtp2WwJUmSJGm1YQEkSZIkqWlYAEmSJElqGhZAkiRJkpqGBZAkSZKkplFVARQRa0TEVyLiwYh4oWh7b0QcX994kiRJklQ71fYAfR04CDgVyKJtJvCpeoSSJEmSpHqotgD6Nyo3Pf0NsLhomw201CGTJEmSJNVFtQXQWsDTHdreBCyobRxJkiRJqp9qC6DJwFEd2v4NuKu2cSRJkiSpfvpXedwXgFsi4jBgrYj4LdAK7FW3ZJIkSZJUY1UVQJk5IyJGAEcAfwceAY7NzKfqGU6SJEmSaqnHAigi+gP3ADtl5rfqH0mSJEmS6qPHOUCZuRBYjzeWv5YkSZKkVVK1iyB8Bzi76A2SJEmSpFVStQXNp6jc8+eEiHiCN+4FRGZuXYdckiRJklRz1RZAZ9U1hSRJkiQ1QLWrwF1S7yCSJEmSVG9Vz+mJiA2BnYDBQCxpz8xL65BLkiRJkmquqgIoIt4D/Bp4jcqKcM8XP2cBFkCSJEmSVgnVrgJ3LvBfmTkYmF/8/Drww7olkyRJkqQaq7YAejvw7eL5kuFv5wEn1ziPJEmSJNVNtQXQy8CaxfN5EbE58CZg/bqkkiRJkqQ6qLYA+iswrnh+A3Ad8CfgzjpkkiRJkqS6qHYVuMN5o1j6AvAfwDrABfUIJUmSJEn1UO19gF5p93wBcHbdEkmSJElSnfTmPkC7Aq1Uen6WysxzqnjtQOBWKvOI+gNXZeZ/RsQw4ApgQ6AN+HhmvlZ9fEmSJEmqXlVzgCLiDOAvwMeBfds93lPldV4F9s7MHYAdgf0jYhcqK8n9T2ZuBTwHHNOb8JIkSZLUG9X2AB0PjM3Mv63IRTIzgfnF5oDikcDewL8V7ZcAZwA/WJFrSJIkSVJPql0FLoC7V+ZCEbFGREwBngZuAv4BPJ+ZC4tD5gBvW5lrSJIkSVJ3qi2ALmIlh6dl5qLM3BEYAuwMbFPtayPiuIiYFBGT5s6duzIxJEmSJDWxLofARcRNVIapQaVQOiUiTgKeaH9cZu7Xmwtm5vMRMRHYFVgvIvoXvUBDgMe7eM0EYAJAa2trdnaMJEmSJPWkuzlAt3fYvm1FLxIRg4HXi+LnzVQWUDgPmAh8mMpKcEcC167oNSRJkiSpJ10WQJl5Zg2vswlwSUSsQaU36crM/F1E3AdcERFnAfcAP6nhNSVJkiRpGd2uAhcR/YHIzNfbtX2CylLWt2bmb6q5SGZOA97RSfvDVOYDSZIkSVLd9bQIwi+Bo5ZsRMRXqMzF2R24PCKOrWM2SZIkSaqpngqgVuB37bY/Axybma3A4cAJ9QomSZIkSbXWUwG0fmb+EyAiRgBvAa4s9l0DtNQtmSRJkiTVWE8F0EsRsXbxvBWYkZkLiu2ghzlEkiRJktSX9FQA3QZ8PSK2AT4F/KHdvuF0uCeQJEmSJPVlPfXgnAr8HvgsMAO4oN2+8Sx/ryBJq6gxp1xadgQ1kbbzjyg7giSpSXVbAGXmLGBERGyQmc922P0N4LW6JZMkSZKkGqtqDk8nxQ+Z+XzN00iSJElSHfU0B0iSJEmSVhsWQJIkSZKahgWQJEmSpKZhASRJkiSpaXS5CEJETASypxNk5t41TSRJkiRJddLdKnB/avf8rcAngWuAWUALMA74cZ1ySZIkSVLNdVkAZebZS55HxHXAIZl5U7u291C5QaokSZIkrRKqnQO0J/DnDm0TgXfXNI0kSZIk1VG1BdBjwKEd2j4MzKltHEmSJEmqn+7mALX3ReDXEXE8MJvKHKB3UimCJEmSJGmVUFUPUGZeD2wL/BF4CbgR2DYzf1fHbJIkSZJUU9X2AJGZ/wDOqWMWSZIkSaqrqm+EGhGHR8SNETGt2N4jIj5Yv2iSJEmSVFtVFUAR8XngTOAPwOZF81wqc4MkSZIkaZVQbQ/QCcD7MvMCIIu2mcBWdUklSZIkSXVQbQG0QWbOLJ4vKYCi3XNJkiRJ6vOqLYDui4gDOrTtD0ytcR5JkiRJqptqV4E7Dbg+Iq4E1oyI7wGHAR2LIkmSJEnqs6q9D9BtwC7AK8DE4nV7Zubf6phNkiRJkmqqqh6giGjJzPuAz3RoH5qZj9QlmSRJkiTVWLVzgKZ10X5PrYJIkiRJUr1VWwDFcg0RA3AVOEmSJEmrkG6HwEXETVSKnDUj4sYOuzcHJtcrmCRJkiTVWk9zgG4vfr4buKNd+2LgSeBX1VwkIjYDLgU2plJQTcjM70TEBsAvgRZgNvDRzHyu2vCSJEmS1BvdFkCZeSZARNyfmVeuxHUWAv+RmZMjYh2grehd+gTw58w8NyK+BHwJOHUlriNJkiRJXapqFbjMvDIi1qZy358hwBzg+sx8scrXPwE8UTx/MSLuB94GHAzsWRx2CXALFkCSJEmS6qTaZbBbgd9TuQ/Qo1Tm/3w3It6fmZN6c8GIaAHeAfwN2LgojqAypG7jLl5zHHAcwOabb96by0mSJEnSUtWuAvd94FuZOTQzx2bmUOCbwA96c7GiF+nXwMmZ+a/2+zIz6WJVucyckJmtmdk6ePDg3lxSkiRJkpaqtgAaAXyrQ9sFwDbVXqhYNvvXwOWZ+Zui+amI2KTYvwnwdLXnkyRJkqTeqrYAmgJs16Ft+6K9RxERwE+A+zPzgna7rgOOLJ4fCVxbZR5JkiRJ6rWq5gABNwK/i4iLgEeoLFt9NDAhIv5tyUGZ+fMuXr8b8HFgekRMKdpOA84FroyIY4rzfrS3v4AkSZIkVavaAuho4HXe6K2BytLWR7fbTqDTAigzbweii3PvU2UGSZIkSVop1S6DPazeQSRJkiSp3qqdAwRARGwaEbvUK4wkSZIk1VNVBVBEbBQRf6JyA9Q/FW2HRsT36xlOkiRJkmqp2h6g7wKzgMFU5gIB3AzsW49QkiRJklQP1S6CsBcwNDMXREQCZObciNioftEkSZIkqbaq7QF6lQ7FUkRsADxb80SSJEmSVCfVFkA3At+KiAHt2s4Erq99JEmSJEmqj2qHwH0RuAZ4DhgYEc8DU4GD6xNLkiRJkmqv2vsAPQvsERFjgGHAI8CkzMx6hpMkSZKkWqqqAIqI9YHXMrMNaCvaBkXEgMx8vo75JEmSJKlmqp0DdB2wbYe27YBraxtHkiRJkuqn2gJoW2BSh7ZJwPa1jSNJkiRJ9VNtAbQAWKtD2yDeuCmqJEmSJPV51RZAtwPnREQ/gIgI4L+AO+oVTJIkSZJqrdplsE8BbgY+FBEPU1kJ7jVg73oFkyRJkqRaq3YZ7EciYjvgAKAFmA1cn5kv1y+aJEmSJNVWtT1AZOYrwK/qmEWSJEmS6qqqOUARcWNE7N2hbZ+IuKE+sSRJkiSp9qpdBGE0cGuHttuAnWobR5IkSZLqp9oCaDEwoEPbGkDUNo4kSZIk1U+1BVAb8JkObScCk2sbR5IkSZLqp9pFEE4FbomIDwEzgbcDw4E965RLkiRJkmquqh6gzJwGjASuAv4F/BoYmZlT65hNkiRJkmqqN8tgPwmcv2Q7IraNiC9n5kl1SSZJkiRJNVbtHCAAImLNiDgiIu4AplNZHU6SJEmSVglV9QBFxEjgU8DhwFpUCqf9M/PGOmaTJEmSpJrqtgcoIj4eEbcBM4B3A2cAbwOeBabUO5wkSZIk1VJPPUCXAPOAD2TmDUsaI7z9jyRJkqRVT09zgL4KzAeuiYirI+LAiOjVvCFJkiRJ6iu6LWYy82xgC2AckFSWv34cWA/YtM7ZJEmSJKmmeuzNyYobMvODwFDg+8CTwN0RcWW9A0qSJElSrfRqOFtmPpGZX6fSK3Qw8KZqXhcRP42IpyNiRru2DSLipoh4sPi5fq+SS5IkSVIvrdB8nqJX6PeZOa7Kl/wM2L9D25eAP2fm24E/F9uSJEmSVDdV3QdoZWXmrRHR0qH5YGDP4vklwC3AqY3II0lSV8accmnZEdRE2s4/ouwIXfK9oEZq5HuhzBXdNs7MJ4rnTwIbd3VgRBwXEZMiYtLcuXMbk06SJEnSaqdPLGmdmUlllbmu9k/IzNbMbB08eHADk0mSJElanZRZAD0VEZsAFD+fLjGLJEmSpCZQZgF0HXBk8fxI4NoSs0iSJElqAg0pgCLiF8CdwPCImBMRxwDnAvtGxIPAe4ptSZIkSaqbRq0C97Eudu3TiOtLkiRJEvSRRRAkSZIkqREsgCRJkiQ1DQsgSZIkSU3DAkiSJElS07AAkiRJktQ0LIAkSZIkNQ0LIEmSJElNwwJIkiRJUtOwAJIkSZLUNCyAJEmSJDUNCyBJkiRJTcMCSJIkSVLTsACSJEmS1DQsgCRJkiQ1DQsgSZIkSU3DAkiSJElS07AAkiRJktQ0LIAkSZIkNQ0LIEmSJElNwwJIkiRJUtOwAJIkSZLUNCyAJEmSJDUNCyBJkiRJTcMCSJIkSVLTsACSJEmS1DQsgCRJkiQ1DQsgSZIkSU3DAkiSJElS07AAkiRJktQ0LIAkSZIkNQ0LIEmSJElNo/QCKCL2j4gHIuKhiPhS2XkkSZIkrb5KLYAiYg3gQuB9wEjgYxExssxMkiRJklZfZfcA7Qw8lJkPZ+ZrwBXAwSVnkiRJkrSaisws7+IRHwb2z8xji+2PA+/MzBM7HHcccFyxORx4oKFB1dFbgWfKDiH1Eb4fpArfC1KF74W+YWhmDu5sR/9GJ1kRmTkBmFB2DlVExKTMbC07h9QX+H6QKnwvSBW+F/q+sofAPQ5s1m57SNEmSZIkSTVXdgF0N/D2iBgWEW8CDgOuKzmTJEmSpNVUqUPgMnNhRJwI/BFYA/hpZt5bZiZVxeGI0ht8P0gVvhekCt8LfVypiyBIkiRJUiOVPQROkiRJkhrGAkiSJElS07AAkiRJktQ0LIAkSZIkNY1V4kaoktQXRMTnu9ufmRc0KotUpogY3d3+zJzcqCxSmSLit0CXK4pl5kENjKMqWQCpWxHxIt2/sddtYBypbOuUHUDqI75V/BwItAJTgQBGAZOAXUvKJTXaN8sOoN6zAFK3MnMdgIj4OvAE8H9UPuTGA5uUGE1quMw8s+wMUl+QmXsBRMRvgNGZOb3Y3g44o8RoUkNl5l/KzqDe8z5AqkpETM3MHXpqk5pBRAwEjgG2pfINOACZeXRpoaQSRMS9mbltT23S6i4i3g78NzCSZT8XtigtlLrkIgiq1ksRMT4i1oiIfhExHnip7FBSSf4P+P+A9wJ/AYYAL5aaSCrHtIi4KCL2LB4/BqaVHUoqwcXAD4CFwF7ApcBlpSZSl+wBUlUiogX4DrAblTlBdwAnZ+bsEmNJpYiIezLzHRExLTNHRcQA4LbM3KXsbFIjFb2hJwB7FE23Aj/IzAXlpZIaLyLaMnNMREzPzO3bt5WdTctzDpCqUhQ6B5edQ+ojXi9+Pl/MeXgS2KjEPFIpikLnf4qH1MxejYh+wIMRcSLwOLB2yZnUBQsgdSsivkf3q8Cd1MA4Ul8xISLWB74KXEflQ+5r5UaSGiciptP9Z8OoBsaR+oLPAmsBJwFfB/YGjiw1kbrkEDh1KyK6ffNm5iWNyiJJ6hsiYmh3+zPzkUZlkaTesgBSr0TE2gCZOb/sLFJZImJN4ENAC+160jPzv8rKJJUlIjYGdio278rMp8vMI5UhIrYGTgGGsuznwt6lhVKXHAKnqhTzHP4P2KCyGXOBIzLz3nKTSaW4FngBaANeLTmLVJqI+ChwPnALlXvEfS8iTsnMq0oNJjXer4AfAj8GFpWcRT2wB0hViYi/Aqdn5sRie0/gnMx8V5m5pDJExIzM3K7sHFLZImIqsO+SXp+IGAz8yXvEqdm44tuqxfsAqVqDlhQ/AJl5CzCovDhSqf4aEduXHULqA/p1GPI2D/9voeb024j4dERsEhEbLHmUHUqdswdIVYmIq4HJVIbBARwOjMnMQ8pLJZUjIu4DtgJmURkCF0C68pWaTUScD4wCflE0HQpMy8xTy0slNV5EzOqkOTNzi4aHUY8sgFSVYsnfM4Hdi6bbgDMy87nyUknl6GoFLFe+UjOKiA/S7rMhM68uM48k9cQCSJJWQETsAIwtNm/LzKll5pEklSciBgAnAHsUTbcAP8rM17t8kUrjOF11KyJ2j4gj2m1fFRE3Fw+XdlRTiojPApcDGxWPyyLiM+WmkhonIo6JiFPabc+JiH9FxIsRcXyZ2aSS/AAYA3y/eIwp2tQH2QOkbkXEn4HPZOZ9xfZ04BNUFkA4LTP3LzGeVIqImAbsmpkvFduDgDudA6RmERF3A/tn5rxi+57MfEdEDAT+mJnvLjeh1FgRMbXj6oedtalvsAdIPVl3SfFTeDAz2zLzVmCdskJJJQuWvc/DoqJNahaxpPgp/AogMxcAby4nklSqRRGx5ZKNiNgC7wfUZ3kjVPVkvfYbmfnBdpsbNzaK1GdcDPytWB0xgIOBn5QbSWqo9dpvZOY5ABHRD3hrGYGkkp0CTIyIh6l8LgwFjio3krriEDh1KyJ+C/wwM6/v0H4AcEJmfqCcZFK5ImI0y658dU+ZeaRGiojvA89m5lc6tJ8FvDUznQekphMRawLDi80HMvPVMvOoaxZA6lZEbAVcD/yVyn2AoDKx713AAZk5s6xsUlmKYQ5zMvPViNgL2B64NDOfLzeZ1BjFvLeLgJ2AJSsg7gBMAo7NzPllZZPKEBEfAf6QmS9GxFeA0cBZmTm5h5eqBBZA6lHxjcZ4YNui6V7g58VYb6npRMQUoBVoofIFwXXAtpn5/hJjSQ1XzHNY8tlwX2b+o8w8UlkiYlpmjoqI3YGvA98EvpaZ7yw5mjphAaSqFN/2LcjMRRGxNbANcIPr26sZRcTkzBwdEV8EXsnM7y1ZBavsbFKjRcTbqMx3WDqvuFgoR2oa7VZC/G9gemb+3M+FvstFEFStW4GxEbE+cCNwN3AolZ4hqdm8HhEfA44ADizaBpSYRypFRJxH5bPgXmBx0ZxUPjOkZvJ4RPwI2Bc4rxg942rLfZQ9QKpKu2+8PwO8OTO/ERFTMnPHsrNJjRYRI4Hjqdz75xcRMQz4aGaeV3I0qaEi4gFglJO91ewiYi1gfyq9Pw9GxCbA9pl5Y8nR1Al7gFStiIhdqfT4HFO0rVFiHqkUEbEGcHpmLu39zMxZgMWPmtHDVHo/LYDUtIrPhcmZuc2Stsx8AniivFTqjgWQqnUy8GXg6sy8t5j4OrHcSFLjFfPghkbEmzLztbLzSCV7GZgSEX+mXRGUmSeVF0lqrOJz4YGI2DwzHy07j3rmEDhJ6qWIuBQYQWX1t5eWtGfmBaWFkkoQEUd21p6ZlzQ6i1SmiLgVeAdwF8t+LhxUWih1yR4gVSUiJlKZ2LqMzNy7hDhS2f5RPPoB65ScRSqNhY601FfLDqDq2QOkqkTEmHabA4EPAQsz84slRZJKFxFrZebLZeeQGi0irszMj0bEdDr/cmxUCbGkUkXEUODtmfmnYlGENTLzxbJzaXkWQFphEXFXZu5cdg6p0YoFQX4CrJ2Zm0fEDsCnMvPTJUeTGiIiNsvMx4r/8C0nMx9pdCapTBHxSeA4YIPM3DIi3g78MDP3KTmaOuH65KpKRGzQ7vHWiHgv8Jayc0kl+TbwXmAeQGZOBfYoM5DUYNfC0kLnC5n5SPtHydmkMvw7sBvwL4DMfBDYqNRE6pJzgFStNirDHAJYCMzijeWwpaZTfPvdvmlRWVmkErT/x79baSmkvuPVzHxtyedCRPSnk+Gh6hssgFSVzBxWdgapD3ksIt4FZEQMAD4L3F9yJqmR/I+dtKy/RMTpwJsjYl/g08BvS86kLjgHSN2KiA92tz8zf9OoLFJfERFvBb4DvIfKN+E3Ap/NzHmlBpMaJCJeBh6i8u9/y+I5xXa6CIKaTUT0ozIyZj8q74M/Ahel/9Huk+wBUk8O7GZfAhZAaioRMQ7YCrg0M8eXHEcqy4iyA0h9QURsBJxG5XNhOnBMZv6r3FTqiT1AklSliPg+sC3wV2Af4LeZ+fVyU0nliYhBwCuZuTgitga2AW7IzNdLjiY1RET8gco86VuBA4B1MvMTpYZSjyyA1K2IeCcwgcoQh+nA0ZnpXAc1pYiYAeyQmYuKezzclpljenqdtLqKiDZgLLA+cAdwN/CavaNqFhExNTN3aLc9OTNHl5lJPXMZbPXkQuALwIbABVSW/5Wa1WuZuQiguAFq9HC8tLqL4r3wQeD7mfkRKr2kUtOIiPWX3CoEWKPDtvog5wCpJ/0y86bi+a8i4sulppHKtU1ETCueB7Blse3EbzWrKG4MPJ43bo2wRol5pEZ7C5UhcO2/EJtc/Exgi4YnUo8sgNST9TqsBLfMtqvAqck48Vta1snAl4GrM/PeiNgCmFhuJKlxMrOl7AzqPecAqVsRcTHL3u+h/TccmZlHNziSVDonfkuSOoqItwFDadfBkJm3lpdIXbEAUrci4j86NC0GngFuz8xZJUSSSufEb6kiIibSyU1RM3PvEuJIpYmI84BDgfuARUVzZuZB5aVSVxwCp56s3UlbC3B6RJyRmVc0OI/UF0RmvhwRx1CZ+P2NiJhSdiipBF9o93wg8CFgYUlZpDKNA4Zn5qtlB1HPLIDUrcw8s7P2YmWTPwEWQGpGTvyWgMxs69B0R0TcVUoYqVwPAwMAC6BVgAWQVkhmPhsRLgGsZnUyTvyW6LDMbz9gDJVVsaRm8zIwJSL+TLsiKDNPKi+SumIBpBUSEXsBz5WdQypDZv4F+Eu77YcBP+TUjNqozAEKKkPfZvFGr6jUTK4rHloFuAiCuhUR01l+gusGwD+BIzLz741PJZXLid+SJK267AFSTw7osJ3AvMx8qYwwUh/hxG81tQ73h1uO94hTs4iIKzPzo118YYw3yO6b7AGSpBqIiLsyc+eyc0iNUNwjriveI05NIyI2y8zHImJoZ/sz85FGZ1LPLIAkqZe6mPj93cwcXlIkSVIJImJyZo4unn8vMz9Tdib1zCFwktR7TvxWU4uIdwITgC2B6cDRmXl/uamkUrRfEXe30lKoVyyAJKmXMnNY2Rmkkl1IZS7crcBBwLeB95YZSCqJQ6lWQQ6Bk6QqOfFbqmg/7KezbalZRMTLwENUeoK2LJ5TbKeLIPRN9gBJUvUO7GZfAhZAahbrdfhCYJltvwxQExlRdgD1nj1AkiSpV4pV4Nr/B6L9PAhXgVPTiYhBwCuZuTgitga2AW7IzNdLjqZOWABJUpWc+C1VRMR/dGhaDDwD3J6Zs0qIJJUqItqAscD6wB3A3cBrmTm+1GDqVL+yA0jSKmTJxO8NgQuoTPyWmtHaHR7rAq3ADRFxWJnBpJJEZr4MfBD4fmZ+BNi25EzqgnOAJKl6/TLzpuL5ryLiy6WmkUqSmWd21l7cI+tPwBWNTSSVLiJiV2A8b9wWYY0S86gbFkCSVD0nfkvdyMxnIyJ6PlJa7ZwMfBm4OjPvjYgtgInlRlJXnAMkSVVy4rfUvYjYC/hqZu5ddhZJ6oo9QJJUvRkdtp34raYUEdNZ/gaQGwD/BI5ofCKpXBExkU5uiuqXAX2TBZAkVW/tTtpagNMj4ozMdN6DmsUBHbYTmJeZL5URRuoDvtDu+UDgQ8DCkrKoBw6Bk6SVtGTid2aOLjuLJKlviIi7MnPnsnNoefYASdJKcuK3JDW34ouwJfoBY4C3lBRHPbAAkqSVVEz8fq7sHJKk0rRRGQoaVIa+zeKN5bDVx1gASVKVnPgtSepMZg4rO4Oq5xwgSapSRAzt0OTEb0lqYh3uDbcc7w/XN9kDJElVysxHys4gSepTDuxmXwIWQH2QPUCSJEmSmka/sgNIkiRJq6KIeGdETI2I+RFxZ0SMKDuTemYBJEmSJK2YC6ncBHVD4ALg26WmUVUsgCRJkqQV0y8zb8rMVzPzV8DgsgOpZy6CIEmSJK2Y9TqsBLfMtqvA9U0ugiBJkiStgIi4mGXvDxftnmdmHt3gSKqCPUCSJEnSipnRYXsx8Axwe2bOKiGPquAcIEmSJGnFrN3hsS7QCtwQEYeVGUxdcwicJEmSVEMRsQHwp8wcXXYWLc8eIEmSJKmGMvNZlp0PpD7EAkiSJEmqoYjYC3iu7BzqnIsgSJIkSSsgIqaz7CpwABsA/wSOaHwiVcM5QJIkSdIKiIihHZoSmJeZL5WRR9WxAJIkSZLUNJwDJEmSJKlpWABJkiRJahoWQJIkSZKahgWQJKnPiYhbIiIjYo8O7Q9FxCdKiiVJWg1YAEmS+qp5wDcjwpsJSpJqxgJIklQXETE7Ir4WEbdHxPyImBQRO0XEiIh4LSI2andsRMSsiPh4u1P8GBgCfKyL868VEb+JiCcj4l8RMTki9m23/xNFj9HnImJORLwYEd+MiA0j4tfFa/4eEbt3OO8nI2JGRLwQEfdExH41/tNIkkpkASRJqqfjgc9SuTHgVcDvgceB/wcc2e64fYH1imOWeAn4GnBORKzZybn7Ab8B3g5sCPwC+HVEDG53zNDivFsAuwOfAW4AzgfWL15/8ZKDI+KTwKnA+GL/6cBvImKr3v7ikqS+yQJIklRPP8nMtsx8DTgPeAU4AJgAHN3uuGOAyzLzlQ6vvxiYT6WIWkZmzs/MyzLzxcx8PTPPB14Ddmp32CvAmZn5WmZOBaYCd2fm/8vMRcBlwFYR8Zbi+M8C/5WZUzNzcWb+HpgIHLZyfwZJUl9hASRJqqfZS55k5c7bj1IZ1nYVsFFE7B4RGwLjqAx5W0ZRpJwCnFYct1REvDki/jciHi6Gsz1PpdemfQ/Q05m5uN32y8ATHbYB1il+DgMujIjnlzyAvYC39eq3liT1Wf3LDiBJWq21LHlSLGawOTAnMxdExCVUen6mAlMyc1pnJ8jMGyLibirD4dr7PLAHsA8wOzMzIp4BVmbRhEeA/8zMX63EOSRJfZgFkCSpno6OiKuB6cDngLWA64t9E4BJwLuozMnpzheAvwGvtmtbt9ieB7wpIk6lMt9nZfwPcEZEPEilMBsIjAGeycy/r+S5JUl9gEPgJEn1NAH4LvAccCjwgcx8AaAoKNqATYErujtJMX/nF1SKniUuAJ4H/gn8g8pwttkrEzYzfwx8g8rco+eoDNn7KjBgZc4rSeo7ojIkW5Kk2oqI2cBXMvOybo75GfBaZh7XqFySpObmEDhJUikiYmvgI8A7y84iSWoeDoGTJDVcRFxFZfjbf2fmjLLzSJKah0PgJEmSJDUNe4AkSZIkNQ0LIEmSJElNwwJIkiRJUtOwAJIkSZLUNCyAJEmSJDUNCyBJkiRJTeP/Bw6yxqh18d85AAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plot_outcome_share_graph(df[df['pyChannel']=='Web'], 'Accepted', 'pyName', 'pyGroup')" + "plot_outcome_share_graph(df[df[\"pyChannel\"] == \"Web\"], \"Accepted\", \"pyName\", \"pyGroup\")\n", + "\n", + "# Wouldn't the tree show this?" ] }, { @@ -825,8 +289,12 @@ "metadata": {}, "outputs": [], "source": [ - "click_share_name_daily = get_outcome_share_time(df[df['pyChannel']=='Web'], 'Clicked', 'pyName', time='daily')\n", - "click_share_name_weekly = get_outcome_share_time(df[df['pyChannel']=='Web'], 'Clicked', 'pyName', time='weekly')" + "click_share_name_daily = get_outcome_share_time(\n", + " df[df[\"pyChannel\"] == \"Web\"], \"Clicked\", \"pyName\", time=\"daily\"\n", + ")\n", + "click_share_name_weekly = get_outcome_share_time(\n", + " df[df[\"pyChannel\"] == \"Web\"], \"Clicked\", \"pyName\", time=\"weekly\"\n", + ")" ] }, { @@ -840,23 +308,14 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAFwCAYAAACGgdwmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABUDklEQVR4nO3dd3hUZfrG8e9DQu+E0GtIQLHQIh2l2NdVsaNiFwtFd92++9vedFfXig0b9oZlV9dGU0BKQKRLCb1DINSElOf3RwYXMZAEMnNmJvfnunJl5pw5c26MJ3ny5j3Pa+6OiIiIiIiUr0pBBxARERERiUcqtEVEREREwkCFtoiIiIhIGKjQFhEREREJAxXaIiIiIiJhoEJbRERERCQMEiNxEjNrCYwFGgMOPOXuD5lZJ+AJoBawCrjG3XcVc/y5wENAAjDG3f9e0jkbNmzobdq0Kbd/g4iIiIjI4WbPnr3N3ZOL22eR6KNtZk2Bpu4+x8xqA7OBi4EXgJ+4+2Qzuwlo6+7/d9ixCcBS4CxgHTALGOLui452zvT0dM/IyCj/f4yIiIiISIiZzXb39OL2RWTqiLtvdPc5oce7gcVAc6A98HnoZZ8ClxZzeHdgubtnuvsB4DXgovCnFhERERE5dhGfo21mbYAuwAxgIf8rmi8HWhZzSHNg7SHP14W2iYiIiIhErYgW2mZWC3gbuDs0F/sm4E4zmw3UBg4c5/sPM7MMM8vYunXr8QcWERERETlGESu0zawyRUX2y+4+DsDdl7j72e7eDXgVWFHMoev57kh3i9C273H3p9w93d3Tk5OLnZMuIiIiIhIRESm0zcyAZ4DF7v7AIdsbhT5XAn5DUQeSw80C0sysrZlVAa4C3g9/ahERERGRYxepEe0+wFBgoJnNDX2cDwwxs6XAEmAD8ByAmTUzsw8B3D0fGAF8TNFNlG+4+8II5RYREREROSYRae8XBLX3ExEREZFwC7y9n4iIiIhIRaNCW0REREQkDFRoi4hIhZSTV8DarH1BxxCROKZCW0REKpxN2Tlc/NhUBj0wmXU7VGyLSHio0BYRkQplyaZdDB49lbVZ+3B3nphc3BIOIiLHT4W2iIhUGFOWbePyx7+k0J03bu/F5ekteWPWOjZm7w86mojEIRXaIiJSIbyZsZYbnptJ8/rVeefOPpzUrC53nNGOQneenJwZdDwRiUMqtEVEJK65Ow9+tpSfvjWPHikNeOP2XjSrVx2Alg1qcGnXFrwycw1bduUEnFRE4o0KbRERiVt5BYX89K15PPjZMi7t2oLnbuhOnWqVv/OaOwe0o6DQeUKj2iJSzlRoi4hIXNqdk8dNz8/irdnruGtQGv+8/FSqJH7/x17rpJpc3Lk5L89YzZbdGtUWkfKjQltEROLOxuz9XP7El3y5Yjv3XXYqPzqrPWZ2xNePGJhKXkEhY75YGcGUIhLvVGiLiEhcWbRhF4Mfm8a6Hft57sbTuCK9ZYnHtG1Yk4s6N+fFL1ezfU9uBFKKSEWgQltEROLG50u3csWTXwLw5u296JeWXOpjhw9IJSe/gDFTNKotIuVDhbaIiMSFNzLWctPzs2hRvzrvDO/NiU3rlOn41Ea1uODUZoydtoodew+EKaWIVCQqtEVEJKa5O//6dCk/e2sePVOSeOP2XjStW/2Y3mvkwFT2Hijg2aka1RaR46dCW0REYtaB/EJ+8uY8Hhq/jMu6teC5G0/7Xvu+smjfuDbnn9KE56euIntfXjkmFZGKSIW2iIjEpF05edz4/EzenrOOH53Znn9cdiqVE47/x9rIgWnszs3XqLaIHDcV2iIiEnM27NzP5Y9/yYzMLP55eSfuOjPtqO37yuLEpnU456TGPDt1JbtyNKotIsdOhbaIiMSURRt2MXj0VNbv3M/zN3bnsm4tyv0cIwemsTsnnxemrir39xaRikOFtoiIxIyD7fsM483be9E3rWFYznNy87qceWIjxkxZyZ7c/LCcQ0TinwptERGJCW/MWsuNx9G+r6xGDkwje38eY79cFdbziEj8UqEtIiJRzd154JNv+Nnb8+jdLok3j6N9X1l0almP/h2SGfPFSvZqVFtEjoEKbRERiVoH8gu5542veXjCci7v1oJnbziN2sfRvq+sRg5MI2vvAV6esTpi5xSR+BGRQtvMWprZRDNbZGYLzeyu0PbOZjbdzOaaWYaZdT/C8QWh18w1s/cjkVlERIK1KyePG56bybiv1vPjs9pzXzm17yuLbq3r0y+tIU99nsn+AwURPbeIxL5IfcfKB+5x945AT2C4mXUE7gP+4O6dgd+Gnhdnv7t3Dn1cGJHEIiISmIPt+2auLGrfN2pQ+bXvK6tRg9LYtkej2iJSdhEptN19o7vPCT3eDSwGmgMOHLybpS6wIRJ5REQkei3ckM3g0VPZsHM/L9wUnvZ9ZXFamwb0bpfEk59nkpOnUW0RKb2Iz9E2szZAF2AGcDfwDzNbC/wT+OURDqsWmloy3cwujkROERGJvEnfbOGKJ76kkhlv3tGLPqnhad9XVqMGpbF1dy6vzVwTdBQRiSERLbTNrBbwNnC3u+8C7gB+5O4tgR8Bzxzh0Nbung5cDTxoZu2O8P7DQgV5xtatW8PwLxARkXB5fdYabn4hg1ZJNXnnzj6c0CS87fvKomdKEt3bNuDxySs0qi0ipRaxQtvMKlNUZL/s7uNCm68HDj5+Eyj2Zkh3Xx/6nAlMomhEvLjXPeXu6e6enpycXI7pRUQkXNyd+z/5hp+/PZ8+qQ158/ZeNKlbLehY33PXoDQ278rlzdnrgo4iIjEiUl1HjKLR6sXu/sAhuzYAZ4QeDwSWFXNsfTOrGnrcEOgDLApvYhERiYSD7fsembCcK9Nb8sz16dSqmhh0rGL1bpdEt9b1eXzicg7kFwYdR0RiQKRGtPsAQ4GBh7TpOx+4FbjfzL4G/goMAzCzdDMbEzr2RCAj9JqJwN/dXYW2iEiMy96fx/XPFrXvu+es9vz90lMi3r6vLMyMUYPS2JCdw1sa1RaRUjB3DzpDWKSnp3tGRkbQMUREpBjrd+7nxudmsnLbXu699FQu6RpsZ5HScncuHj2NbbtzmfTT/lH9i4GIRIaZzQ7dS/g9+g4hIiIRtWB9NoMfm8rG7BxeuLF7zBTZUDSqffegNNbv3M87c9YHHUdEopwKbRERiZiJ32zhyie/JLGS8dbtvekdJe37yqJ/h2ROaV6XRycuJ79Ac7VF5MhUaIuISES8OnMNt7yQQeukmrwzvA8dmtQOOtIxOThXe03WPt6bq3XWROTIVGiLiEhYuTv/+HgJvxw3n76pDXnj9l40rhN97fvK4swTG3Fi0zo8OnE5BYXxea+TiBw/FdoiIhI2ufkF/Oj1uTw2cQVXndaSMVHcvq8szIy7BqWyctte/jNPo9oiUjwV2iIiEhYH2/e9O3cDPz2nA3+7JLrb95XV2R2b0KFxbR6ZoFFtESle/HzHExGRqLFuxz4ue3was1fv4F9XdmL4gFSK1i6LH5UqGSMHpbJ8yx4+nL8x6DgiEoVUaIuISLlasD6bwaOnsWlXDi/c1J3BXWKnfV9ZnXdyU1Ib1eKRCcso1Ki2iBxGhbaIiJSbiUu2cMWTX1IloRJv39Gb3u1ir31fWSRUMkYOTGXp5j18vHBT0HFEJMqo0BYRkXLxyow13DI2g7YNa/LOnb1p3zg22/eV1QWnNiOlYU0eGq9RbRH5LhXaIiJyXAoLnfs+WsKv3plPv7SGvHFbLxrFePu+skioZIwYmMqSTbv5bPHmoOOISBRRoS0iIscsN7+AH70xl9GTVjCke0vGXJdOzTho31dWF3ZqRuukGjw8YRnuGtUWkSIqtEVE5Jhk78vjumdm8l6ofd9fB59CYhy17yuLxIRKDB+QyoL1u5j4zZag44hIlKiY3xFFROS4rM3ax6VPTGPOmh08dFXnuGzfV1aDuzSnRf3qPPSZRrVFpIgKbRERKZP567K55PFpbNmVw9ibenBR5+ZBR4oKlUOj2l+vy2by0q1BxxGRKKBCW0RESm3iki1c+dT/2vf1apcUdKSocmnXFjSvV52HxmtUW0RUaIuISCm9PGM1N78wi5TkovZ9aRWkfV9ZVEmsxB392/HVmp1MXb496DgiEjAV2iIiclSFhc69Hy3h1+8s4Iz2ybw+rGK17yury9Nb0KRONR4av1Sj2iIVnAptERE5otz8Au56fS6PT1rB1T1a8XQFbd9XFlUTE7ijfztmrdrB9MysoOOISIBUaIuISLGy9+Ux9JmZ/PvrDfz83BP4y8UnV9j2fWV15WktaVS7Kg+PXxZ0FBEJkL5jiojI9xxs3zd3zU4euqozd/RvV+Hb95VFtcoJ3HZGO77M3M7MlRrVFqmoVGiLiMh3zFu3k8GjQ+37bu6u9n3H6OrurWhYq4pGtUUqMBXaIiLyrfGLN3Plk9OpmliJcXf2pmeK2vcdq+pVEhh2egpTlm9j9mqNaotURBEptM2spZlNNLNFZrbQzO4Kbe9sZtPNbK6ZZZhZ9yMcf72ZLQt9XB+JzCIiFc2L01dz69gMUhvV4p3hvUltpPZ9x+vanq1pULMKD49fHnQUEQlApEa084F73L0j0BMYbmYdgfuAP7h7Z+C3oeffYWYNgN8BPYDuwO/MrH6EcouIxL3CQudv/13M/727gAEdGvHasJ40qq32feWhRpVEbu2XwuSlW5m7dmfQcUQkwiJSaLv7RnefE3q8G1gMNAccqBN6WV1gQzGHnwN86u5Z7r4D+BQ4N/ypRUTi38H2fU9OzuTanq14cmg3te8rZ0N7taZejco8ornaIhVOxL+bmlkboAswA7gb+NjM/klR0d+7mEOaA2sPeb4utE1ERI7Dzn0HGDZ2NjNXZfGL807gttNT1FkkDGpVTeSWvm355ydLWbA+m5Ob1w06kohESERvhjSzWsDbwN3uvgu4A/iRu7cEfgQ8c5zvPyw01ztj69atxx9YRCROrc3axyWPT2Pu2p08PKQLt5+h9n3hdF3vNtSplqgOJCIVTMQKbTOrTFGR/bK7jwttvh44+PhNiuZgH2490PKQ5y1C277H3Z9y93R3T09OTi6f4CIicebrtTsZPHoq2/cc4MWbu3Nhp2ZBR4p7dapV5qa+bflk0WYWbdgVdBwRiZBIdR0xikarF7v7A4fs2gCcEXo8ECjuV/2PgbPNrH7oJsizQ9tERKSMPlu0mauemk61ygm8fUdveqh9X8Tc2Lsttasm8sgEjWqLVBSRGtHuAwwFBoZa+c01s/OBW4H7zexr4K/AMAAzSzezMQDungX8CZgV+vhjaJuIiJTBi1+uYtiLGaQ1rsU7d/YhtVGtoCNVKHVrVObGPm3474JNfLNpd9BxRCQCzN2DzhAW6enpnpGREXQMEZHAFRY69360hCc/z2TQCY145Oou1KiiziJB2LnvAH3+PoEBJzTi0au7Bh1HRMqBmc129/Ti9mllSBGROJaTV8DI177iyc//175PRXZw6tWowvW92/DB/I0s36JRbZF4p0JbRCRO7dh7gKHPzOCDeRv55Xkn8KeLTiYxQd/2g3ZLvxSqV07g0QlaLVIk3uk7rohIHFqzfR+XPjGNr9dm88iQLtym9n1Ro0HNKgzt2Zr3v95A5tY9QccRkTBSoS0iEme+XruTSx4vat/30i09+KHa90WdW/qlUCWxEo9O1Ki2SDxToS0iEkc+XbSZK5/6kupVEhh3Z2+6t20QdCQpRnLtqlzTozXvzd3Aqm17g44jImGiQltEJE6M/XIVt72YQYfGtRl3Rx/aJat9XzS77fQUEisZoydpVFskXqnQFhGJcYWFzl8+WMRv31vIwBMa8eqwniTXrhp0LClBozrVGNK9FePmrGdt1r6g44hIGJRYaJtZLTO72Mz+aGaPhj4PNrPakQgoIiJHlpNXwMhXv+LpL1ZyXa/WPDk0Xe37YsjtZ7SjkhmjJ60IOoqIhMERC20za2BmDwMbgX8BnYF6oc/3A+vN7GEz0/q9IiIB2LH3ANeOmcEH8zfy6/NP5A8XnkRCJXUWiSVN6lbjytNa8tbstazfuT/oOCJSzo42oj0H2A10cfe27n6hu18b+pwCdAX2AFp+UUQkwlZv38ulj09j3vpsHru6K7eenqL2fTHq9v7tAHhCo9oicedohXa6u//a3Yu9S8Pdl7v7r4DTwhNNRESK89WaHVwyehpZ+w7w8i09+MGpTYOOJMeheb3qXNatJa/PWsum7Jyg44hIOTpioe3u20rzBqV9nYiIHL9PFm5iyNPTqVE1gbfv6M1pbdS+Lx7c2b8dhe48MVmj2iLxpExdR8yskZm9ZGYLzOxdM2sfrmAiIvJdL0xbxW0vzaZDkzq8c6fa98WTlg1qcEnX5rwycw1bdmlUWyRelLW936PAZOBS4HPglXJPJCIi33Gwfd/v3l/ImSc25rVbe9Kwltr3xZvhA1IpKHSe/Dwz6CgiUk6OWmib2QNmVu2QTY2AMe7+DTAaaBfOcCIiFV1OXgEjXp3D01+s5IbebXji2m5Ur5IQdCwJg9ZJNbm4c3NenrGarbtzg44jIuWgpBHtLcAsMzsj9Hw88KmZ/SX0+J1whos1GauyWLF1T9AxRCROZO09wDVjZvDh/E385gcn8rsfdlT7vjg3fEA7DuQXMuYLjWqLxIOjrmrg7n83s3eAp81sEfBTYAbQiaJe2u+GPWEM+cO/F7FgQzY/PLUZIwam0r6x1vQRkWOzevtebnhuFut37mf0NV05/xR1FqkIUpJrcWGnZoz9cjXDTk8hSVOERGJaiXO03f0bdz8dWEhRkZ3g7v9w93HuXhj2hDHkuRtPY9jpKXy2eDPnPPg5d748m8UbdwUdS0RizMH2fTv3HeCVW3qoyK5gRgxMJSe/gGemrAw6iogcJ3P3o7/ArDOQCswHcoCnKJpSMsrdd4Q74LFKT0/3jIxg1tLJ2nuAZ6es5Plpq9iTm8/ZHRszalAaJzevG0geEYkdHy/cxF2vfUWj2tV4/sbTSFFnkQppxCtzmLhkC1N+PpD6NasEHUdEjsLMZrt7enH7SroZ8o/AOIq6jHwAnO3u5wATgWlmdll5h40HDWpW4SfndGDqzwdy95lpTM/czgWPTOGm52fx1Zqo/d1ERAL23NSV3P7SbE5oUod37uytIrsCGzkwjb0HCnh2qka1RWLZUUe0zWwrkObuO80sCfjE3buF9jUFRrv74MhELZsgR7QPtysnj7HTVjFmykp27svj9PbJjBqYSroWmhARQu37PlzMM1NWcnbHxjx0VRd1FhHueGk2U5ZtY8rPB1K3RuWg44jIERzziDaQBfQ2s8pAX2D7wR3uvjFai+xoU6daZUYMTGPKzwfyi/NOYOH6bC574kuufno60zO3l/wGIhK3cvIKGP7KHJ6ZUtS+73G175OQkQPT2J2bz3PTNKotEqtKGtHuQ1G/7FTga+Bmd19c5pOYtQTGAo0BB55y94fM7HWgQ+hl9YCd7t65mONXAbuBAiD/SL81HCqaRrQPt+9APq/MWMOTn2eydXcu3ds0YNSgNPqkJmGm1l0iFUXW3gPc8sIsvlq7k9/8oCM3920bdCSJMsPGZjA9cztTfjGQOtU0qi0SjY42ol3izZDlFKAp0NTd55hZbWA2cLG7LzrkNfcD2e7+x2KOXwWku/u20p4zmgvtg3LyCnht5hqemJzJpl05dG1Vj5GD0ujfPlkFt0icW7VtLzc8N5ON2Tk8eGVnzlNnESnGgvXZXPDIFH5ydntGDEwLOo6IFOOYpo6YWan+dlma14WmmcwJPd4NLAaaH/IeBlwBvFqac8aLapUTuKFPWyb/rD9/vvhkNu/K5cbnZnHRY1P5bNFmIvFLkIhE3uzVO7jk8Wlk78/jlVt7qsiWIzq5eV0GndCIMVNWsic3P+g4IlJGR5ujvdDMrj9sCfZvmVlVM7ueorZ/pWZmbYAuFPXkPqgfsNndlx3hMAc+MbPZZjasLOeLBVUTE7i2Z2sm/qQ/f7/kFHbsO8AtYzP4wcNT+GjBRgoLVXCLxIuPFmzk6qenU7taIuPu7EO31vWDjiRRbuSgNHbuy+PFL1cHHUVEyuiIU0fM7GTgPqA3MA1YBOwC6gAdgV7Al8BP3X1hqU5mVguYDPzF3ccdsv1xYLm733+E45q7+3ozawR8Cox098+Led0wYBhAq1atuq1eHZvflPIKCnlv7gYem7icldv20qFxbUYMTOX8U5pq+WWRGPbslJX86YNFdG5ZjzHXpWvVPym165+dyfz12XzxswHUrHrURZ1FJMKOa462mXUALgK6AvWBHcBXwHvuvqQMISoD/wE+dvcHDtmeCKwHurn7ulK8z++BPe7+z6O9LhbmaJckv6CQ/8zbyCMTlrFi617aJddk5MA0Lji1KYkJJS7qKSJRoqDQ+csHi3l26krOOamofV+1yuosIqU3e/UOLn18Gr86/wSGnd4u6DgicohouBnSgBeALHe/+7B95wK/dPczjnBsTaCSu+8OPf4U+KO7f3S0c8ZDoX1QQaHz3wUbeWT8cr7ZvJu2DWsyfEAqF3VuRmUV3CJRLSevgLtfm8tHCzdxU5+2/PoHJ+ovU3JMhj4zg8Ubd/HFzwaqBaRIFDmePtrlpQ8wFBhoZnNDH+eH9l3FYTdBmlkzM/sw9LQxMMXMvgZmAh+UVGTHm4RKxgWnNuO/d/XjiWu7Ub1yAj9582sG3j+J12au4UB+YdARRaQY2/fkMuTp6Xy8aBO/vaAjv/1hRxXZcsxGDUpj254DvDJzTdBRRKSUIjKiHYR4GtE+nLszfvEWHp6wjHnrsmlerzq392/HFektqJqoUQ6RaLAy1L5vU3YOD13VmXNPVmcROX5DnprO8q17+OJnAzT9SCRKRMOItpQjM+PMjo15b3gfnr/xNBrXqcr/vbuAM+6bxHNTV5KTVxB0RJEKbfbqHVwyeiq7c/J55daeKrKl3IwalMbW3bm8Pmtt0FFEpBRUaMcwM6N/h0a8fUdvXr6lB60a1OAP/15E33snMuaLTPYdUM9VkUj77/yi9n11q1dm3B291b5PylXPlAZ0b9OAxyetIDdfgyoi0a5MhXZo7nTPcIWRY2Nm9EltyBu39+K1YT1p37gWf/5gMf3uncjjk1ZokQORCHlmykrufGUOJzWrw7g7+9CmYc2gI0mcMTNGDUpj064c3sgosVGXiASsVIW2mTUys8+AdcBnoW1XmtnocIaTsuuZksQrt/bkrdt7cVLzutz70RL63juBRycsY1dOXtDxROJSQaHz+/cX8qf/LOLck5rwyq09aVCzStCxJE71SU2ia6t6PD5xuW6GF4lypR3RfhhYCSQDB6u1CcBZ4Qglxy+9TQPG3tSdd4f3oVur+vzzk6X0/fsE/vXpUrL3qeAWKS/7DxRw58uzeX7aKm7u25bHru6qm9QkrA6Oam/IzuHtORrVFolmpeo6YmabgdbunmNmWe7eILQ9293rhjvksYjnriPHYv66bB6ZsIxPFm2mVtVEbujdhpv7tqW+Rt1Ejtn2Pbnc/EIGX6/byW8v6MiNfdoGHUkqCHfn4tHT2L4nl4k/6a81FUQCVB5dR3KB76z5amYNgKzjzCYRckqLujx1XTr/vasfZ7RP5rFJy+lz7wT+9t/FbNuTG3Q8kZiTuXUPlzw+jcUbd/H4Nd1UZEtEmRl3DUpl3Y79vPPV+qDjiAQqv6CQ4a/MYcqybUFH+Z7SFtqfAPeHllE/6A/AB+UfScLpxKZ1eOyarnx89+mceWJjnv48k773TuBP/1nEll05QccTiQmzV2dx6ePT2J2Tz6vDenLuyU2CjiQV0IAOjTi5eR0em7ic/ALN1ZaK6y8fLuaDeRvZkL0/6CjfU9pC+2fAicAOoI6Z7QROBX4TplwSZu0b1+bhIV349MdncP4pTXl+2ir63jeR37+/kI1R+D+qSLT4cP5Ghjw9g3o1qvDOnb3p2krt+yQYZsaogWms3r6P97/eEHQckUC8PmsNz00tukfmivSWQcf5nhLnaJuZUXQT5DagC9AWWA1keBQvK6k52mWzevteRk9cwdtz1lHJjMvTW3BH/3a0qF8j6GgiUcHdeWbKSv7y4WK6tqrP09elq7OIBM7dOf/hKeTmFfDpj88goZIFHUkkYjJWZTHk6en0TEniuRtOIzGgexWONke7NIV2JWAvUNvdY6YhswrtY7M2ax+PT17BmxlrcYfLurXgzv6ptEpSwS0VV0Gh86f/LOL5aas4/5QmPHBFZ3UWkajx3/kbuePlOTx0VWcu6tw86DgiEbF+534uenQKtatV5t07+1C3RuWSDwqT47oZ0t0LgUygQXkHk+jTskEN/jr4FCb/dADX9GjFuK/WM+D+Sdzzxtdkbt0TdDyRiNt/oIDbXypq33drv7Y8OkTt+yS6nHNSEzo0rs0jE5ZTUBi1f2gWKTf7DuRz6wsZ5OYV8vR16YEW2SUp7Rj7/cDLZtbHzFqEVohsZmbNwhlOgtOsXnX+cNHJfPGzAVzfqw0fzN/AmQ9M5q7XvmLZ5t1BxxOJiG17crnq6el8tngzv/9hR379g45U0p/mJcpUqmSMHJTK8i17+O+CjUHHEQkrd+enb85j8aZdPHx1F1Ib1Qo60lGVto/2obczHzzAAHf3qBza0dSR8rV1dy5jvsjkxemr2Z9XwPmnNGXkwFROaFIn6GgiYbFi6x5ufG4WW3bn8PBVXTj7JHUWkehVUOic8+DnJJjx37v66RdCiVuPjF/G/Z8u5Vfnn8Cw09sFHQc4+tSRxOI2FkMNYiu45NpV+eX5J3LbGe14ZkomL0xbzQfzNnLOSY0ZOTCNk5tH5bpFImWyKTuHGSu3Mz1zOx/O30RiJeO1Yb3o3LJe0NFEjiqhkjFyYCp3vTaXTxZt4tyTmwYdSaTcfbRgE/d/upTBXZpza7+UoOOUSqlGtGORRrTDa+e+Azw7dRXPTV3J7px8zjyxESMHptFJBYnEkI3Z+5mRmcX0zKLietX2fQDUrpZIr5Qkfv2DE2mdVDPglCKlU1DonPXAZKpWTuDDUX0pahomEh+WbNrFJaOnkda4Nq8P6xlV98ocV9eRQ97kLGAQRa3+vr163f2m8ghZ3lRoR0b2/jzGTlvFmCkryd6fxxntkxk1KI1urdVbWKLPhp37i0asV2QxfeV2VocK6zrVEuneNomeKQ3omZLEiU3rqE2axKS3Z6/jnje/5unr0jmrY+Og44iUi6y9B7jw0SkcyC/k3yP70rhOtaAjfcdxF9pmdhfwN4pWgrwA+A9wHjDO3a8rx6zlRoV2ZO3JzefFL1fz9BeZZO09QJ/UJEYNTKNHSlLQ0aQCW79zPzNCo9XTM7NYk/W/wrpHShI9U5Lo0baBCmuJG/kFhQy8fzJ1qify7xEa1ZbYl1dQyNBnZjBnzU7euC06p/KVR6G9DLjV3SeZ2Q53r29mPwAucfebyzlvuVChHYx9B/J5efoanvw8k217cunRtgF3DUqjV7skfcOXsFu3Y9//poKs3M7arKJVTutWr0yPtkWj1T1SGnBCExXWEr/emLWWn709j2dvSGfgCRrVltj2f+8u4MXpq/nXlZ0Y3KVF0HGKVR6F9m53rx16nOXuDUIrRm5194blG7d8qNAO1v4DBbw6cw1Pfr6Czbty6da6PqMGpXF6WkMV3FJu1mbtY8bK/82xXrejqLCuV+N/hXXPlCQ6NK6tLgxSYeQVFDLgn5NIqlWVd+/sre+5ErNemr6a37y7gNvOSOGX550YdJwjKo+uI1vMrLG7bwbWmVkPipZkD2atS4l61askcFPftlzdoxVvZqzl8UkruP7ZmXRqUZdRg9IYeEIjffOXMnF31u3Y/+00kOmZ21m/s6iwrl+jMj3aJnFL37b0bJdE+0YqrKXiqpxQieEDUvnluPl8vmwbZ7RPDjqSSJlNz9zO799fyIAOyfzsnBOCjnPMSjui/Rdgobu/YmajgL8D+cBYdx8R5ozHRCPa0eVAfiFvz1nHYxOXs27Hfk5qVodRg9I468TGKoikWO7O2qz9TA+125uRmfVtYd2gZpXvjFinNaql/49EDnEgv5D+/5hIk7rVePsOjWpLbFmbtY8LH51Cg5pVeGd4H+pUi96VH6Gcuo4c9oZ9gNrAxx6l/QFVaEenvIJC3v1qPY9NXM6q7fs4oUltRg5M47yTm6hQquDcnTVZ+77Tbm9Ddg5QVFgf7AjSMyWJ1GQV1iIleXH6av7v3QW8fEsP+qRG5SxPke/Zm5vPpY9PY8PO/bw3oi9tG0Z/i9VyL7SPIUBLYCzQmKKVJZ9y94fM7HWgQ+hl9YCd7t65mOPPBR4CEoAx7v73ks6pQju65RcU8u95G3hkwnIyt+4lrVEtRgxM5YJTm+kmtQrC3Vm9fV/RaHVonvXGUGGdVLNKqKguKq5TG9XSiJxIGeXmF3DGfZNolVSDN27rFXQckRIVFjp3vDybTxdt5oWbutMvLTamPR33HG0zqwmMAtIpGsn+lrufXYq3yAfucfc5ZlYbmG1mn7r7lYec434gu5hzJwCPAWcB64BZZva+uy8qTXaJTokJlRjcpQUXdmrOB/M38uiEZdz12lwe+mwZwwekclHnZiQm6BaAeOLurDpYWIfmWW/aVVRYN6xV5dt2e71SGtAuWYW1yPGqmpjA7Wek8Pt/L+LLFdvp1U7tViW6PTh+GR8v3MxvL+gYM0V2SUp7M+SzQBfgXWBvWU/i7huBjaHHu81sMdAcWAQQ6mByBTCwmMO7A8vdPTP02teAiw4eK7EtoZJxYadmXHBKUz5euImHJyznnje/5qHxyxg+oB2Du7SgSqIK7ljk7qzctpfpmVnfLmu+eVcuAMm1q35njnW75JoqrEXC4KrurXhs0goeHr9MhbZEtQ/mbeTh8cu4vFsLbuzTJug45aa0hfbZQHt333q8JzSzNhQV7TMO2dwP2Ozuy4o5pDmw9pDn64Aex5tDokulSsZ5pzTl3JOb8NniLTw8fhk/f3s+D49fzp0D2nFZtxZUTYye5Vbl+9ydzG17v71xcXrmdrbs/l9hfehUkJSGKqxFIqFa5QRuOz2FP3+wmJkrs+jetkHQkUS+Z8H6bO55cy5dW9Xjz4NPjqufD6UttLcDe473ZGZWC3gbuNvddx2yawjwajm8/zBgGECrVq2O9+0kAGbGWR0bc+aJjZj0zVYeGr+MX7+zgEcnLOf2M9px5WktqVZZBXc0cHdWbN377Y2LM1ZmsTVUWDf6trAuKq7bqrAWCcw1PVrzxOQVPDJhGS/erHEqiS5bd+cybGwG9WtU4Ymh3eJuUK20hfavgIfN7OfunnUsJzKzyhQV2S+7+7hDticClwDdjnDoeqDlIc9bhLZ9j7s/BTwFRTdDHktOiQ5mxoATGtG/QzJTlm/j4fHL+N37C3ls4nKGnZ7CNT1aU71KfF2M0a6osN7Dl6HR6hmZWWzbU1RYN65Tld7tkr4trtsk1VBhLRIlqldJYNjpKfz1wyXMXr2Dbq3rBx1JBChqQ3nHS7PJ2neAt27vTaPa1YKOVO6O2HXEzPIo6hByUGLoecGhr3P3KiWepOgn7gtAlrvffdi+c4FfuvsZRzg2EVgKDKKowJ4FXO3uC492TnUdiS/uzvTMLB4ev4wvM7fTsFYVbu2XwrU9W1Ozaml/X5SycHeWb9nz7QIxM1ZuZ9ueAwA0qVPtO+32WquwFolqe3Pz6XffRE5tUZfnb+wedBwR3J1fjpvPa7PW8siQLvywU7OgIx2zY+06cmY5ZugDDAXmm9nc0LZfufuHwFUcNm3EzJpR1MbvfHfPN7MRwMcUtfd7tqQiW+KPmdGrXRK92iUxc2UWj0xYxt/+u4QnJq/gln4pXNerNbWjvKF9tHN3ln1bWBeNWG/fW1RYN61bjdPTkukRKq5bNVBhLRJLalZN5JZ+bbnvo2/4eu1OOrWsF3QkqeDGfrma12atZcSA1JgusksSkT7aQdCIdvybs2YHj4xfxsRvtlKnWiI3903hhj5tqFtdBXdpFBYeVlivzCIrVFg3q1uNnu2S6Nm2aMS6ZYPqKqxFYtye3Hz63juB9Nb1GXP9aUHHkQps6vJtXPfsTAae0Ignr+0W8wuQHfOCNWZ2EpDv7t+EntelaOGYzsDnwE/dPbfcE5cDFdoVx7x1O3lkwnI+XbSZ2lUTuaFPG27q05b6NUuc1VShFBY6S7fsZvqK/00F2bEvD4Dm9arTMyWJHikN6JWSRIv6KqxF4tEj45dx/6dL+c/IvpzcvG7QcaQCWr19Lxc+OpXGdaoy7s4+1IqD6Z/HU2h/Bjzm7u+Enj8PDADeBAYDb7j7L8s9cTlQoV3xLNyQzaMTlvPfBZuoWSWBob3acGu/tiTVqhp0tEAUFjpLNu3+tof1jJVZ7AwV1i3qV6dH2/+122vZoEbAaUUkEnbl5NH37xPomZLEU9cVWxeIhM3unDwuGT2NrXtyeX94X1olxcfPnuMptDdR1D97l5lVAbKA8939czPrArzu7u3Dkvo4qdCuuL7ZtJtHJy7nP/M2UC0xgWt7tuLW01Pi8m7mQxUWOos37fq2h/XMVf8rrFs2qE7Ptkn0SEmiR9sGKqxFKrAHP1vKg58t48NR/ejYrE7QcaSCKCh0ho3NYNLSrbx4c3d6t2sYdKRyczyF9i53rxN63B34FKjnoYPMLNvdo/JvTyq0ZfmWPTw2cTnvzV1P5YRKDOneitvPaEeTuvFRcBcUOos37mLGylBhvTKL7P1FhXWrBjXomdKAHm2LpoO0qK/CWkSKZO/Lo++9E+jXviGjrzlSZ12R8nXfR0sYPWkFf7roJIb2ahN0nHJ1rF1HALLMrIW7rwN6AhmHFNk1OazVn0g0SW1Ui39d2Zm7BqXx2MTlvDh9Na/MWMOVp7Xk9v7taF6vetARy+RgYX2w3d7MldvZlZMPQOukGpx7UhN6pDSgR0pSzP3bRCRy6taozA192vDIhOV8s2k3HZrUDjqSxLn35q5n9KQVDOneimt7tg46TkSVNKL9AEXLpb8O/AK4z91Hh/b1Ax5096j8dVgj2nK4tVn7GD1pBW/NXgvAZd1acGf/1KidRlFQ6CzasCs0v7pojvXuUGHdJqnGtzcv9mibRDMV1iJSBjv2HqDvvRMYeGJjHhnSJeg4EsfmrdvJ5U98SacW9Xjplh5USawUdKRydzxTR6oDDwK9KOoycpe7F4T2/R7Idfe/lXfg8qBCW45k/c79PDFpBa/PWkuBO4O7NGf4gFTaNqwZaK78gkIWhUasZ2RmMXNlFrtziwrrtg1rfmcqSNO6KqxF5Pjc+1HRWgSf/ugMUhvVCjqOxKEtu3K48NGpJFQy3h/RJ26bExxzoR3LVGhLSTZl5/Dk5yt4ZcYa8goKuahzUcEdqR84+QWFLPx2xDqLWYcU1ikNa9Ij5X9dQRrXiY955SISPbbvyaXvvRM556TGPHiVRrWlfOXkFTDk6eks2bibt+/oHdc33qrQFjmKLbtzGPPFSl78cjU5+QX84JSmjByYVu7zFvMLClmwYde3C8RkrNrBnoOFdXLNb5cz79m2AY1UWItIBPz1w8WM+SKTz358BinJGtWW8uHu/PStebw1ex2PX9OV805pGnSksFKhLVIK2/fkMmbKSsZOW8XeAwWcd3ITRgxM5aRmx9ZYJ6+gkAXrs5keareXsSqLvQeK7h9ud0hh3UOFtYgEZOvuXPrdN4EfnNKM+6/oFHQciRNjvsjkzx8s5u4z07j7zKjsAl2uVGiLlMGOvQd4bupKnpu6it25+Zx5YmNGDUrl1Bb1jnpcXkEh89dnf9sVZPYhhXVqo1rfTgPp3rZB3Pf0FpHY8af/LOL5aauYcM8ZtE4K9l4ViX2Tl27lxudmcs5JTXjs6q4xv7x6aajQFjkG2fvzeH7qKp6dupLs/XkM6JDMyEFpdG1VHygqrOety/52Ksjs1TvYFyqs0xrV+nbEunvbBiTXjs8bQEQk9m3ZlUPf+yYyuHNz7r3s1KDjSAzL3LqHix6bSov6NXj7jl7UqBL7y6uXhgptkeOwOyePsV+uZswXmezYl0fvdkkkVDIyVu1gf15RYd2+8XcL64Zxeme1iMSn37+/kJemr2biT/pHbctTiW7Z+/MYPHoq2fvyeG9Enwq1UNoxFdpmVgiUWIW7e8LxxQsPFdpS3vbm5vPyjNU8P3UVtatV/s5UkHhtWSQiFcPG7P2ccd8kLktvwV8HnxJ0HIkxBYXOTc/PYurybbxya0+6t20QdKSIOtaVIfsd8jgduB24H1gJtAV+BDxZXiFFol3NqokMO70dw05vF3QUEZFy1bRuda44rQWvz1rLiAGpWgRLyuTej5YweelW/nbJKRWuyC7JEZfncfepBz+AG4AL3H2Mu4939zHAhcCNEcopIiIiYXRH/1QAHp+0IuAkEkvenr2Opz7P5LperRnSvVXQcaJOadfBbAesPWzbeiClfOOIiIhIEJrXq85l3YpGtTdl5wQdR2LAnDU7+OW4+fRKSeL/LugYdJyoVNpCezbwTzOrBhD6fC/wVbiCiYiISGTd2T+VQneemKxRbTm6Tdk53PbibBrXrcroa7pSOaG0JWXFUtr/KrcCZwM7zGw1sAM4BxgWrmAiIiISWS0b1OCSrs15deYatuzSqLYULyevgGEvZrAvN58x151G/ZpVgo4UtUpVaLv7cuAk4Ezgp6HPJ7n70jBmExERkQgbPiCV/ELnqc8zg44iUcjd+fnb85i/PpsHr+pChya1g44U1Uo9zu/uBcA04IvQTZIF4YslIiIiQWidVJOLOjfjpRmr2bYnN+g4EmWe/DyT9+Zu4Cdnd+Csjo2DjhP1SlVom1ktM3sG2A8sD2272Mx+F85wIiIiEnnDB6RyIL+Qp7/QqLb8z4Qlm7n3oyVccGpT7uyvVrelUdoR7fuBxkAf4EBo2yzgynCEEhERkeC0S67FDzs148UvV5O190DJB0jcW75lN6NenctJzerwj8s6YWZBR4oJpS20LwCucffZhFaLdPf1QLPSHGxmLc1sopktMrOFZnbXIftGmtmS0Pb7jnD8KjObb2ZzzUzLPYqIiITZiAGp7M8rYIxGtSu8nfsOcMsLGVSrnMBTQ9OpXiUqFwWPSkdbGfJQlSiaNvItM6sF7Cnl8fnAPe4+x8xqA7PN7FOKRskvAjq5e66ZNTrKewxw922lPJ+IiIgch7TGtTn/lKa8MG0Vw05PoV4NdZaoiPILChnxylds2JnDq8N6atXQMirtiPYU4JeHbRsJTCzNwe6+0d3nhB7vBhYDzYE7gL+7e25o35ZS5hEREZEwGzUwjb0HCnh2ysqgo0hA/vLhYqYs38afB59Mt9b1g44Tc0pbaP8YuNbMlgG1zGw+Rcuy/6KsJzSzNkAXYAbQHuhnZjPMbLKZnXaEwxz4xMxmm5l6d4uIiERAhya1Oe/kJjw3dRXZ+/OCjiMR9vqsNTw3dRU39WnLFektg44Tk0rbR3stcDJFhfWvgD8CnYEyjUCHppu8Ddzt7rsomrrSAOhJUX/uN6z42fV93b0rcB4w3MxOP8L7DzOzDDPL2Lp1a1miiYiISDFGDExld24+z09dFXQUiaCMVVn85t0F9EtryK/OPyHoODGrtO39fufuue7+trv/093fpKj7yOulPZGZVaaoyH7Z3ceFNq8DxnmRmUAh0PDwY0M3Xh6cWvIO0L24c7j7U+6e7u7pycnJpY0mIiIiR3BSs7qc1bExz0zJZHeORrUrgvU793P7S7NpXq86jw7pSqKWVz9mpf0vd5WZDT1s2/NAqSbrhEapnwEWu/sDh+x6FxgQek17oAqw7bBja4ZuoMTMalK0FPyCUuYWERGR4zRqYBq7cvIZ++XqoKNImO07kM+tL2SQm1fImOvTqVujctCRYlppC+0fAH8zs4NF8WigA/DDUh7fBxgKDAy16JtrZucDzwIpZrYAeA243t3dzJqZ2YehYxsDU8zsa2Am8IG7f1TK84qIiMhxOqVFXQae0Iinv8hkT25+0HEkTNydn745j8WbdvHwkC6kNtLy6serVO393D3TzC4HxpnZf4F04Ax3L1V7P3efAhyps/m1xbx+A3D+wXMDnUpzHhEREQmPUYPSuPixqbz45Wru0KqAcemxicv5YP5GfnneCQw44Wgdl6W0jjiiHRpV/vYDWA38HTiXoo4j1UPbRUREJM51blmPM9on8/QXmew7oFHtePPJwk3885OlDO7SnGGnpwQdJ24cberIOmDtYR//AppQtPz6wf0iIiJSAYwalEbW3gO8PH1N0FGkHC3ZtIsfvT6XTi3r8bdLTtHy6uXoaFNH2kYshYiIiES9bq3r0ze1IU9+nsm1PVtrKe44kLX3ALeOzaBm1USeGtqNapX1NS1PRxzRdvfVpfmIZFgREREJ1qhBaWzbk8urMzWqHevyCgq58+XZbN6Vy1PXpdO4TrWgI8Wd0vbR/sTMBh62bVDoxkgRERGpILq3bUDPlAY8MXkFOXkFQceR4/DHfy9iemYW9156Cp1b1gs6TlwqbXu/rsDnh237AjjSkukiIiISp0YNSmPL7lxen6VbtWLVS9NX8+L01dx2egqDu7QIOk7cKm2hXQgc3rE8gSO37BMREZE41SslidPa1OfxSSvIzdeodqyZnrmd37+/kP4dkvnZuVpePZxKW2jPBkYetm0EMKd844iIiEi0MzNGDUpj064c3sxYF3QcKYO1Wfu446XZtEqqwcNDupBQSWOm4VSqBWuAnwOTzOxSYCmQRtHKkP3DlEtERESiWN/UhnRtVY/HJ63givSWVEks7didBGVvbj63js2goNAZc106dappefVwK9VV4e7zgI7AW8Au4G2go7t/HcZsIiIiEqUOjmqv37mfcXM0qh3tCgudH78xl6Wbd/Po1V1JSa4VdKQKobQj2rj7JuAfYcwiIiIiMeSM9sl0alGXxyYt59JuLaicoFHtaPXg+GV8vHAz/3dBR05vnxx0nArjiIW2mV3u7m+GHl99pNe5+yvhCCYiIiLR7eCo9s0vZPDuV+u5PL1l0JGkGB/O38jD45dxebcW3NSnTdBxKpSjjWj/Dngz9PgvR3iNAyq0RUREKqiBJzTipGZ1eHTicgZ3aU6iRrWjysIN2dzzxtd0bVWPPw8+WcurR9jRVoY8+ZDHbY/wkRKZmCIiIhKNDo5qr96+j/e/3hB0HDnEtj25DBs7m3o1KvPE0G5UTdTy6pGmXztFRETkuJx1YmNOaFKbRycsp6DQg44jwIH8Qu54aTbb9+by9HXpNKqt5dWDcLQ52k+V5g3cfVj5xREREZFYU6mScdegNO54eQ7/mbeBizo3DzpShebu/Pa9BcxatYNHhnTh5OZ1g45UYR1tRLtyKT9ERESkgjvnpCa0b1yLRyYsp1Cj2oEa++VqXpu1luED2vHDTs2CjlOhHXFE291vjGQQERERiV2VKhkjB6Yx8tWv+O+CTfzg1KZBR6qQpi7fxh//s4gzT2zEPWd1CDpOhXfUOdpm1tjMrjjCvivMrFF4YomIiEisOf+UprRLrskjE5ZpVDsAq7fv5c6X59AuuSb/urIzlbS8euBKuhny5xQtt16clNB+ERERERJCo9pLNu3mk0Wbg45ToezOyeOWFzIwg6evS6e2llePCiUV2ucDY46w7znggvKNIyIiIrHsglOb0rZhTR4evwx3jWpHQkGhc/drc8nctpfRV3eldVLNoCNJSEmFdhN3L/ZX0tD2JuUfSURERGJVYkIlhg9IZdHGXXy2eEvQcSqE+z/5hvFLtvC7H3akd2rDoOPIIUoqtA+YWbF3M4S255V/JBEREYllF3duRqsGNTSqHQHvzV3P6EkrGNK9FUN7tg46jhympEJ7KjDyCPuGA1+U5iRm1tLMJprZIjNbaGZ3HbJvpJktCW2/7wjHn2tm35jZcjP7RWnOKSIiIsFITKjEiAGpzF+fzaRvtgYdJ27NX5fNz96aR/c2DfjDhSdpefUodMT2fiF/Ab4ws2TgVWA90BwYAlwD9C3lefKBe9x9jpnVBmab2adAY+AioJO75xbXxcTMEoDHgLOAdcAsM3vf3ReV8twiIiISYYO7Nueh8ct4aPwy+ndIVhFYzrbszuHWsRk0rFWVx6/tSpVELfYdjY76VXH3DOBC4AzgM2BR6PMZwIXuPqc0J3H3jQdf6+67gcUUFex3AH9399zQvuImc3UHlrt7prsfAF6jqDgXERGRKFU5NFd77tqdfLFsW9Bx4kpufgG3vTib7P15PH1dOkm1qgYdSY6gxF9/3P1Td28PdAD6AR3cvb27f3YsJzSzNkAXYAbQHuhnZjPMbLKZnVbMIc2BtYc8XxfaVtx7DzOzDDPL2LpVf6oSEREJ0qXdmtOsbjUe0lztcuPu/PqdBXy1ZicPXNGJjs3qBB1JjqLUf2dw92XuPs3dlx3rycysFvA2cLe776Jo6koDoCfwU+ANO46/Lbn7U+6e7u7pycnJx/o2IiIiUg6qJiZwR/92zF69g2krtgcdJy48M2Ulb81ex12D0jjvFK2+Ge0iNqHHzCpTVGS/7O7jQpvXAeO8yEygEDi8L816oOUhz1uEtomIiEiUuzy9JY3rVOWh8cc8Tichk5du5a8fLubck5pw16AjrSco0SQihXZolPoZYLG7P3DIrneBAaHXtAeqAIdP5JoFpJlZWzOrAlwFvB/20CIiInLcqlVO4PYz2jFzZRbTMzWqfawyt+5hxCtzaN+4Nvdf0UnLq8eISI1o9wGGAgPNbG7o43zgWSDFzBZQdJPj9e7uZtbMzD4EcPd8YATwMUU3Ub7h7gsjlFtERESO05DurUiuXZWHNap9TLL353HL2AwqJ1Ti6evSqVm1pKZxEi0i8pVy9ynAkX71uraY12+gaPn3g88/BD4MTzoREREJp2qVE7jt9BT+/MFiZq3K4rQ2DYKOFDMKCp1Rr37Fmu37ePmWHrRsUCPoSFIGarooIiIiYXdNj9Y0rFVFo9pldO9HS5i8dCt/vOhkeqQkBR1HykiFtoiIiIRd9SoJ3NovhS+WbWPOmh1Bx4kJb89ex1OfZ3Jdr9Zc3aNV0HHkGKjQFhERkYi4tmdr6teozCMa1S7RV2t28Mt35tMrJYn/u6Bj0HHkGKnQFhERkYioWTWRW/qlMPGbrXy9dmfQcaLWpuwcbntxNo3rVGX0NV2pnKByLVbpKyciIiIRc12v1tStXplHJmhUuzg5eQXc9mIGe3PzGXPdadSvWSXoSHIcVGiLiIhIxNSuVpmb+7bls8VbWLA+O+g4UcXd+cXb85i3PpsHr+pChya1g44kx0mFtoiIiETUDX3aULtaoka1D/Pk55m8O3cD95zVnrM6Ng46jpQDFdoiIiISUXWqVeamPm35eOFmFm/cFXScqDBhyWbu/WgJF5zalOEDUoOOI+VEhbaIiIhE3E192lKraiKPTlgedJTALd+ym1GvzqVj0zr847JOmGl59XihQltEREQirm6NytzQuw0fLtjI0s27g44TmJ37DnDLCxlUq1y0vHr1KglBR5JypEJbREREAnFz37ZUr5zAIxV0VDu/oJARr3zF+p37eXJoN5rVqx50JClnKrRFREQkEPVrVuG6Xm34z7wNLN+yJ+g4EfeXDxczZfk2/nLxKXRr3SDoOBIGKrRFREQkMLf2a0u1xAQem1ixRrXfmLWW56au4qY+bbnitJZBx5EwUaEtIiIigUmqVZWhvVrz3tz1rNy2N+g4EZGxKotfvzuffmkN+dX5JwQdR8JIhbaIiIgE6tZ+KVROqFQhRrXX79zP7S/Npnm96jw6pCuJWl49rumrKyIiIoFKrl2Va3q05p2v1rNm+76g44TN/gMFDBubQW5eIWOuT6dujcpBR5IwU6EtIiIigbvtjBQSKhmjJ8XnqLa785O3vmbRxl08PKQLqY20vHpFoEJbREREAte4TjWGnNaSt2avY92O+BvVfmzicj6Yt5Gfn3sCA05oFHQciRAV2iIiIhIVbu/fjkpmjJ60Iugo5eqThZv45ydLubhzM247PSXoOBJBKrRFREQkKjStW53L01vwZsZaNuzcH3SccrFk0y5+9PpcOrWoy98vPVXLq1cwKrRFREQkatzRvx0AT0yO/VHtrL0HuHVsBjWrJvLk0HSqVdby6hWNCm0RERGJGi3q1+Cybi14beZaNmXnBB3nmOUVFHLny7PZvCuXJ4d2o0ndakFHkgBEpNA2s5ZmNtHMFpnZQjO7K7T992a23szmhj7OP8Lxq8xsfug1GZHILCIiIsG4s38qBe48+Xnsjmr/8d+LmJ6Zxd8vOYUureoHHUcCEqkR7XzgHnfvCPQEhptZx9C+f7l759DHh0d5jwGh16SHPa2IiIgEpmWDGlzSpTmvzFjDlt2xN6r98ozVvDh9NbednsIlXVsEHUcCFJFC2903uvuc0OPdwGKgeSTOLSIiIrFn+IBU8goKefrzzKCjlMmMzO387r2F9O+QzM/O1fLqFV3E52ibWRugCzAjtGmEmc0zs2fN7Eh/W3HgEzObbWbDIpFTREREgtOmYU0u7tycF6evZtue3KDjlMrarH3c8fIcWiXV4OEhXUiopA4jFV1EC20zqwW8Ddzt7ruAx4F2QGdgI3D/EQ7t6+5dgfMomnZy+hHef5iZZZhZxtatW8s9v4iIiETO8IGp5OYX8vQX0T+qvTc3n1vHZpBfUMiY69KpU03Lq0sEC20zq0xRkf2yu48DcPfN7l7g7oXA00D34o519/Whz1uAd47yuqfcPd3d05OTk8PxzxAREZEIaZdcix+e2owXv1xN1t4DQcc5osJC58dvzGXp5t08enVXUpJrBR1JokSkuo4Y8Ayw2N0fOGR700NeNhhYUMyxNc2s9sHHwNnFvU5ERETiz8iBqezPK+CZKdE7qv3Q+GV8vHAzvzr/RE5vr4E++Z9IjWj3AYYCAw9r5XdfqG3fPGAA8CMAM2tmZgc7kDQGppjZ18BM4AN3/yhCuUVERCRAaY1rc/4pTXlh2mp27ou+Ue0P52/kofHLuKxbC27u2zboOBJlEiNxEnefAhR3R0Cx7fzcfQNwfuhxJtApfOlEREQkmo0cmMoH8zby7NRV/Pis9kHH+dbCDdnc88bXdG1Vj78MPlnLq8v3aGVIERERiWonNKnDuSc14bmpK8nenxd0HAC27cll2NjZ1KtRmSeGdqNqopZXl+9ToS0iIiJRb+SgVHbn5PP81FVBR+FAfiF3vDSbbXtyeWpoOo1qa3l1KZ4KbREREYl6JzWry5knNuaZKZnszgluVNvd+e17C5i1agf/uLwTp7SoG1gWiX4qtEVERCQmjBqUyq6cfMZ+uTqwDGO/XM1rs9YyfEA7LuzULLAcEhtUaIuIiEhMOLVFPQZ0SObpLzLZk5sf8fNPXb6NP/5nEWee2Ih7zuoQ8fNL7FGhLSIiIjFj1KA0du7L46XpkR3VXr19L8NfmUO75Jr868rOVNLy6lIKKrRFREQkZnRpVZ/T2yfz9OeZ7DsQmVHt3Tl53PJCBgBPX5dObS2vLqWkQltERERiyl2DUtm+9wCvzFgT9nMVFjo/en0umdv2MvrqrrROqhn2c0r8UKEtIiIiMaVb6wb0SU3iicmZ5OQVhPVc93/6DZ8t3sJvL+hI79SGYT2XxB8V2iIiIhJzRg1MY9ue3LCOar83dz2PTVzBkO4tua5X67CdR+KXCm0RERGJOT1SkujRtgFPTF4RllHt+euy+dlb8zitTX3+cKGWV5djo0JbREREYtJdg9LYsjuXNzLWluv7btmdw61jM2hYqyqPX9uNKokql+TY6P8cERERiUm92iVxWpv6PD5pBbn55TOqnZtfwG0vziZ7fx5PXdeNhrWqlsv7SsWkQltERERikpkxalAaG7NzeGv2uuN+P3fn1+8s4Ks1O3ngik6c1EzLq8vxUaEtIiIiMatvakO6tKrH6IkrOJBfeFzv9cyUlbw1ex13DUrjvFOallNCqchUaIuIiEjMOjiqvX7nft756thHtScv3cpfP1zMuSc14a5BaeWYUCoyFdoiIiIS0/q3T+bUFnV5dOJy8grKPqqduXUPI1+ZQ/vGtbn/ik5aXl3KjQptERERiWlmxqiBaazN2s+7X60v07G7cvK4ZWwGiQmVePq6dGpWTQxTSqmIVGiLiIhIzBt0YiM6Nq3DYxOXk1/KUe2CQmfUq1+xZvs+Hr+mKy0b1AhzSqloVGiLiIhIzDs4V3vV9n38e96GUh1z30dLmPTNVv5w0Un0SEkKc0KpiFRoi4iISFw4u2NjTmhSm0cmLKeg0I/62nFz1vHk55kM7dmaa3poeXUJDxXaIiIiEhcqVSoa1c7cupcP5m884uu+WrODX4ybT8+UBvz2hx0jmFAqGhXaIiIiEjfOPakJaY1q8cj4ZRQWM6q9KTuH216cTeM6VRl9TTcqJ6gUkvCJyP9dZtbSzCaa2SIzW2hmd4W2/97M1pvZ3NDH+Uc4/lwz+8bMlpvZLyKRWURERGJPpUrGyEFpLNuyh48WbvrOvpy8Am57MYO9ufmMue40GtSsElBKqSgi9WtcPnCPu3cEegLDzezg32r+5e6dQx8fHn6gmSUAjwHnAR2BIYccKyIiIvIdPzilKSnJNXn4kFFtd+cXb8/j63XZ/OvKznRoUjvglFIRRKTQdveN7j4n9Hg3sBhoXsrDuwPL3T3T3Q8ArwEXhSepiIiIxLqESsbIgaks2bSbTxZtBuDJzzN5d+4GfnJ2e84+qUnACaWiiPjEJDNrA3QBZoQ2jTCzeWb2rJnVL+aQ5sDaQ56vo/RFuoiIiFRAPzy1GW2SavDw+GWMX7yZez9awgWnNmX4gNSgo0kFEtFC28xqAW8Dd7v7LuBxoB3QGdgI3H+c7z/MzDLMLGPr1q3HG1dERERiVGJCJUYMTGPRxl3c/tJsOjatwz8u64SZlleXyIlYoW1mlSkqsl9293EA7r7Z3QvcvRB4mqJpIodbD7Q85HmL0Lbvcfen3D3d3dOTk5PL9x8gIiIiMeXizs1onVSDutUr8/R16VSvkhB0JKlgEiNxEiv69fEZYLG7P3DI9qbufrDR5WBgQTGHzwLSzKwtRQX2VcDVYY4sIiIiMS4xoRJv3NYLAxrVqRZ0HKmAIlJoA32AocB8M5sb2vYrijqIdAYcWAXcBmBmzYAx7n6+u+eb2QjgYyABeNbdF0Yot4iIiMSwxiqwJUARKbTdfQpQ3KSo77XzC71+A3D+Ic8/PNJrRURERESikZZDEhEREREJAxXaIiIiIiJhoEJbRERERCQMVGiLiIiIiISBCm0RERERkTBQoS0iIiIiEgYqtEVEREREwkCFtoiIiIhIGKjQFhEREREJA3P3oDOEhZltBVYHcOqGwLYAziuRpa9zxaCvc/zT17hi0Ne5Ygjq69za3ZOL2xG3hXZQzCzD3dODziHhpa9zxaCvc/zT17hi0Ne5YojGr7OmjoiIiIiIhIEKbRERERGRMFChXf6eCjqARIS+zhWDvs7xT1/jikFf54oh6r7OmqMtIiIiIhIGGtEWEREREQkDFdoiIiIiImGgQltEREREJAxUaIuUgpklHvK4lpmlm1mDIDOJyLEzswa6hkUk3FRoi5TAzG4ANpvZUjM7D5gH3At8bWZDAg0nIqVmZq3M7LXQysEzgJlmtiW0rU3A8USkHJjZ/KAzHCqx5JeIVHj3AB2A2sDXQBd3X2FmjYFPgVeDDCcipfY68CBwjbsXAJhZAnA58BrQM7hoIlJaZnbJkXYBTSKZpSRq7ydSAjOb6+6dQ483uHuzQ/bNc/dTAwsnIqVmZsvcPa2s+0QkuphZHvAyUFwRe5m7145wpCPSiLZIydaY2d8oGtFeYmb3A+OAM4GNgSYTkbKYbWajgReAtaFtLYHrga8CSyUiZTUP+Ke7Lzh8h5mdGUCeI9KItkgJzKwOMJyi35wfBc4BbgRWA392dxXbIjHAzKoANwMXAc1Dm9cD7wPPuHtuUNlEpPTMrB+w2t3XFLMv3d0zAohVLBXaIiIiIiJhoK4jIiUwswQzu83M/mRmfQ7b95ugcolI2ZhZDTP7mZn91Myqmdn1Zva+md1nZrWCzicipWNmiaGfyx+Z2bzQx3/N7HYzqxx0vkNpRFukBGY2BqgBzASGApPd/cehfXPcvWuQ+USkdMzsDYrmZlenqJPQYoo6kVwINHH3oQHGE5FSMrNXgZ0U3W+xLrS5BUX3WzRw9ysDivY9KrRFSnBoZ5HQwjWjgYbAEGC6u3cJMp+IlM7BDkJmZhTdyNzU3T30/Gt1EBKJDWa21N3bl3VfEDR1RKRkVQ4+cPd8dx8GzAUmAPpzs0iM8aIRpg9Dnw8+16iTSOzIMrPLzezbOtbMKpnZlcCOAHN9jwptkZJlmNm5h25w9z8CzwFtAkkkIsci4+BcbHe/6eBGM2sH7A4slYiU1VXAZfxv1ealwGbgktC+qKGpIyIiUuGZmbl+IIrEHDNLAnD37UFnKY5GtEWOgZk9FXQGETl+B69lFdkiscndt7v79mj9uaxCW+TYpAcdQETKha5lkfgQldeyCm2RY7Ml6AAiUi50LYvEh6i8ljVHW0REREQkDDSiLXIconVOmIiUja5lkdgRSys2a0RbpARm1uBIuyha5KJFJPOIyLHRtSwSH2JpxWYV2iIlMLMCYDVFP4wP8tDz5u5epdgDRSSq6FoWiQ+xtGJzYtABRGJAJjDI3dccvsPM1gaQR0SOja5lkfjwnRWbgWFm9luicMVmzdEWKdmDQP0j7LsvgjlE5Pg8iK5lkXgQMys2a+qIiIiIiEgYaOqISCmY2QnARUDz0Kb1wPvuvji4VCJSVrqWReJDrFzLmjoiUgIz+znwGkU3TM0MfRjwqpn9IshsIlJ6upZF4kMsXcuaOiJSAjNbCpzk7nmHba8CLHT3tGCSiUhZ6FoWiQ+xdC1rRFukZIVAs2K2Nw3tE5HYoGtZJD7EzLWsOdoiJbsbGG9my4CDLcBaAanAiKBCiUiZ3Y2uZZF4cDcxci1r6ohIKZhZJaA7373pYpa7FwSXSkTKSteySHyIlWtZhbbIMTCzYe7+VNA5ROT46FoWiQ/Rei1rjrbIsbk96AAiUi50LYvEh6i8llVoixwbCzqAiJQLXcsi8SEqr2VNHRE5BmbWwt3XBZ1DRI6PrmWR+BCt17JGtEWOwcGL2cxuDDqLiBw7Xcsi8SFar2WNaIscBzNb4+6tgs4hIsdH17JIfIi2a1l9tEVKYGbzjrQLaBzJLCJy7HQti8SHWLqWVWiLlKwxcA6w47DtBkyLfBwROUa6lkXiQ8xcyyq0RUr2H6CWu889fIeZTYp4GhE5VrqWReJDzFzLmqMtIiIiIhIG6joiIiIiIhIGKrRFSmBmp5rZdDNba2ZPmVn9Q/bNDDKbiJSermWR+BBL17IKbZGSjQZ+D5wCLAWmmFm70L7KQYUSkTLTtSwSH2LmWtbNkCIlq+3uH4Ue/9PMZgMfmdlQQDc5iMQOXcsi8SFmrmUV2iKlYGZ13T0bwN0nmtmlwNtAg2CTiUhZ6FoWiQ+xci1r6ohIye4FTjx0g7vPAwYB4wJJJCLHQteySHyImWtZ7f1ERERERMJAI9oiJTCzumb2dzNbYmZZZrbdzBaHttULOp+IlI6uZZH4EEvXsgptkZK9QdEyr/3dvYG7JwEDQtveCDSZiJSFrmWR+BAz17KmjoiUwMy+cfcOZd0nItFF17JIfIila1kj2iIlW21mPzOzxgc3mFljM/s5sDbAXCJSNrqWReJDzFzLKrRFSnYlkARMNrMdZpYFTKKohdAVQQYTkTLRtSwSH2LmWtbUEZFSMLMTgBbAdHffc8j2cw9pmi8iUU7Xskh8iJVrWSPaIiUws1HAe8AIYIGZXXTI7r8Gk0pEykrXskh8iKVrWStDipTsVqCbu+8xszbAW2bWxt0fAizYaCJSBrqWReJDzFzLKrRFSlbp4J+l3H2VmfWn6KJuTZRd0CJyVLqWReJDzFzLmjoiUrLNZtb54JPQxX0B0BA4JahQIlJmupZF4kPMXMu6GVKkBGbWAsh3903F7Ovj7lMDiCUiZaRrWSQ+xNK1rEJbRERERCQMNHVERERERCQMVGiLiIiIiISBCm0RERERkTBQoS0iEmfMbJKZ5ZrZbjPLNrNMM3vRzLqV4T1Wmdm14cwpIhLvVGiLiMSnP7l7bXevCwwAVgPTzWxwwLlERCoMFdoiInHO3Ve7+2+AscAjVuQuM1sSGvVeY2Z/M7MEADP7N9AKGGNme8zsk9D2RDP7lZktNbOdZjbVzNKD+5eJiEQ3FdoiIhXHa0BzoAOwDjgPqANcBNwE3ALg7j8E1gC3uHstdz87dPwfQq89F0gCngU+MrP6kfxHiIjEChXaIiIVx7rQ5yR3f9vdV3qRr4AXgUFHOtDMDBgF/NTdM929wN2fATYCPwh7chGRGJQYdAAREYmYFqHP281sCPBjIIWinwVVgOlHObYhUAv4t5kdutJZ5UPeV0REDqFCW0Sk4rgSWA/sBV4CLgH+6+4HzOyfwKHzrQsPO3Zb6Lgz3X1WJMKKiMQ6TR0REYlzZtbSzP4A3ADcRdHIdCVgK5BnZj2BoYcdtglIO/jE3R14CPinmaWF3reWmZ1jZs3C/68QEYk9VvS9U0RE4oWZTQJ6AQcAB7YD04CH3H1m6DW/BUZSNGVkIrAK6Ozu/UP7zwceARoA0939PDNLpGie9i0UTRfZS9F0k5HufnD+t4iIhKjQFhEREREJA00dEREREREJAxXaIiIiIiJhoEJbRERERCQMVGiLiIiIiISBCm0RERERkTBQoS0iIiIiEgYqtEVEREREwkCFtoiIiIhIGKjQFhEREREJg/8HtUGLioRJBwoAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "get_daily_graph(click_share_name_daily[click_share_name_daily['pyName']=='UPlusGold'], \n", - " 'Date', 'Clicked Share (%)', **{'shrinkTicks':True})" + "get_daily_graph(\n", + " click_share_name_daily[click_share_name_daily[\"pyName\"] == \"UPlusGold\"],\n", + " \"Date\",\n", + " \"Clicked Share (%)\",\n", + " **{\"shrinkTicks\": True},\n", + ")" ] }, { @@ -870,23 +329,14 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAFGCAYAAABUojiWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA6nUlEQVR4nO3deXhcd333/fdXu2XJkjfJlqWR7Xi3492WQ0hI4iTOvpvEVulTaMndlgYoFCgBCoVCgTY8pVfpc98pcPfm7sg2jrMTErJB2DzyvsR2EsfJjDZL3rTYstb5Pn9IocJ4kRONjpbP67pyZXTOnDkfZTLyxz/9zvmZuyMiIiIiIn0rKegAIiIiIiJDkYq2iIiIiEgCqGiLiIiIiCSAiraIiIiISAKoaIuIiIiIJICKtoiIiIhIAqQEHSBRxo0b55MnTw46hoiIiIgMYdu2bTvq7uPPtm/IFu3JkyezdevWoGOIiIiIyBBmZtFz7dPUERERERGRBFDRFhERERFJABVtEREREZEEUNEWEREREUkAFW0RERERkQRQ0RYRERERSQAVbRERERGRBFDRFhERERFJABVtEREREZEEUNEWERERkUHL3dny9nEqTzQHHeUPDNkl2EVERERk6Gpsaeex7VWEI1Ferz3J/7hyKp+/aXbQsX6PiraIiIiIDBq7K+sJb47x5K5qTrd3sqAwh2/fPZ9bFkwMOtofUNEWERERkQHtVGsHT+2qJhyJsaeqgcy0ZO5YNInSkhDzJuUEHe+cVLRFREREZEA6cLiR8OYYj+2o4mRrB7MmZPO1O+Zxx8ICsjNSg453QSraIiIiIjJgtLR38syeGsKRGNuiJ0hLSeKW+RMpLSlmcSgXMws6Yq/1S9E2syLgR0A+4MDD7v5dM9sAzOx+Wi5Q7+4Lzzg2A3gFSO/O+4i7f7k/couIiIhI/zh05CRlkRiPbK+kvrmdqeNG8sWbZ3PPkkJyM9OCjveu9NeIdgfwaXffbmbZwDYze97d733nCWb2ENBwlmNbgWvc/aSZpQK/MrOfuvvm/okuIiIiIonQ1hHn+X21hCNRfvPmMVKTjVVzJ1BaUsyKqWMG1ej12fRL0Xb3GqCm+3GTme0HJgH7AKzrv+IHgWvOcqwDJ7u/TO3+x/shtoiIiIgkQMXxZtaVx/jx1gqOnmyjcPQIPnvDTFYvKWJ8dnrQ8fpMv8/RNrPJwCIg0mPzFUCtu79xjmOSgW3ANOB77h452/NEREREZGDq6Izz8mtHCEei/OL1IxiwcnY+pSUhrpw+nqSkwT16fTb9WrTNLAvYBHzS3Rt77FoDrDvXce7eCSw0s1zgMTOb5+57z/L69wP3A4RCob6MLiIiIiLvwuGGFjZsqWD9lhg1DS3kj0rn49dM577lRUzMGRF0vITqt6LdPb96ExB290d7bE8B7gKWXOg13L3ezF4GbgD+oGi7+8PAwwBLly7V9BIRERGRAMTjzq8OHiUcifLC/jo6486VM8bzldvmsnJWHinJSUFH7Bf9ddcRA34A7Hf375yx+1rggLtXnuPY8UB7d8keAVwHfCuhgUVERETkoh092crGrZWsK48RO97M2JFp3H/lVNYsCxEamxl0vH7XXyPalwMfAvaY2c7ubQ+6+zPAfZwxbcTMCoDvu/tNwETg/3TP004CfuzuT/dTbhERERE5D3cn8tZxwpEYz+6tob3TWTF1DJ9ZNZPr5+aTnpIcdMTA9NddR34FnHWGu7v/yVm2VQM3dT/eTdfFkyIiIiIyQDQ0t7NpeyXhSJQ3j5xiVEYKH1oxmbUlIablZQUdb0DQypAiIiIi0ivuzs6KesKRGE/tqqa1I86iUC7/vHoBt8yfSEbq8B29PhsVbRERERE5r5OtHTyxs4rw5hj7ahoZmZbM6qWFrF1ezJyCUUHHG7BUtEVERETkrF6tbqAsEuPxHVWcautkzsRRfOPOS7ltYQFZ6aqRF6L/QiIiIiLyO6fbOnl6dzXhSIydFfVkpCZx6/wCSlcUs6AwZ9Avi96fVLRFREREhIN1TYQjMTZtq6SxpYNpeVl8+dY53LWokJzM1KDjDUoq2iIiIiLDVGtHJ8+9Wkt4c5TIW8dJTTZunDeR0pIQy6eM0ej1e6SiLSIiIjLMxI41U1YeY+PWCo6daqN4bCZ/e+Ms7llSyLis9KDjDRkq2iIiIiLDQEdnnBf21xGORPnlG0dJTjKum51P6YoQl18yjqQkjV73NRVtERERkSGsuv4067dUsGFLjNrGVibmZPCp62Zw77Ii8kdlBB1vSFPRFhERERliOuPOK28cIbw5xksHanHgqhnj+fodxVw1czwpyUlBRxwWVLRFREREhogjTa38eGsF68pjVJ44zbisdP7iqku4b1mIojGZQccbdlS0RURERAYxd+e3h44RjsR4bu9hOuLO5dPG8uBNs7l2dj5pKRq9DoqKtoiIiMggdOJUG5u2V1IWiXHo6ClyM1P58OWTWbM8xNTxWUHHE1S0RURERAYNd2d77AThzTGe3lNDW0ecpcWjeWDlNG6cN5GM1OSgI0oPKtoiIiIiA1xTSzuP76giHIlx4HATWekp3LesiLUlIWZNGBV0PDkHFW0RERGRAWpPZQNl5VGe2FlNc1snl07K4Zt3XcqtCwoYma4aN9DpHRIREREZQJrbOnhqVzXhSIzdlQ2MSE3m9oUFrC0JMb8wN+h4chFUtEVEREQGgNcON1EWifLo9iqaWjuYkZ/FV2+fyx2LJjEqIzXoePIuqGiLiIiIBKSlvZNn9x4mHImy5e0TpKUkcfOlEyktCbGkeDRmWhZ9MFPRFhEREelnbx09xbryGBu3VnCiuZ0p40byhZtmc/eSQsaMTAs6nvQRFW0RERGRftDeGeeFfbWEIzF+dfAoKUnG9XPzKS0p5rKpY0lK0uj1UKOiLSIiIpJAlSeaWV9ewYatFRxpamVS7gg+s2omq5cWkpedEXQ8SSAVbREREZE+1hl3fv5aHeFIjJdfq8OAa2blUVpSzJUzxpOs0ethQUVbREREpI/UNbawYUsF68pjVDe0kJedzgNXT+Pe5SEm5Y4IOp70MxVtERERkfcgHnd+8+YxwpEoz++rpSPuXDF9HH936xxWzs4nNTkp6IgSEBVtERERkXfh2MlWHtlWybryGG8fa2bMyDT+9P1TWLM8xORxI4OOJwOAiraIiIhIL7k7W94+QTgS5ad7DtPWGWf5lDH89XUzuGHeBNJTkoOOKAOIiraIiIjIBTScbuex7ZWEIzHeqDtJdkYKa0tClJaEmJ6fHXQ8GaBUtEVERETOwt3ZXdlAOBLlyV3VtLTHWVCUy7fvmc+t8wsYkabRazk/FW0RERGRHk61dvDkrmrCkSh7qxrJTEvmzkWFlJaEmDcpJ+h4MoioaIuIiIgA+2saKYvEeGxHFSdbO5g1IZuv3TGPOxYWkJ2RGnQ8GYRUtEVERGTYamnv5Ce7awhHomyP1ZOeksQt8wsoXRFiUVEuZlpYRt49FW0REREZdt48cpKySIxHtlXScLqdqeNH8qVb5nD34knkZqYFHU+GCBVtERERGRbaOuL8bN9hwptj/PbQMVKTjVVzJ1BaUsyKqWM0ei19TkVbREREhrSK482sK4/x460VHD3ZRuHoEXz2hpmsXlLE+Oz0oOPJEKaiLSIiIkNOR2eclw7UEY7EeOWNIxhw7ex8SlcUc8W0cSQlafRaEk9FW0RERIaMww0trN8SY315BYcbW5gwKoNPrJzOvcuKmJgzIuh4MsyoaIuIiMigFo87vzx4lPDmKC8eqCPuzpXTx/PV2+dyzaw8UpKTgo4ow5SKtoiIiAxKR0+2snFrJWXlUSqOn2bsyDTuv3Iqa5aFCI3NDDqeiIq2iIiIDB7uTuSt4/zX5ijPvXqY9k5nxdQxfHbVLFbNnUBaikavZeBQ0RYREZEBr765jU3bqyiLRHnzyClyRqTyx5dNZs3yENPysoKOJ3JWKtoiIiIyILk7OyrqCW+O8fTualo74iwO5fLQ6gXcPH8iGanJQUcUOS8VbRERERlQTrZ28PiOKsKRGPtrGhmZlszqpYWsXV7MnIJRQccT6TUVbRERERkQ9lY1UFYe44kdVZxq62RuwSi+ceel3LawgKx0VRYZfPR/rYiIiATmdFsnT+2uJhyJsauinozUJG6dX0DpimIWFOZoWXQZ1FS0RUREpN+9UdtEOBJj0/ZKmlo6mJ6XxZdvncNdiwrJyUwNOp5In1DRFhERkX7R2tHJs3sPE47EKH/rOGnJSdx46QRKS4pZNnm0Rq9lyFHRFhERkYSKHjtFWXmMjVsrOX6qjeKxmXz+xlncs6SQsVnpQccTSRgVbREREelz7Z1xXtxfRzgS5ZdvHCU5ybhudj6lK0Jcfsk4kpI0ei1DX78UbTMrAn4E5AMOPOzu3zWzDcDM7qflAvXuvrA3x/ZHbhEREbk41fWnWV8eY8PWCmobWynIyeBT183g3mVF5I/KCDqeSL/qrxHtDuDT7r7dzLKBbWb2vLvf+84TzOwhoOEijt3XP9FFRETkfDrjziuvHyEcifLSgTocuHpmHt+4M8RVM/NI1ui1DFP9UrTdvQao6X7cZGb7gUnAPgDruvrhg8A1F3usiIiIBKOuqYWNWyspi8Soqj/NuKx0/vKqady3vIjC0ZlBxxMJXL/P0TazycAiINJj8xVArbu/8S6O7bn/fuB+gFAo1AdpRUREpCd357dvHiMcifHcq4fpiDuXTxvLF26ezXVz8klNTgo6osiA0a9F28yygE3AJ929sceuNcC6d3ns77j7w8DDAEuXLvU+CS0iIiKcONXGI9sqWVce49DRU4zOTOXDl09mzfIQU8dnBR1PZEC6YNHuLrjXAouBMcBxYAfwgrs39fZEZpZKV1EOu/ujPbanAHcBSy72WBEREUkcd2db9AThSIyf7KmhrSPOssmj+fjK6dwwbwIZqclBRxQZ0M5ZtM1sDPAV4MPAUWAP0AiEgD8CxpnZfwJ/7+7HzneS7jnYPwD2u/t3zth9LXDA3SvfxbEiIiLSxxpb2nl8RxXhzTFeq20iOz2FNcuKWFtSzMwJ2UHHExk0zjeivR0IA4vc/eCZO81sGvARYCsw5QLnuRz4ELDHzHZ2b3vQ3Z8B7uOMaSNmVgB8391vusCxIiIi0kf2VDYQjkR5Ymc1p9s7mV+Yw7fuvpRbFxSQmaalN0QulrmffSqzmY1z96MXfIFePq+/LV261Ldu3Rp0DBERkQGtua2DJ3dWU1YeY3dlAyNSk7l9YQFrS0LML8wNOp7IgGdm29x96dn2nfOvp70tzwOxZIuIiMj5vXa4iXAkymPbq2hq7WBmfjZfvX0udyyaxKiM1KDjiQwJF/V7IDPLA74DLAQOAp9199cTkEtERET6WEt7Jz/dW0N4c4yt0ROkpSRxy6UTKV0RYnFoNF2XRYlIX7nYCVf/BjwPfA24GSgDzjpULiIiIgPDoSMnWVceY+O2Suqb25kybiRfvHk2dy8uZPTItKDjiQxZ5y3aZvYdui48bOnelEfXRYpuZlHgS4kOKCIiIhevvTPO8/tqCUei/PrgMVKSjFVzJ1BaEuKyS8Zq9FqkH1xoRLsO2GJmf+XuvwBeBJ43swhwFfBYgvOJiIjIRag80cz68go2bK3gSFMrk3JH8JlVM1m9tJC87Iyg44kMK+ct2u7+TTN7DPgPM9sHfIau5c8XAA8Bjyc8oYiIiJxXZ9x5+UAd4UiUn79+BAOumZVP6YoQV04fT3KSRq9FgnDBOdru/hpwpZk9QFfJ/rS7/1PCk4mIiMh51Ta2sGFLBevLY1Q3tJCXnc4D10znvmVFFOSOCDqeyLDXmyXYFwLTgJ8BTwIPm9la4OPufiKx8URERKSneNz59ZtHCW+O8fz+WjrjzhXTx/F3t85l5ew8UpOTgo4oIt0udDHkV+labj0CfBP4lruvMrOPAL8xsy+5+yP9kFNERGRYO3aylUe2VVJWHiN6rJkxI9P4syumsHZ5iOKxI4OOJyJncaER7b8Aprt7vZmNpWtU+z/c/Ydm9lPg3wEVbRERkQRwd7a8fYJwJMpP9xymrTPO8ilj+NR1M7hh3gTSU5KDjigi53Ghon0ceJ+ZPQ+8Hzj2zg53rwHuTGA2ERGRYanhdDuPbq8kHIlxsO4kozJSKF0RYu3yENPzs4OOJyK9dKGi/RG6Rq03AruAP014IhERkWHI3dlV2UB4c5SndlfT0h5nYVEu/3TPfG6ZX8CINI1eiww2F7q936/pupWfiIiIJMCp1g6e2FlNOBLl1epGMtOSuWtxIWuXh5g3KSfoeCLyHpyzaJtZsrt3XugFevs8ERER+W/7qhspK4/y+I5qTrZ2MHviKP7hjnncvrCA7IzUoOOJSB8434j2q2b2j8CGHkuw/46ZpQP3AZ8D5iQon4iIyJDR0t7J07trCEei7IjVk56SxC3zCyhdEWJRUa6WRRcZYs5XtO8Bvg1818x+A+wDGoFRdBXry4DfAqsTHVJERGQwO1h3krJIjE3bK2k43c4l40fypVvmcPfiSeRmpgUdT0QS5JxF2933AjeZ2UzgdmAxMBo4AfwC+JS7H+iXlCIiIoNMW0ec5149TDgSZfOh46QmGzfMm0hpSYiSKWM0ei0yDPR2CfZv90MWERGRQS92rJl1W2Js3FrB0ZNtFI0ZwedumMXqpYWMy0oPOp6I9KMLFm0RERE5v47OOC8dqCMcifHKG0dIMmPlrDxKVxRzxbRxJCVp9FpkOFLRFhEReZdqGk6zvryCDVsqONzYwoRRGXxi5XTuXVbExJwRQccTkYCpaIuIiFyEeNx55Y0jhCMxXtxfiwMfmDGer90xj6tnjiclOSnoiCIyQKhoi4iI9MKRplY2bqtgXXmMiuOnGZeVxp9/4BLWLA9RNCYz6HgiMgBdVNE2swIg5O6bE5RHRERkwHB3Nh86TjgS5blXD9Pe6Vw2dSyfu2EW18+ZQFqKRq9F5Nx6VbTNLA8oA64BmoEsM7sX+IC7/2UC84mIiPS7+uY2HtlWSVl5jENHTpEzIpU/vmwya5aHmJaXFXQ8ERkkejui/a/AW8B44GD3tpeAf0hEKBERkf7m7myP1ROORPnJ7hpaO+IsDuXy0OoF3Dx/IhmpyUFHFJFBprdF+2qg2N1bzMwB3P1I90i3iIjIoNXU0s7jO6sJb45y4HATWekpfHBpEWtLQsyeOCroeCIyiPW2aLee+VwzGwMc7/NEIiIi/WBvVQPhSIwndlbR3NbJ3IJR/ONdl3LbggJGputeASLy3vX2J8nPgIfM7K96bPt74Cd9H0lERCQxTrd18tSuasLlMXZV1JORmsRtCwooLSlmfmGOlkUXkT7V26L9WeBx4ASQYWb1wC7g9sTEEhER6Tuv1zZRFomxaXslTS0dTM/L4iu3zuHOxYXkjEgNOp6IDFEXLNrW9df7FOAqYBEwBYgCW93dE5pORETkXWrt6OTZvYcJb45R/vZx0pKTuOnSCZSuKGZp8WiNXotIwvVmRNvoKtbZ7r4N2JbYSCIiIu/e20dPsa48xsZtlRw/1Ubx2EwevGkW9ywpYszItKDjicgwcsGi7e5xMzsEjAHqEh9JRETk4rR3xnlxfy3hSIxfvnGU5CTj+jn5lJYU875LxpKUpNFrEel/vZ2j/RAQNrOv0DW6HX9nh7tXJyCXiIjIBVXVn2ZDeYz1Wyqoa2qlICeDT183gw8uKyJ/VEbQ8URkmOtt0f5+979XAu/My7bux7qDv4iI9JvOuPOL1+sIb47x8mt1OHD1zDxKS0JcNTOPZI1ei8gA0duiPSWhKURERC6grqmFH2+pYF15BVX1pxmfnc7Hrp7GvcuKKBydGXQ8EZE/0Kui7e7RRAcRERE5Uzzu/PbQMcKRKD97tZaOuPP+aeP44s2zuXZOPqnJSUFHFBE5p14vfWVm19E1dWQ8XdNGAHD3jyQgl4iIDGPHT7WxaVslZeUx3jp6itGZqXzk/VNYszzElHEjg44nItIrvSraZvYJ4B/pWgnyFuBp4Ebg0cRFExGR4cTd2RY9QTgS4yd7amjriLNs8mg+sXI6N8ybQEaqLgkSkcGltyPafwXc5O4/N7MT7r7azG4G7kpgNhERGQYaW9p5bHsV4UiU12tPkp2ewpplRawtKWbmhOyg44mIvGu9LdoT3P3n3Y/fuevIM8D/Af60r0OJiMjQt7uynvDmGE/uquZ0eycLCnP41t2XcuuCAjLTej2zUURkwOrtT7I6M8t391qg0sxKgKOArkIREZFeO9XawVO7qglHYuypamBEajJ3LCpg7fJiLi3MCTqeiEif6m3RXk/XhZBldN1T+2WgA/hRgnKJiMgQcuBwI2WRGI9tr6KptYOZ+dl87fa53L5oEqMyUoOOJyKSEL29vd8Xejz+VzPbBmQDzyUqmIiIDG4t7Z08s6eGcCTGtugJ0lKSuOXSiZSuCLE4NBozLSwjIkPbu5oE5+6/7usgIiIyNBw6cpKySIxHtldS39zO1HEj+eLNs7l7cSGjR6YFHU9EpN/09vZ+I4GPA0vpGsn+HXe/PgG5RERkEGnriPP8vlrCkSi/efMYKUnGqnkTKC0JcdnUsRq9FpFhqbcj2j8EFgGPA6cSlkZERAaViuPNrN8SY8OWSo6ebGVS7gg+s2omq5cWkpedEXQ8EZFA9bZoXw/McPcjiQwjIiIDX0dnnJdfO0I4EuUXrx/BgGtm5VO6IsSV08eTnKTRaxER6H3RPgacTGQQEREZ2A43tLBhSwXrt8SoaWghf1Q6D1wznfuWFVGQOyLoeCIiA05vi/aDwL+a2efc/fjFnsTMiui6FWA+XQvePOzu3zWzDcDM7qflAvXuvvAsx/+QrqXf69x93sWeX0RE3p143PnVwaOEI1Fe2F9HZ9y5csZ4vnLbXFbOyiMlWcspiIicyzmLtpm189+rQL7z3I+YWWfP57l7by4h7wA+7e7bzSwb2GZmz7v7vT3O9xDQcI7j/xP4N3TfbhGRfnHsZCsbt1VSFokRO97M2JFpfPSKqaxZXkTx2JFBxxMRGRTON6J9bV+dxN1rgJrux01mth+YBOwDsK7L0T8IXHOO418xs8l9lUdERP6Qu1P+1nHCkRg/3VtDe6dTMmUMf7NqJqvm5pOekhx0RBGRQeWcRdvdf5GIE3YX5kVApMfmK4Bad38jEecUEZFza2huZ9P2SsrKYxysO8mojBT+aEUxpSUhpuVlX/gFRETkrM47R9vM5gId7v5a99c5wHeBhcArwGfcvbW3JzOzLGAT8El3b+yxaw2w7uKin/X17wfuBwiFQu/15UREhix3Z2dFPeFIjKd2VdPaEWdhUS7/dM98bplfwIg0jV6LiLxXF7oY8rvA94DXenx9NbARuJOue2p/vjcnMrNUukp22N0f7bE9BbgLWHJRyc/C3R8GHgZYunSpX+DpIiLDzsnWDp7YWUV4c4x9NY2MTEvmniWFrC0JMbcgJ+h4IiJDyoWK9jzgRQAzSwPuAW7qnjMdBjbQi6LdPQf7B8B+d//OGbuvBQ64e+XFhhcRkd7ZV91IOBLl8R1VnGrrZPbEUXz9znncvnASWem9vQGViIhcjAv9dM3sMcVjIdAJ/BLA3XeYWX4vz3M58CFgj5nt7N72oLs/A9zHGdNGzKwA+L6739T99TrgKmCcmVUCX3b3H/Ty3CIiw9Lptk6e3l1NWXmMHbF60lOSuHVBAaUlIRYW5WpZdBGRBLtQ0T5uZoXdo80rgK3u7gBmNpKu4n1B7v4r4Kw/0d39T86yrRq4qcfXa3pzHhERgYN1TYQjMTZtq6SxpYNLxo/k726Zw92LC8nJTA06nojIsHGhov0o8H+7F5b5FPDtHvsWA28lKpiIiPRea0cnz71aS3hzlMhbx0lNNm6YN5E/KgmxfMoYjV6LiATgQkX7C8C/AH8JPA38rx77VgKPJCaWiIj0RuxYM2XlMTZureDYqTZCYzL52xtncc+SQsZlpQcdT0RkWDtv0Xb308D/OMe+ryQikIiInF9HZ5wXD9QRjsR45fUjJCcZ187Oo7SkmPdPG0dSkkavRUQGAl1qLiIySNQ0nGZ9eQXrt8SobWxlYk4Gf33tDO5dVsSEnIyg44mIyBlUtEVEBrDOuPPKG0cIb47x0oFaHPjAjPH8wx3FXD1zPCnJSUFHFBGRc1DRFhEZgI40tfLjrRWsK49ReeI047LS+IurLuG+ZSGKxmQGHU9ERHpBRVtEZIBwd3576BjhSIzn9h6mI+6875KxfP7G2Vw3J5+0FI1ei4gMJiraIiIBq29u45FtlZRFYhw6eoqcEan8yfsms6YkxCXjs4KOJyIi79I5i7aZxQG/0Au4e3KfJhIRGQbcne2xE4Q3x3h6Tw1tHXGWFI/mO9dM46ZLJ5KRqh+tIiKD3flGtK/o8Xgp8OfAQ3QtUjMF+Gt+/77aIiJyAU0t7Ty+o4pwJMaBw01kpadw79Ii1paEmD1xVNDxRESkD52zaLv7r995bGb/Btzi7m/22PYyXQvW/GtCE4qIDAF7qxoIR6I8sbOa5rZO5k0axT/edSm3LShgZLpm8YmIDEW9/el+CVBxxrYqYGrfxhERGTqa2zp4elcN4UiUXZUNZKQmcfuCSZSuCDG/MDfoeCIikmC9LdrbgH82s8+6e4uZZQDfAnYkLpqIyOD0em0TZZEYm7ZX0tTSwYz8LP7+trncsWgSOSNSg44nIiL9pLdF+6PA08BHzawOyAOiwG2JCiYiMpi0tHfy7N7DlEVilL99nLTkJG66dAKlK4pZWjwaMy2LLiIy3PSqaLv7QTObC6wAJtE1bWSzu3cmMpyIyED31tFTrCuPsXFrBSea25k8NpMHb5rFPUuKGDMyLeh4IiISoF5fgePunWb2G2CCu9ckMJOIyIDW3hnnhX21hCMxfnXwKClJxvVz8yktKeayqWNJStLotYiI9LJom1kW8F2gFOgERprZHcACd//7xMUTERk4qupPs748xvotFRxpamVS7gj+5voZfHBpEXmjMoKOJyIiA0xvR7QfAvKBy4EXurdtAb4BqGiLyJDVGXd+/lodZZEYL79WhwPXzMyjdEWID8zII1mj1yIicg69Ldq3AHPcvcHMHMDdq8ysIHHRRESCU9fYwoYtFazfUkFV/WnGZ6fzsaunce+yIgpHZwYdT0REBoHeFu0k4HTPDd3TSU72eSIRkYDE485v3jxGOBLl+X21dMSd908bxxdvns21c/JJTU4KOqKIiAwivS3avwI+z+9PE3kAeLnPE4mI9LPjp9p4ZFsFZZEYbx9rZnRmKn/6/imsWR5i8riRQccTEZFBqrdF+1PAS2b2R0CWme0B0oBrEpZMRCSB3J2t0ROEN0d5Zs9h2jrjLJ88hr++bgar5k4gIzU56IgiIjLI9fY+2hVmNo+uudpT6Fqs5mmgI4HZRET6XMPpdh7fUUU4EuX12pNkZ6SwtiTE2pIQM/Kzg44nIiJDSG9v7/fl7tv4beqxLRnYCNyVoGwiIn3C3dld2UA4EuXJXdW0tMdZUJjDt++ezy0LJpKZ1uslBURERHqtt3+63Gdmh9z9//bY9p/A6L6PJCLSN061dvDkrmrCkSh7qxrJTEvmzkWTKC0pZt6knKDjiYjIENfbon0z8IqZVbr7y2b278BMNEdbRAag/TWNlEViPLajipOtHcyakM3X7pjHHQsLyM5IDTqeiIgME72do33IzFYDj5rZT4GlwAfcXbf3E5EBoaW9k2f21BCOxNgWPUFaShK3zJ9IaUkxi0O5mGlhGRER6V/nLNpnWYwmCnwT+BxdF0WOMLMR7l6dwHwiIuf15pGTlEViPLKtkobT7UwdN5Iv3jybe5YUkpuZFnQ8EREZxs43ol0J+Bnb3hkS2tL92AHdA0tE+lVbR5yf7TtMeHOM3x46RkqSsWreBEpLQlw2daxGr0VEZEA4X9Ge0m8pRER6oeJ4M+vKY/x4awVHT7ZROHoEn71hJquXFDE+Oz3oeCIiIr/nnEXb3aP9GURE5Gw6OuO8/NoRwpEov3j9CAasnJ1PaUmIK6ePJylJo9ciIjIw9fY+2j8DvunuL/XYthL4G3e/MVHhRGT4OtzQwoYtFazfEqOmoYX8Uel8/Jrp3LusiILcEUHHExERuaDe3t5vMfDKGdt+CWzo2zgiMpzF484vDx4lvDnKiwfq6Iw7V84Yz1dum8vKWXmkJCcFHVFERKTXelu040Aqv7/kejL/fXGkiMi7dvRkKxu3VrKuPEbseDNjR6bx0SumsnZ5iNDYzKDjiYiIvCu9LdrbgAeAb/fY9lfA9j5PJCLDgrsTees44UiMZ/fW0N7prJg6hr9ZNZNVc/NJT9ENjUREZHDrbdH+HPBzM7sbeB2YTtfKkFclKJeIDFENze1s2l5JOBLlzSOnGJWRwodWTGZtSRHT8rKDjiciItJnersy5G4zmwN8CJgMbAL+y91rEphNRIYId2dHRT3hzTGe3l1Na0ecRaFc/nn1Am6ZP5GMVI1ei4jI0NPbEW3c/TDwTwnMIiJDzMnWDh7fUUU4EmN/TSMj05K5Z0kha0tCzC3ICTqeiIhIQp1vCfbV7r6x+/Hacz3P3csSEUxEBq9XqxsIR2I8saOKU22dzJk4iq/fOY/bF04iK73Xf78XEREZ1M73J96XgY3dj79+juc4oKItIpxu6+Tp3dWEIzF2VtSTnpLEbQsKKF1RzILCHC2LLiIiw875Voac1+OxlmMXkbM6WNdEOBJj07ZKGls6mJaXxZdvncNdiwrJyUwNOp6IiEhg9DtcEblorR2dPLv3MGWRGJG3jpOabNw4byKlJSGWTxmj0WsRERHOP0f74d68gLvf33dxRGQgix47RVl5jI1bKzl+qo3QmEz+9sZZ3LOkkHFZ6UHHExERGVDON6Kt3/mKCB2dcV7YX0c4EuWXbxwlOcm4dnYepSXFvH/aOJKSNHotIiJyNuebo/3h/gwiIgNLdf1p1m+pYMOWGLWNrUzMyeBT183g3mVF5I/KCDqeiIjIgHfeOdpmlg98wN1/fJZ9HwR+7u51iQonIv2rM+688voRwpEYLx2oxYGrZozn63cUc9XM8aQkJwUdUUREZNC40MWQnwOOnWPfVKAE+HSfJhKRflfX1MLGrZWURWJU1Z9mXFY6f3HVJdy3LETRmMyg44mIiAxKFyraNwEfOMe+/w28goq2yKDk7vz2zWOEIzGee/UwHXHnfZeM5cGbZnPdnHzSUjR6LSIi8l5cqGhPcPfas+1w91ozm9Cbk5hZEfAjIJ+uRW4edvfvmtkGYGb303KBendfeJbjbwC+CyQD33f3b/bmvCLyh06camPT9q7R60NHT5GbmcqHL5/MmuUhpo7PCjqeiIjIkHGhot1mZhPdvebMHWY2EWjv5Xk6gE+7+3Yzywa2mdnz7n5vj9d7CGg4y3mSge8B1wGVwBYze9Ld9/Xy3CLDnruzLXqCskiMp/fU0NYRZ2nxaB5YOY0b500kIzU56IgiIiJDzoWK9q+BB4AHz7LvY8Ave3OS7qJe0/24ycz2A5OAfQDWtbrFB4FrznL4cuCgux/qfu564PZ3jhWRc2tsaefxHVWEN8d4rbaJrPQU7ltWxNqSELMmjAo6noiIyJB2oaL9deCXZjYeWAdU0VWQ1wClwPsv9oRmNhlYBER6bL4CqHX3N85yyCSgosfXlXRdhCki57CnsoFwJMoTO6s53d7JpZNy+OZdl3LrggJGpmtBWBERkf5w3j9x3X2rmd1G19SNP6VrfrUBB4Hb3H37xZzMzLKATcAn3b2xx641dBX598TM7gfuBwiFQu/15UQGlea2Dp7aVU04EmN3ZQMjUpO5bUEBpStCzC/MDTqeiIjIsHPBoS13fx6YYWbTgfHAkXOMPJ+XmaXSVbLD7v5oj+0pwF3AknMcWgUU9fi6sHvb2bI+DDwMsHTpUr/YjCKD0WuHmyiLRHl0exVNrR3MyM/iq7fP5Y5FkxiVoQVeRUREgtLr3yF3l+uLLtjwuznYPwD2u/t3zth9LXDA3SvPcfgWYLqZTaGrYN8HrH03OUSGipb2Tp7de5hwJMqWt0+QlpLEzZdOpLQkxJLi0XR95ERERCRI/TVZ83LgQ8AeM9vZve1Bd3+GruL8e9NGzKyArtv43eTuHWb2V8BzdN3e74fu/mo/5RYZUN46eoqySJRHtlVyormdyWMz+cJNs7l7SSFjRqYFHU9ERER66Jei7e6/omtu99n2/clZtlXTtVjOO18/AzyTqHwiA1l7Z5zn99USjkT59cFjpCQZ18/Np7SkmMumjiUpSaPXIiIiA5FuPyAyQFWeaGZ9eQUbtlZwpKmVSbkj+MyqmaxeWkhedkbQ8UREROQCVLRFBpDOuPPz1+oIR2K8/FodBlwzK4/SkmKunDGeZI1ei4iIDBoq2iIDQF1jCxu2VLCuPEZ1Qwt52ek8cPU07l0eYlLuiKDjiYiIyLugoi0SkHjc+fWbRwlvjvH8/lo6484V08fxd7fOYeXsfFKTk4KOKCIiIu+BirZIPzt2spVHtlVSVh4jeqyZ0Zmp/Nn7p7BmeYjJ40YGHU9ERET6iIq2SD9wd7a8fYJwJMpP9xymrTPO8ilj+NR1M7hh3gTSU5KDjigiIiJ9TEVbJIEaTrfz2PZKwpEYb9SdJDsjhbUlIUpLQkzPzw46noiIiCSQirZIH3N3dlU2EN4c5and1bS0x1lQlMu375nPrfMLGJGm0WsREZHhQEVbpI+cau3giZ3VhCNRXq1uJDMtmTsXFVJaEmLepJyg44mIiEg/U9EWeY/21zQSjkR5fEc1J1s7mDUhm6/dMY87FhaQnZEadDwREREJiIq2yLvQ0t7JT3bXEI5E2R6rJz0liVvmF1C6IsSiolzMtLCMiIjIcKeiLXIRDtadpCwSY9P2ShpOtzN1/Ei+dMsc7l48idzMtKDjiYiIyACioi1yAW0dcZ579TDhSJTNh46TmmysmjuB0pJiVkwdo9FrEREROSsVbZFzqDjeTFl5jI1bKzh6so3C0SP47A0zWb2kiPHZ6UHHExERkQFORVukh47OOC8dqCMcifHKG0cwYOXsfEpLQlw5fTxJSRq9FhERkd5R0RYBDje0sH5LjPXlFRxubGHCqAw+sXI69y4rYmLOiKDjiYiIyCCkoi3DVjzuvPLGEcoiMV48UEfcnSunj+ert8/lmll5pCQnBR1RREREBjEVbRl2jjS1snFbBevKY1QcP83YkWncf+VU1iwLERqbGXQ8ERERGSJUtGVYcHc2HzpOOBLluVcP097prJg6hs+umsWquRNIS9HotYiIiPQtFW0Z0uqb29i0vYpwJMqhI6cYlZHCh1ZMZm1JiGl5WUHHExERkSFMRVuGHHdne6yeskiMp3dX09oRZ3Eol39evYBb5k8kIzU56IgiIiIyDKhoy5DR1NLO4zurCW+OcuBwEyPTklm9tJC1y4uZUzAq6HgiIiIyzKhoy6C3t6qBcCTGEzuraG7rZM7EUXzjzku5bWEBWen6X1xERESCoRYig9Lptk6e2l1NOBJjV0U9GalJ3Dq/gNIVxSwozNGy6CIiIhI4FW0ZVN6obSIcibFpeyVNLR1Mz8viy7fO4a5FheRkpgYdT0REROR3VLRlwGvt6OTZvYcJR2KUv3WctOQkbrx0AqUlxSybPFqj1yIiIjIgqWjLgPX20VOsK4+xcVslx0+1UTw2k8/fOIt7lhQyNis96HgiIiIi56WiLQNKe2ecF/fXEo7E+OUbR0lOMq6bnU/pihCXXzKOpCSNXouIiMjgoKItA0JV/Wk2lMdYv6WCuqZWJuZk8KnrZnDvsiLyR2UEHU9ERETkoqloS2A6484rrx8hHIny0oE6HLhqxni+UVLMVTPHk5KsZdFFRERk8FLRln5X19TCxq2VlEViVNWfZlxWOn951TTuXVZE0ZjMoOOJiIiI9AkVbekX8bjz20PHCEei/OzVWjrizuXTxvKFm2dz3Zx8UjV6LSIiIkOMirYk1IlTbTyyrZKy8hhvHT1FbmYqH758MmuWh5g6PivoeCIiIiIJo6Itfc7d2RY9QTgS4yd7amjriLO0eDQfXzmNG+dNJCM1OeiIIiIiIgmnoi19prGlncd3VBHeHOO12iay01NYs6yItSXFzJyQHXQ8ERERkX6loi3v2e7KesKbYzy5q5rT7Z3ML8zhW3dfyq0LCshM0/9iIiIiMjypBcm70tzWwZM7qwlHYuypamBEajK3LyxgbUmI+YW5QccTERERCZyKtlyUA4cbKYvEeGx7FU2tHczMz+art8/ljkWTGJWRGnQ8ERERkQFDRVsuqKW9k5/urSG8OcbW6AnSUpK4+dKJlJaEWFI8GjMtiy4iIiJyJhVtOadDR05SFonxyPZK6pvbmTJuJF+8eTZ3Ly5k9Mi0oOOJiIiIDGgq2vJ72jriPL+vlnAkym/ePEZKkrFq7gRKS0JcdslYjV6LiIiI9JKKtgBQcbyZ9VtibNhSydGTrUzKHcFnVs1k9dJC8rIzgo4nIiIiMuioaA9jnXHn5QN1hCNRfv76EQy4ZlYepSXFXDljPMlJGr0WERERebdUtIeh2sYWNmypYH15jOqGFvKy03ng6mncuzzEpNwRQccTERERGRJUtIeJeNz51cGjlEViPL+/ls64c8X0cfzdrXNYOTuf1OSkoCOKiIiIDCkq2kPcsZOtbNxWSVkkRux4M2NGpvFnV0xhzbIQk8eNDDqeiIiIyJCloj0EuTvlbx0nHInx7N7DtHXGWT5lDJ++fgY3zJtAekpy0BFFREREhjwV7SGkobmdR3dUEo7EOFh3kuyMFNaWhCgtCTE9PzvoeCIiIiLDior2IOfu7KyopywS46nd1bS0x1lYlMu375nPrfMLGJGm0WsRERGRIPRL0TazIuBHQD7gwMPu/t3ufQ8AHwM6gZ+4+2fPcvwngI8CBvyHu/9Lf+QeyE62dvDEzirCm2Psq2kkMy2ZuxYXsnZ5iHmTcoKOJyIiIjLs9deIdgfwaXffbmbZwDYze56u4n07sMDdW80s78wDzWweXSV7OdAGPGtmT7v7wX7KPqDsq24kHIny+I4qTrV1MmtCNv9wxzxuX1hAdkZq0PFEREREpFu/FG13rwFquh83mdl+YBJdBfqb7t7ava/uLIfPBiLu3gxgZr8A7gK+3R/ZB4KW9k6e3l1DOBJlR6ye9JQkbplfQOmKEIuKcrUsuoiIiMgA1O9ztM1sMrAIiAD/BFxhZl8HWoC/cfctZxyyF/i6mY0FTgM3AVv7L3FwDtadpCwS45FtFTS2dHDJ+JF86ZY53L14ErmZaUHHExEREZHz6NeibWZZwCbgk+7eaGYpwBhgBbAM+LGZTXV3f+cYd99vZt8CfgacAnbSNZ/7bK9/P3A/QCgUSuS3kjBtHXGee/Uw4UiUzYeOk5ps3DBvIqUlIUqmjNHotYiIiMgg0W9F28xS6SrZYXd/tHtzJfBod7EuN7M4MA440vNYd/8B8IPu1/lG93F/wN0fBh4GWLp0qZ/tOQNV7FgzZeUxNm6t4NipNorGjOBzN8xi9dJCxmWlBx1PRERERC5Sf911xOgqyvvd/Ts9dj0OXA28bGYzgDTg6FmOz3P3OjML0TU/e0XiUydeR2ecFw/UEY7EeOX1IyQnGStn5VG6opgrpo0jKUmj1yIiIiKDVX+NaF8OfAjYY2Y7u7c9CPwQ+KGZ7aXrjiL/j7u7mRUA33f3m7qfu6l7jnY78DF3r++n3AlR03Ca9eUVrN8So7axlQmjMvjktdO5d1kRE3NGBB1PRERERPpAf9115Fd03QP7bP7oLM+vpuuix3e+viJB0fpNPO688sYRwpEYL+6vxYErp4/na7eHuGZWHinJSUFHFBEREZE+pJUhE+xIUys/3lrBuvIYlSdOMy4rjT//wCWsWR6iaExm0PFEREREJEFUtBPA3fntoWOEIzF+9uph2judy6aO5W9vnMX1cyaQlqLRaxEREZGhTkW7D9U3t/HItkrKymMcOnKKnBGp/PFlk1mzPMS0vKyg44mIiIhIP1LR7kN//MNydlc2sDiUy0OrF3Dz/IlkpCYHHUtEREREAqCi3Yc+f+NsckakMqdgVNBRRERERCRgKtp96LJLxgYdQUREREQGCF2VJyIiIiKSACraIiIiIiIJoKItIiIiIpIAKtoiIiIiIgmgoi0iIiIikgAq2iIiIiIiCaCiLSIiIiKSACraIiIiIiIJoKItIiIiIpIAKtoiIiIiIglg7h50hoQwsyNANIBTjwOOBnBe6V96n4cHvc9Dn97j4UHv8/AQ1Ptc7O7jz7ZjyBbtoJjZVndfGnQOSSy9z8OD3uehT+/x8KD3eXgYiO+zpo6IiIiIiCSAiraIiIiISAKoaPe9h4MOIP1C7/PwoPd56NN7PDzofR4eBtz7rDnaIiIiIiIJoBFtEREREZEEUNEWEREREUkAFW0RERERkQRQ0RYRERERSQAV7T5gZvlmtrj7n/yg84hI3zCz24LOICIXz8xSejzOMrOlZjYmyEwyPKVc+ClyLma2EPifQA5Q1b250Mzqgb909+0BRRORi2Rmd525CfjeO39gu/uj/Z9KRC6Wmf0J8JCZHQM+AXwPeAuYYWafdfd1QeaT4UW393sPzGwn8D/cPXLG9hXA/3L3BYEEE5GLZmbtwHNAHV0lG+Ae4BHA3f0jQWUTkd4zsz3A1UA2sAtY5O5vdv/G+Xl3nx9oQBlWNKL93ow8s2QDuPtmMxsZRCARedfeB3wT2OLu/x+AmV3l7h8ONpaIXKROdz8KHDWzk+7+JoC715rZBQ4V6Vsq2u/NT83sJ8CPgIrubUXAHwPPBpZKRC6au28xs+uAB8zsZeBzgH7lJzL4xMzsH+ka0T5gZg8BjwLXAjWBJpNhR1NH3iMzuxG4HZjUvakKeNLdnwkulYi8F2ZWAPwLsNTdpwYcR0QugpmNAj5G11+U/w1YBXwYiAL/4O4q29JvVLRFRERERBJAt/dLEDO7P+gMItI39HkWGRr0WZb+pqKdOLriQmTo0OdZZGjQZ1n6lYp24rQFHUBE+ow+zyJDgz7L0q80RztBzCzm7qGgc4jIe6fPs8jQoM+y9Dfd3u89MLPd59oFaCl2kUFEn2eRoUGfZRlIVLTfm3y6bht04oztBvym/+OIyHugz7PI0KDPsgwYKtrvzdNAlrvvPHOHmf2839OIyHuhz7PI0KDPsgwYmqMtIiIiIpIAuuuIiIiIiEgCqGiLiIiIiCSAiraIiFw0M/uKmb0QdA4RkYFMRVtEZAgws4+b2ZtnbHvAzNzMbuyxbYSZtZjZbf2fUkRkeFHRFhEZGl4EpppZcY9tK4FXgWt6bLscSAZ+3n/RRESGJxVtEZEhwN1fBWroKteYWTLwAeDL72zrthLYAjSb2YNm9rqZ1ZvZr81sac/XNLOPmtleM2swsx1mdv25zm9mHzazSjMr6evvTURksFLRFhEZOl7iv0v1EuAw8CRwiZmN7d6+EngB+HvgduAGYCzwQ+BZMxsNXSUb+BxQCowGvgA8ambTzjypmX2t+7lXunskMd+aiMjgo6ItIjJ0vMB/TxNZCbzk7u10rYZ3tZnlAIvpmmbyceAz7n7I3Tvd/Qd0jYjf3H38J4Cvuvsud4+7+zPAy8B9Pc6XZmb/RdfI+fvc/VCiv0ERkcFEK0OKiAwdLwITzGwOXYX7f3Zvf7n763agBdgHZAFPmVnPVctSgcLux1OA75nZv/bYnwJU9vh6NnAZcL27H+/j70VEZNBT0RYRGSLcvcLMXqdrVPoy4N7uXS8BYaADeAU4CpwCrnX3Led4uSjwZXffeJ5T7gL+HdhkZh90d93uT0SkB00dEREZWl4EPgW80WOUeQeQB6wGXnB3B74L/LOZTQcwsywzW2VmBd3H/L/AV8xsoXUZYWbvN7NZPU/m7o8Ca4Afm9ntif/2REQGDxVtEZGh5QVgAl2j2AC4eyddI9kTuvdD191IngCeMLNG4A3gz+n+c8Hd/wP4NvC/gRNADPgSXdNLfo+7P0fXhZU/NLPShHxXIiKDkHUNbIiIiIiISF/SiLaIiIiISAKoaIuIiIiIJICKtoiIiIhIAqhoi4iIiIgkgIq2iIiIiEgCqGiLiIiIiCSAiraIiIiISAKoaIuIiIiIJICKtoiIiIhIAvz/91T/OnMOZqcAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "get_daily_graph(click_share_name_weekly[click_share_name_weekly['pyName']=='UPlusGold'], \n", - " 'Week', 'Clicked Share (%)', **{'shrinkTicks':True})" + "get_daily_graph(\n", + " click_share_name_weekly[click_share_name_weekly[\"pyName\"] == \"UPlusGold\"],\n", + " \"Week\",\n", + " \"Clicked Share (%)\",\n", + " **{\"shrinkTicks\": True},\n", + ")" ] }, { @@ -902,30 +352,23 @@ "metadata": {}, "outputs": [], "source": [ - "click_share_direction_daily = get_outcome_share_time(df, 'Accepted', 'pyDirection', time='daily')" + "click_share_direction_daily = get_outcome_share_time(\n", + " df, \"Accepted\", \"pyDirection\", time=\"daily\"\n", + ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAFwCAYAAACRo0zvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA3wUlEQVR4nO3deZiddX338fd39kxmsk+CGGKi7ChQjD5VsOJaFypqtWhFQVSU1oUqpWrbp1qLtWJb69qHx+US6/qouFaKG611D4qgQBExxKCQBZLMJJn9+/xxzpk5M5nlnGTmzJmZ9+u65jrn/O5z3+d7Mt7kk5/f+3dHZiJJkiSpMg1zXYAkSZI0nxigJUmSpCoYoCVJkqQqGKAlSZKkKhigJUmSpCoYoCVJkqQqNM11AdVas2ZNbty4ca7LkCRJ0gJ3ww037MrMrvHj8y5Ab9y4kS1btsx1GZIkSVrgIuKuicZt4ZAkSZKqYICWJEmSqmCAliRJkqow73qgJUmSFrOBgQG2b99Ob2/vXJeyYLS1tbF+/Xqam5srer8BWpIkaR7Zvn07nZ2dbNy4kYiY63Lmvcxk9+7dbN++nU2bNlW0jy0ckiRJ80hvby+rV682PM+QiGD16tVVzegboCVJkuYZw/PMqvbP0wAtSZKkqnR0dEz7no0bN7Jr164aVFNw/fXXc84559TkswzQkiRJUhUM0FrQhoeTvsEhevoGuX9/Pzv29bL9/gP8atd+fnFvNz//zV5u/PUefrT1Pr57xy5+cOdu7tjRw54D/WTmXJcvSVJdu/766zn77LN5znOew4knnsgLXvCCMX9/vv3tb+dhD3sYj3zkI7njjjsA2Lp1K49//OM59dRTecITnsC2bdsAuPDCC/nMZz4zsm9plnuqz7j22ms58cQTOeOMM/jc5z5Xq6/tKhyqTmYyOJwMDiX9Q8MMlH4Gx70eyrLnw/QPjn09dnvSPzh2W//QMINlzwcGx24bGBpmsHiM/rIayl8PDhVqPVxNDcHqjhZWL21lTWcra5a2sKazldVLW1jT0crqjsJj6Xlzo/8elSQtPj/5yU/4+c9/ztFHH82ZZ57Jd77zHc466ywAli9fzs0338zVV1/NpZdeype//GVe9apXccEFF3DBBRfwoQ99iFe/+tV8/vOfr/ozNm/ezMte9jK++c1vcuyxx3LeeefV4NsWGKDrwPBwefhMBkdCYDEQDk4cOkfD6RTbyoLlQNlxB0deZ1k4Lb4eH15HAmzh9WxpbgyaGxtoaghamhpobiz9FMZHx4KO1qax20rvbYpD9hvdHjQ3TbatMDYwlOze38eunn529fSxu6fwfHdPH7/c0cPOnj76Byf+M1i+pJk1HS2s7milqyxgl0J4V+doGF/a0ugFIJKkI/bmL/2cW36zb0aPefLRy/ibPzil4vc/8pGPZP369QCcfvrpbN26dSRAP//5zx95/LM/+zMAvve9743MFr/whS/k8ssvP6zP6OjoYNOmTRx33HEAnH/++Vx11VUV130kDNAVuO2efXz9lnvpLwue082Mjp+dHT8zWh6Yh45glnQqEdBSCohNhYDY1FAKomMDZGtzAx1tTcXt5SG0gZbSe5saaG4oe16+rTjWUvyM0ue1lB+neNymxkO3NTfGvAiUmUlP3yC7iwF7NGgXH/f3sau7n1vv2ceu7j729Q5OeJzWpobi7PXY2ezVZWOl8ZXtLTQ21P+fjSRpcWptbR153tjYyODg6N995X+3T/f3fFNTE8PDhUmq4eFh+vv7K/qMuWCArsAtv9nHO667HWBM8BwJkE2FWdOJZklbysLlaPgcN2taGmsYO0taPjNa2H/0+Zhtxc9tahi7zdA18yKCzrZmOtua2bhm6bTv7x8c5r79pbA9OptdCt07e/r47d5ebr57L7v390/4j6mGgFVLx85ml56Pn+le09FKW3PjbHx1SVIdqmameC586lOf4vWvfz2f+tSneNSjHgXAox/9aD75yU/ywhe+kI997GM85jGPAQqrdtxwww380R/9EV/84hcZGBiY8tgnnngiW7du5Ze//CUPechD+MQnPjHr36fEAF2BZ5x2NOecevS8mSVV/WhpauCo5W0ctbxt2vcODyd7Dw6we38fO7v7i7PZfeze3z+mpeTG+/awu6eP/f1DEx6no7VppJVk9LFstrvYy71maSvLljT5v2lJ0qy5//77OfXUU2ltbR0JuO9+97t58YtfzJVXXklXVxcf/vCHAXjZy17Gueeey2mnncZTnvIUli6deqKqra2Nq666iqc//em0t7fzmMc8hu7u7ln/TgAx31Ya2Lx5c27ZsmWuy5Dm3MH+oZGZ7dEWkn52lkJ3d6GlZHdPP/cd6GeiU725MVi9dOwsdtckLSWrlnqhpCTVg1tvvZWTTjpprstYcCb6c42IGzJz8/j3OgMtzVNLWho5ZlU7x6xqn/a9g0PD3H9gYEzYPqSlZH8/v7i3m109/ZNeLLqivXncLPbYoF0euNu9UFKStEAZoKVFoKmxga7OVro6W6d9b2bSXXah5O6ePnaO693e3dPPrb/Zx66eyS+UbGtuGAnXXSPLAU68LODK9hYa7NmXJM0TBmhJY0QEy9qaWdbWzKYKLpTsGxwqXCjZ3c+u8r7tkf7tPu7e08tPt+/lvikvlCxfgaQwm7267HUpjK9e2uKFkpKkOWWAlnREWpsaecDyJTxg+ZJp31u6UHLsEoDFVpKyiyfv2raf3T39HJjkQsnO1qZJbmpzaEvJsjYvlJQkzSwDtKSaaWgIVi5tYeXSFo5bN/37D/QPjiz3t3tc4C61k/xyZw8/3NrP/ZNcKNnS2FBY/m9kFZJWVi1tZkV7C8uWNLNiSTMr2ptZvqSZFUtaWL6kmc62JltKJEmTMkBLqlvtLU20r2qq+ELJ+w4UWkkKd5PsmyB89/M/93Rz/4F+egcmv6tmROHOksuLAXvZkkLgXlEaa28uC98tI2PLlzTbXiJJi4ABWtKC0NTYwNrONtZ2Tr/mNkDvwBD7Dg6w9+AAew4OsOdA8fmBfvaNHzs4wPb7D7LnQD97Dw4w1c1DW5saxsxoLyuG60PC97hA3tnW7M2PJM0r27dv50//9E+55ZZbGB4e5pxzzuHKK6+kpaVl0n3e+ta38sY3vhGArVu3cs455/Czn/2sViXzpje9iY6ODi677LIjOo4BWtKi1NbcSFtzI2uXVRa4S4aHk57+QfaOBO4B9hzsH3m+b9zY9vsPcMtvCiF8sp5uKMx6d7Y2sby9ELzLZ7mXj4Tw0UBePtbW3GCft6Saykye/exnc8kll/CFL3yBoaEhLr74Yv7yL/+SK6+8ctL9ygP0fGaAlqQqNDSMrlJyTJX79g8Os/fgAHvLAvdoCC+F7/6RWe+77z848nyi1UtKWhobWN4+2nIyGr5H20sODeQtLGtrosmb40g6DN/85jdpa2vjxS9+MQCNjY388z//M5s2bWLTpk3ccsstvOc97wHgnHPO4bLLLuPaa6/l4MGDnH766ZxyyilcccUVDA4O8oIXvIAf//jHnHLKKVx99dW0t7fzjW98g8suu4zBwUEe8YhH8P73v5/W1lY2btzIli1bWLNmDVu2bOGyyy7j+uuv501vehPbtm3jzjvvZNu2bVx66aW8+tWvBuCKK67gIx/5CGvXruWYY47h4Q9/+BF/fwO0JNVIS1Pl63GXy0z29w+x50D/6Cz3wbEz4PvKAvlv9vRy62+72XtwgJ6+idfpLinNek86yz0+kBfbUbxRjrS4/fznPz8kiC5btowNGzYwODjxf3fe9ra38Z73vIcbb7wRKLRw/M///A8f/OAHOfPMM7nooot43/vexytf+UouvPBCvvGNb3D88cfzohe9iPe///1ceumlU9Z022238a1vfYvu7m5OOOEELrnkEm666SY++clPcuONNzI4OMgZZ5xhgJakxSAi6GhtoqO1ifUrq9t3YGh4TODeW2ovOTB+rPD8tr372HtwkL0H+xkYmnzWu7kxRi60XF7W0z2+xaSwvaXsebO3hJdm0ldfD/fcPLPHPOph8NS3zewxJ3HMMcdw5plnAnD++efzrne9iyc96Uls2rSJ448/HoALLriA9773vdMG6Kc//em0trbS2trK2rVruffee/n2t7/Ns571LNrbCxejP+MZz5iRug3QkrSANTc2FG9KU/2s94H+oTGtJuWtJ+Xhe+/BAXZ093L7vYVZ7+5J7k5ZsrSlcWT1kjFBu31s28n47R2trukt1YuTTz6Zz3zmM2PG9u3bx7Zt21ixYgXDw6MrHfX29k56nPHn9HTneFNT08ixxx+3tXX0v3ONjY2TzoTPhJoF6IjYCnQDQ8BgZm4ujr8K+NPi+Fcy8/Ja1SRJmlhEsLS1iaWtTRy9Yvqb5JQbHBqmu3ewuJJJfzF8l7WcjAvkd+zoGdnWPzT58oKNDTHSVlLe810I32NXNSms591MR1vTyOy9q5xoQarRTPF4T3jCE3j961/P1VdfzYte9CKGhoZ43etex4UXXsiDH/xg/vVf/5Xh4WHuvvtufvjDH47s19zczMDAAM3NzQBs27aN733vezzqUY/i4x//OGeddRYnnHACW7du5Y477uDYY4/lox/9KI997GMB2LhxIzfccANPfepT+exnPzttnb/3e7/HhRdeyBve8AYGBwf50pe+xMtf/vIj/v61noF+XGbuKr2IiMcB5wKnZWZfRKytcT2SpBnW1NgwcsMcmP528CWZSe/AcPHCybI2k5HlBMdefHnf/n7u3LmfvQcH2Nc7MOGNdMq1tzQWwnRZqC697hwZbx59Pf69xeetTa56IkUE11xzDX/yJ3/CW97yFoaHh3na057GW9/6VlpaWti0aRMnn3wyJ510EmecccbIfhdffDGnnnoqZ5xxBldccQUnnHAC733ve7nooos4+eSTueSSS2hra+PDH/4wz33uc0cuInzFK14BwN/8zd/wkpe8hL/+67/m7LPPnrbOM844g/POO4/TTjuNtWvX8ohHPGJmvn9O91+cGVKcgd48LkB/GrgqM79e6XE2b96cW7ZsmYUKJUnz1dBw0t07MCZg9/QN0tM7SHfxsaevMNbdOziyrfR6f3/hcarVTkqaG6MsUDeXhe/pwnkTnWUhvb250Tte6rDceuutnHTSSXNdxoIz0Z9rRNxQ6pooV8sZ6ASui4gE/k9mXgUcDzwmIq4AeoHLMvNHNaxJkrQANDZE4ULG9hYetPrwjpGZ9A0OjwnY3X0DI0G7PHzvHxfOd3b38atd+4vbB6a802VJBHS0FNpkxgbsyWfGO1obC8/L39vW5IWZUo3VMkCflZl3F9s0vhYRtxU/fxXwu8AjgE9HxINz3LR4RFwMXAywYcOGGpYsSVosImLkBjvVLjU43sDQMPvHBe7uslnvqWbG793XO7q9b3Da1hQo3AGzc4pZ8I7W5inC+ejjkmaXJ5QqUbMAnZl3Fx93RMQ1wCOB7cDnioH5hxExDKwBdo7b9yrgKii0cNSqZkmSDkdzY8PIjPiRGB5ODg4MTdB6MjBtOP/Nnt6ymfOBKZclLGkIirPbzWPD+DR94Z1jZsm9aFMLX00CdEQsBRoys7v4/MnA3wI9wOOAb0XE8UALsGvyI0mStHg0NIyuhrJu2ZEdq29wiJ7eQfb3DU3amlLeF16aGd9TvCV9afv+KW5JX86LNmdXZvrnMoOqvSawVjPQ64Brir/oJuDjmXltRLQAH4qInwH9wAXj2zckSdKRa21qpLWjkdUdR3acoeFkf//4sF3ejjI06UWb2/YfGH3dd+QXbXa2NdHV2crazrbiYytrl7WypqN1QfeFt7W1sXv3blavXm2IngGZye7du2lra6t4n5oE6My8EzhtgvF+4Pxa1CBJko5cY0OwrK2ZZW3NR3Sc0rKFpTA91UWbY1ZM6Ru9aHPPgX7uPzBwyLEjYFV7SyFUL2ujq6MQrNcWw3bpeVdnK+0t8++ecuvXr2f79u3s3Llz+jerIm1tbaxfv77i98+//9VIkqR5LyJY0tLIkpYju2izf3CY3fv72LGvjx3dfezo7h15vrO7j53dvfzi3m52dvcxOMGMd0dr00iYXrusbfT5uLC9fElz3cz2Njc3s2nTprkuY1EzQEuSpHmrpamBByxfwgOWT33HzOHh5P4D/SPBujxs7yw+v3n7HnZ093Fggj7vlsYGusrD9bJiwB4ZK4Tt1UtbaFrA7SMqMEBLkqQFr6EhWN3RyuqOVk56wNTv7ekbZMe+3mLIHg3YO4sz21t37+eHW+9jzyTtI6uXjobs0faRtjHBu6uzlbbmxln6tpptBmhJkqQyHa1NdHR18OCuqa+47BscYldP/9iwva+XnT2jLSW3/nYfu3r6J7xgsrOtacK+7PKw3dXZxrK2prppH1GBAVqSJOkwtDY18sAVS3jgiqnbR4aGk/v294/MZI/MapcF759s28OO7t4J72LZ2tQwOptdFrbXdrbRVTbLvXppq+tv14gBWpIkaRY1NsRI//TJTL6gd2bS3TdYnL3uLV4EWezXLobtO3b28N1f7mJf7+CEn7N6actom0jZ6iNd42a5W5tsHzkSBmhJkqQ6EDG6ROCxa6duH+kdGBoJ1zuLs9rlF0Teu6+Xm+/ey+6ePiZabnv5kuYJL4bsGtdS0tFq+8hEDNCSJEnzTFtzI8esaueYVe1Tvm9oONndM/ZiyPIl/3Z29/Gjrfexo7uP/sFD20eWNDcesvLI6OvRWe5V7S00LKL2EQO0JEnSAtXYEIX1rZdNfZe9zGTfwcGxPdrjwvZt93Tz7V/sonuC9pGmhmDNmJaRYttIWdhe21m4S2RL0/xf5s8ALUmStMhFBMvbm1ne3sxx6zqnfO/B/qExF0SW+rNLLSV37+nlxl/vYff+fnKC9pGV7c0jbSKH3Iq9LGwvba3fmFq/lUmSJKnuLGlpZMPqdjasnrp9ZGBomN09/ePC9tiVSO7cuZ8d3b0MDB2atJcW71J5/LpOrnrR5tn6OofFAC1JkqQZ19zYwFHL2zhqeRuwfNL3ZSZ7DgyM6csuD9tL6vCGMwZoSZIkzZmIYOXSFlYubeGEo6ZuH6kX87+LW5IkSaohA7QkSZJUBQO0JEmSVAUDtCRJklQFA7QkSZJUBQO0JEmSVAUDtCRJklQFA7QkSZJUBQO0JEmSVAUDtCRJklQFA7QkSZJUBQO0JEmSVIWaBeiI2BoRN0fEjRGxZdy210VERsSaWtUjSZIkHY6mGn/e4zJzV/lARBwDPBnYVuNaJEmSpKrVQwvHPwOXAznXhUiSJEnTqWWATuC6iLghIi4GiIhzgbsz86c1rEOSJEk6bLVs4TgrM++OiLXA1yLiNuCNFNo3plQM3BcDbNiwYXarlCRJkqZQsxnozLy7+LgDuAZ4LLAJ+GlEbAXWAz+OiKMm2PeqzNycmZu7urpqVbIkSZJ0iJoE6IhYGhGdpecUZp1/lJlrM3NjZm4EtgNnZOY9tahJkiRJOhy1auFYB1wTEaXP/HhmXlujz5YkSZJmTE0CdGbeCZw2zXs21qIWSZIk6UhUFKAjYiXwO8Aq4D7gxsy8bzYLkyRJkurRlAE6Ip4FXAqcBewH9gHLgPaI+A7wzsy8ZraLlCRJkurFpBcRRsR/ApcBHwUenJnLMnN9Zi4DHgxcDVxWfJ8kSZK0KEw1A/3WzPyPiTZk5jbgg8AHI2LadZwlSZKkhWLSGejJwvME77tu5sqRJEmS6ltVq3BERAPwYgoXFN4B/J/MPDgbhUmSJEn1qNobqVwJPA64GXgM8G8zXpEkSZJUx6ZbheMZmfnFsqGHZ+bZxW0fBHbOYm2SJElS3ZluBvoVEfHxiFhVfP2riPibiHgS8A/ATbNbniRJklRfpgzQmfk04D+A70bEecCrgVYKa0M3A8+b7QIlSZKkejLtRYSZ+ZGI+A/gPcAfA6/IzN/OemWSJElSHZr2IsKICGAwM59D4aYq34yIi2a9MkmSJKkOTRmgI+JcYAdwc0RsA34DPBp4bER8LSIeVIMaJUmSpLox3Qz0u4CnZOYDgOcAb8vM+zPzAuCfgH+f7QIlSZKkejJdgG6lMOsM8NviawAy86vA/5qluiRJkqS6NN1FhG8GboqI24FNwJ+Ub8zMntkqTJIkSapHUwbozHx/RHwWeBBwR2beX5uyJEmSpPpUyTJ2OyhcSChJkiQtepP2QEfERyJi41Q7R8TGiPjIjFclSZIk1ampZqC/B/wgIm4CvgbcAuwDlgEnA08CTgX+eraLlCRJkurFpAE6M/81Iq4GzgeeCbwWWAncD/wE+AxwbmYeqEGdkiRJUl2Y7iLCA8BVxR9JkiRp0Zv2Vt6SJEmSRhmgJUmSpCoYoCVJkqQqTLsO9EyJiK1ANzAEDGbm5oi4EvgDoB/4JfDizNxTq5okSZKkatV6BvpxmXl6Zm4uvv4a8NDMPBW4HXhDjeuRJEmSqlJRgI6Ixoj4q4j4RUTsLY79fkS84kg+PDOvy8zB4svvA+uP5HiSJEnSbKt0BvotwDOAvwCyOHY78PIqPiuB6yLihoi4eILtFwFfreJ4kiRJUs1V2gP9x8CjMvO3EfGB4thWYGMVn3VWZt4dEWuBr0XEbZn5XwAR8ZfAIPCxiXYsBu6LATZs2FDFR0qSJEkzq9IZ6HZgx7ixFqC30g/KzLuLjzuAa4BHAkTEhcA5wAsyMyfZ96rM3JyZm7u6uir9SEmSJGnGVRqgfwy8eNzYHwM/rGTniFgaEZ2l58CTgZ9FxFOAy4FneEtwSZIkzQeVtnBcBlwfEc8D2iPiS8Bm4HEV7r8OuCYiSp/58cy8NiLuAFoptHQAfD8zj+jCREmSJGk2VRSgM/NnEXES8CLgNuAu4KWZeW+F+98JnDbB+LFV1CpJkiTNuWkDdEQ0AT8BHpGZ/zj7JUmSJEn1a9oe6OI6zSsYXb5OkiRJWrQqvYjwX4ArirPRkiRJ0qJVaSB+OYU1ny+JiN8Cw6UNmXn8LNQlSZIk1aVKA/TfzWoVkiRJ0jxR6SocH5ntQiRJkqT5oOKe5ohYDTwC6AKiNJ6ZV89CXZIkSVJdqihAR8QTgc8C/RRW5NhTfPwVYICWJEnSolHpKhxvA/42M7uAnuLjW4B/nbXKJEmSpDpUaYA+Dnhn8XmpfeMfgEtnuB5JkiSprlUaoA8ArcXnuyNiA9ACrJyVqiRJkqQ6VWmA/i7wzOLzrwJfBL4OfG8WapIkSZLqVqWrcJzPaNi+DHgd0An802wUJUmSJNWrSteBPlj2vBe4YtYqkiRJkupYNetAPwrYTGHmeURmvnWmi5IkSZLqVaXrQL8JeCNwI7C/bFMCBmhJkiQtGpXOQL8CeExm/mA2i5EkSZLqXaWrcATwo9ksRJIkSZoPKg3QHwBeMpuFSJIkSfPBpC0cEfE1Cj3OUAjafx4RrwZ+W/6+zHzy7JUnSZIk1ZepeqD/e9zrb89mIZIkSdJ8MGmAzsw317IQSZIkaT6YchWOiGgCIjMHysYuBE4H/iszPzer1UmSJEl1ZrqLCD8FvLj0IiL+CrgKOAv4WES8dBZrkyRJkurOdAF6M/DlstevAl6amZuB84FLZqswSZIkqR5NF6BXZuZvACLiJGA58Onits8DGyv9oIjYGhE3R8SNEbGlOLYqIr4WEb8oPq6s+htIkiRJNTRdgN4fER3F55uBn2Vmb/F1UPmdDEsel5mnF2ewAV4PfCMzjwO+UXwtSZIk1a3pAvS3gbdExInAy4Fry7adwLg1oQ/DucBHis8/AjzzCI8nSZIkzarpAvRfAE8BbgGWAf9Utu0FHLpW9FQSuC4iboiIi4tj6zKzFMLvAdZVcTxJkiSp5qZswcjMXwEnRcSqzLxv3Oa3A/1VfNZZmXl3RKwFvhYRt437rIyInGjHYuC+GGDDhg1VfKQkSZI0s6abgQZggvBMZu7JzAOVflBm3l183AFcAzwSuDciHgBQfNwxyb5XZebmzNzc1dVV6UdKkiRJM66iAH2kImJpRHSWngNPBn4GfBG4oPi2C4Av1KIeSZIk6XBVu4rG4VoHXBMRpc/8eGZeGxE/Aj4dES8B7gL+qEb1SJIkSYelJgE6M+8ETptgfDfwhFrUIEmSJM2EmrRwSJIkSQvFpDPQEfEtCkvPTSkzHz+jFUmSJEl1bKoWjq+XPV8DvIzC7bt/ReEW3s8E/u8s1SVJkiTVpUkDdGZeUXoeEV8EnpWZXysbeyLwmtktT5IkSaovlfZAnw18Y9zYt4DHzmg1kiRJUp2rNED/Gjhv3NhzgO0zW44kSZJU3ypdxu5y4LMR8QpgK4Ue6P9FIURLkiRJi0alt/L+CnAK8B/AfuA64JTM/PIs1iZJkiTVnYpvpJKZvwTeOou1SJIkSXWv4hupRMT5EXFdRNxUfP17EfHs2StNkiRJqj8VBeiIeC3wZuBaYENxeCeF3mhJkiRp0ah0BvoS4KmZ+U+M3p3wduDYWalKkiRJqlOVBuhVmXl78XkpQAcV3OpbkiRJWkgqDdC3RMQ548aeAvx0huuRJEmS6lqlq3C8EfhKRHwaaI2IdwPPA8aHakmSJGlBq3Qd6G8DvwscpHAL7wbg7Mz8wSzWJkmSJNWdimagI2JjZt4CvGrc+IMy865ZqUySJEmqQ5X2QN80yfhPZqoQSZIkaT6oNEDHIQMRzbgKhyRJkhaZKVs4IuJrFEJya0RcN27zBuDHs1WYJEmSVI+m64H+7+LjY4HvlI0PA/cA/282ipIkSZLq1ZQBOjPfDBARt2bmp2tTkiRJklS/KlqFIzM/HREdFNZ9Xg9sB76Smd2zWZwkSZJUbypdxm4z8O8U1oHeRqH/+V0R8bTM3DKL9UmSJEl1pdJVON4H/GNmPigzH5OZDwLeAbx/9kqTJEmS6k+lAfok4B/Hjf0TcGI1HxYRjRHxk4j4cvH1EyLixxFxY0T8d0QcW83xJEmSpFqrNEDfCDx03NjDiuPVeA1wa9nr9wMvyMzTgY8Df1Xl8SRJkqSaqqgHGrgO+HJEfAC4C9gIXARcFRF/XHpTZn58sgNExHrg6cAVwGtLuwDLis+XA7+ppnhJkiSp1ioN0BcBA8AFZWODxfGSpDCLPJl3ApcDnWVjLwX+PSIOAvuA362wHkmSJGlOVLqM3aYj+ZCIOAfYkZk3RMTZZZv+DHhaZv4gIv6cQl/1SyfY/2LgYoANGzYcSSmSJEnSEam0BxqAiDg6Ig5nlvhM4BkRsRX4JPD4iPgKcFpm/qD4nk8Bj55o58y8KjM3Z+bmrq6uw/h4SZIkaWZUFKAjYm1EfJ3CDVS+Xhw7LyLeV8n+mfmGzFyfmRuB5wHfBM4FlkfE8cW3PYmxFxhKkiRJdafSHuh3Ab8CuoA7imPfBP7ucD84Mwcj4mXAZyNiGLifsT3VkiRJUt2pNEA/DnhQZvZGRAJk5s6IWFvtB2bm9cD1xefXANdUewxJkiRprlTaA93HuLAdEauA+2a8IkmSJKmOVRqgrwP+MSKay8beDHxl5kuSJEmS6lelLRyXA5+n0KfcFhF7gJ9SuBBQkiRJWjQqXQf6PuD3IuLhwCYKdyPckpk5m8VJkiRJ9aaiAB0RK4H+zLwBuKE4tjQimjNzzyzWJ0mSJNWVSnugvwicMm7socAXZrYcSZIkqb5VGqBPAbaMG9sCPGxmy5EkSZLqW6UBuhdoHze2FBiY2XIkSZKk+lZpgP5v4K0R0QAQEQH8LfCd2SpMkiRJqkeVLmP35xRu3f2HEXEnhZU4+oHHz1ZhkiRJUj2qdBm7uyLiocA5wEZgK/CVzDwwe6VJkiRJ9afSGWgy8yDw/2axFkmSJKnuVdQDHRHXRcTjx409ISK+OjtlSZIkSfWp0osIzwD+a9zYt4FHzGw5kiRJUn2rNEAPA83jxhqBmNlyJEmSpPpWaYC+AXjVuLFXAj+e2XIkSZKk+lbpRYR/AVwfEX8I3A4cB5wAnD1LdUmSJEl1qaIZ6My8CTgZ+AywD/gscHJm/nQWa5MkSZLqTjXL2N0DXFl6HRGnRMQbMvPVs1KZJEmSVIcq7YEGICJaI+JFEfEd4GYKq3NIkiRJi0ZFM9ARcTLwcuB8oJ1C8H5KZl43i7VJkiRJdWfKGeiIeGFEfBv4GfBY4E3AA4H7gBtnuzhJkiSp3kw3A/0RYDfw9MwcuetghMs/S5IkaXGargf6r4Ee4PMRcU1E/EFEVNU3LUmSJC0kU4bhzLwCeDDwTCApLF93N7ACOHqWa5MkSZLqzrSzyVnw1cx8NvAg4H3APcCPIuLT1XxYRDRGxE8i4svF1xERV0TE7RFxa0S4JJ4kSZLqWlXtGJn528x8C4VZ6XOBlio/7zXArWWvLwSOAU7MzJOAT1Z5PEmSJKmmDqufuTgr/e+Z+cxK94mI9cDTgQ+UDV8C/G1mDhePu+Nw6pEkSZJqpZYXBL4TuBwYLht7CHBeRGyJiK9GxHE1rEeSJEmqWk0CdEScA+zIzBvGbWoFejNzM/B/gQ9Nsv/FxZC9ZefOnbNcrSRJkjS5Ws1Anwk8IyK2UuhzfnxE/BuwHfhc8T3XAKdOtHNmXpWZmzNzc1dXVy3qlSRJkiZUkwCdmW/IzPWZuRF4HvDNzDwf+DzwuOLbHgvcXot6JEmSpMM13Z0IZ9vbgI9FxJ9RuGHLS+e4HkmSJGlKNQ/QmXk9cH3x+R4KK3NIkiRJ84K35ZYkSZKqYICWJEmSqmCAliRJkqpggJYkSZKqYICWJEmSqmCAliRJkqpggJYkSZKqYICWJEmSqmCAliRJkqpggJYkSZKqYICWJEmSqmCAliRJkqrQNNcFSJIkaZEb6IWee0d/uu8ZfWxeAk+7cq4rHMMALUmSpJmXCX37oGfH2EDccw90jwvLvXsO3T8aYGkXrD255qVPxwAtSZKkyg0Pw4HdZUH4nmIw3jFu7F4YPHjo/o2t0LkOOo6CNcfBxsdAx7rRsdLj0jXQ0Fj771cBA7QkSZJgsH/yNoqRxx2wfwcMDx66f+vyYvhdBw/cDJ1HFYNx+eNaaFsBETX/ejPJAC1JkrSQ9XWXtVGMmyEuD8sH75tg5yjMBJdmhtc9dNxMcdlPS3vNv9pcMUBLkiTNN5lw4L6y9ol7J2+jGNh/6P6NLaPBd9WDYcOjJmmj6IJG4+J4/olIkiTVi6GBcSF4knaKnh0wPHDo/i2do+H3AafD8RO1UayDJSvnfRvFXDJAS5Ikzbb+/cUAfO8EbRT3jLZYHNgN5KH7t68ZnSHuOnHiNorOo6Blac2/2mJkgJYkSTocmXDw/okvtBsfkvu7D92/oWk0/K7YAOsfMXEbRcdaaGyu/ffTpAzQkiRJ5YYGYf/OCtoo7oWh/kP3b146Gn7XPRSOfeIkbRSroMGbQs9HBmhJkrQ4DBycfIa4557RFov9O5mwjWLJytGZ4dXHjp0h7jxqdFtrZ82/mmrLAC1JkuavTOjdO/kMcXdZMO7be+j+0VgIwB3rYNkD4egzxrVRFGeLO9ZCU2vtv5/qkgFakiTNjczCxXX9PYW1ivv2FR/Lf8aPlb23d09hNnmw99BjNy0ZDcFdJ8KDzy5rnyi7+K59dd3e7U71q6YBOiIagS3A3Zl5Ttn4u4CLMrOjlvVIkqTDMDRYuChuyrA7VSgu/vR3Qw5P/3mNrYW2iJGfZYXZ4rUnFWePjzr04rvWZS7TpllT6xno1wC3AstKAxGxGVhZ4zokSVpcMgsztZOG3cmC7rjx/h4YOFDZZ7aUAm/HaPgthdsxgbhzdKylY9xYh60Tqjs1C9ARsR54OnAF8NriWCNwJfDHwLNqVYskSfPG8FBZi0MVrQ4TtUUMD07/eQ1Nh4bajrWw+iGj4y3jw2/noaG4pcMVJrRg1XIG+p3A5UD5pamvBL6Ymb8N/28WSdJCMthfWdidsv+3uL0Sze2HBtgVD5og6E4Qdst/mtpsfZCmUZMAHRHnADsy84aIOLs4djTwXODsCva/GLgYYMOGDbNWpyRpkStd1DZl2K2w1WGi9YEPEYeG2bYVsPyYSYJux8Tht6UTGl0XQKqVyJxgncOZ/pCIvwdeCAwCbRR6oPuKP6VLZzcAd2bmsVMda/Pmzblly5ZZrFaSNC8N9sHBPYU7w/XuOYxVHY7worYxIXeqmd6yseZ2Z3ulOhYRN2Tm5vHjNfnnama+AXhDsZCzgcvKV+EojvdMF54lSQvc8HAh4JZC8MH7iz97xo3tOXSskgvbJryo7SgvapNUFf//HknSzBvorSAETzDWu3fqGeCmtsLd4JasLLQ6rNwIS1YUx1YUxkrPW5d7UZukWVHzAJ2Z1wPXTzDuGtCSVE+GhwqBdtLAu2fyYDzRjS1GxGjoLQXeVZvGBuNSCB4ztgKal8zqV5akSjgDLUkLWSYMHKx+Jvjg/dC7D5jiOpnm9nEh+MGHBuNDQvDKQhuEM8GS5jEDtCTNB6XZ4GpD8ME9MNQ3+XGjYWy4bV8Dq48dF4JXThyM7QOWtEgZoCWpVjILF7qVh92KQvBe6Ns79bFbOsaG2zXHVRCCVxZ6g10FQpKqYoCWpGoNDY5bDaKSFSOK24YHJj9uQ9PYcNuxDrpOnD4Ety2HppbZ/taSpCIDtKTFKbNwU4wJZ4InGivOBB+8v7BW8FRal41e9LZkBaw9cfq+4CUrCrPIzgZLUt0zQEtaGEp3kNu/A/bvgp4dsH/n6E9PcXz/TjiwuxCMhwcnP15D89hZ32UPhHUPnT4Ety2HxuZafGNJ0hwxQEuqX8NDhRnfCcPwzmIgLj7v2QmDByc+TttyWLoWlnZB1wnQvnrqELxkpXeIkyRNygAtqbYGeseG4TGzw2VheP9OOLBr4ptqRGMhDC/tgo6uwqoRI6/Xjj4v/dgfLEmaQQZoSUcms7C82iGzwxOF412F2zRPpHlpIQwv7SrcXW795kPDcOl12wrXEZYkzRkDtKRDDQ0U+oSn6iPeX/Z8qH+CgwS0ryq2TqyBo39n4jC8dE3hsWVpzb+mJEmHwwAtLRb9+ydvlRh/4d3B+yY+RmNLIRB3dBWWWFv3sNEA3FEMyqVe4/bV0Oh/YiRJC49/u0nz1fBw4QK7kTA8TTgeODDxcdqWj84Kd50AG886NAyXXrcu88I6SdKiZ4CW6slg38RheKJl2fbvghw69BjRWBZ+18Dqh0zQOlG23dsxS5JUFQN0JW76NHz+Twr/93Vjc/Fx/POmCcbLtjdMtL2SY5Q9b2ieYt+ybQ2Nc/0nppLMwkVzY2aDJ2ud2DX57ZqblxbCbsdaWLEBHvjwcWG4a3S2eMlKL7CTJGkWGaArseZ4ePQrCxdWDQ0ULpgaeZzg+WDvFO8pO8ZEs4czIRomD/OVhvAJ39Nc3H+qfwRMd4wJ/jEx31oChgYLF9hN10dc+pnyArvirPADThtdkq08DJdee4GdJEl1wwBdiaNPL/zMtOGhiQP58ODk4Xx8CJ/oPcMVhPzS68E+6Ouu7Biz5ZBQPl3Ir2RGf5pjTPYPgaGBsvA7SSA+cB+Qh36PxpaxrRLrThnXR1wWjr3ATpKkecu/wedSQ2Php7ltriuZXua4YD9NyB+eJuRPOZM/xTH6uqc4Rtl7Z2p2v3X5aOvEmuPgQWdOPlPsBXaSJC0KBmhVJmJ0Vpd50E4wPFzBTPwE/whoaBoNx+1r5sc/biRJUk0ZoLUwNTRAQ6srTEiSpBnnpfqSJElSFQzQkiRJUhUM0JIkSVIVDNCSJElSFQzQkiRJUhUM0JIkSVIVDNCSJElSFQzQkiRJUhUM0JIkSVIVDNCSJElSFSIz57qGqkTETuCuOfjoNcCuOfhc1Za/58XB3/Pi4O954fN3vDjM5e/5QZnZNX5w3gXouRIRWzJz81zXodnl73lx8Pe8OPh7Xvj8HS8O9fh7toVDkiRJqoIBWpIkSaqCAbpyV811AaoJf8+Lg7/nxcHf88Ln73hxqLvfsz3QkiRJUhWcgZYkSZKqYICWJEmSqmCAliRJkqpggNaiFhFNZc87ImJzRKyay5okHb6IWOU5LGm2GaC1aEXEhcC9EXF7RDwVuAn4B+CnEfH8OS1OUsUiYkNEfLJ4p9ofAD+MiB3FsY1zXJ6kGRIRN891DSVN079FWrBeB5wAdAI/BX4nM38ZEeuArwGfmMviJFXsU8A7gRdk5hBARDQCzwU+Cfzu3JUmqRoR8ezJNgFH1bKWqbiMnRatiLgxM08vPv9NZh5dtu2mzDx1zoqTVLGI+EVmHlftNkn1JyIGgI8BEwXU52RmZ41LmpAz0FrMtkXE31OYgb4tIv4R+BzwROC3c1qZpGrcEBHvAz4C/Lo4dgxwAfCTOatK0uG4CXhHZv5s/IaIeOIc1DMhZ6C1aEXEMuBPKfwr9z3A7wMvBu4C/i4zDdHSPBARLcBLgHOBBxaH7wa+CHwwM/vmqjZJ1YmIxwB3Zea2CbZtzswtc1DWIQzQkiRJUhVchUOLVkQ0RsTLI+ItEXHmuG1/NVd1SapORLRHxOUR8ecR0RYRF0TEFyPi7RHRMdf1SapcRDQV/26+NiJuKv58NSJeERHNc11fiTPQWrQi4gNAO/BD4IXAf2bma4vbfpyZZ8xlfZIqExGfptD7vITCyjq3UliZ4xnAUZn5wjksT1IVIuITwB4K1zRsLw6vp3BNw6rMPG+OShvDAK1Fq3yljeINVd4HrAGeD3w/M39nLuuTVJnSijoRERQuAH5AZmbx9U9dUUeaPyLi9sw8vtpttWYLhxazltKTzBzMzIuBG4FvAv7fvtI8k4UZoX8vPpZeO0skzS/3RcRzI2Iko0ZEQ0ScB9w/h3WNYYDWYrYlIp5SPpCZfwt8GNg4JxVJOhxbSr3OmXlRaTAiHgJ0z1lVkg7H84DnMHqn4NuBe4FnF7fVBVs4JEkLVkRE+hedNC9FxGqAzNw917WM5wy0VCYirprrGiQdudK5bHiW5q/M3J2Zu+vx72YDtDTW5rkuQNKM8FyWFo66O58N0NJYO+a6AEkzwnNZWjjq7ny2B1qSJEmqgjPQ0gTqsd9KUvU8l6X5Zb7cJdgZaC1aEbFqsk0Ubr6wvpb1SDo8nsvSwjFf7hJsgNaiFRFDwF0U/pItyeLrB2Zmy4Q7SqornsvSwjFf7hLcNNcFSHPoTuAJmblt/IaI+PUc1CPp8HguSwvHmLsEAxdHxP+mzu4SbA+0FrN3Aisn2fb2GtYh6ci8E89laaGYF3cJtoVDkiRJqoItHFrUIuJE4FzggcWhu4EvZuatc1eVpGp5LksLx3w4n23h0KIVEX8BfJLChUY/LP4E8ImIeP1c1iapcp7L0sIxX85nWzi0aEXE7cApmTkwbrwF+HlmHjc3lUmqhueytHDMl/PZGWgtZsPA0ROMP6C4TdL84LksLRzz4ny2B1qL2aXANyLiF0BpqasNwLHAK+eqKElVuxTPZWmhuJR5cD7bwqFFLSIagEcy9kKFH2Xm0NxVJalansvSwjEfzmcDtFQmIi7OzKvmug5JR8ZzWVo46vF8tgdaGusVc12ApBnhuSwtHHV3PhugpbFirguQNCM8l6WFo+7OZ1s4pDIRsT4zt891HZKOjOeytHDU4/nsDLRUpnSCRsSL57oWSYfPc1laOOrxfHYGWppARGzLzA1zXYekI+O5LC0c9XQ+uw60Fq2IuGmyTcC6WtYi6fB5LksLx3w5nw3QWszWAb8P3D9uPIDv1r4cSYfJc1laOObF+WyA1mL2ZaAjM28cvyEirq95NZIOl+eytHDMi/PZHmhJkiSpCq7CIUmSJFXBAK1FKyJOjYjvR8SvI+KqiFhZtu2Hc1mbpMp5LksLx3w5nw3QWszeB7wJeBhwO/DfEfGQ4rbmuSpKUtU8l6WFY16cz15EqMWsMzOvLT5/R0TcAFwbES8EvDhAmj88l6WFY16czwZoLWoRsTwz9wJk5rci4g+BzwKr5rYySdXwXJYWjvlwPtvCocXsH4CTygcy8ybgCcDn5qQiSYfDc1laOObF+ewydpIkSVIVnIHWohURyyPibRFxW0TcFxG7I+LW4tiKua5PUmU8l6WFY76czwZoLWafpnCr0LMzc1VmrgYeVxz79JxWJqkansvSwjEvzmdbOLRoRcT/ZOYJ1W6TVF88l6WFY76cz85AazG7KyIuj4h1pYGIWBcRfwH8eg7rklQdz2Vp4ZgX57MBWovZecBq4D8j4v6IuA+4nsIyOX80l4VJqornsrRwzIvz2RYOLWoRcSKwHvh+ZvaUjT+lbCF3SXXOc1laOObD+ewMtBatiHg18AXglcDPIuLcss1vnZuqJFXLc1laOObL+eydCLWYvQx4eGb2RMRG4DMRsTEz/wWIuS1NUhU8l6WFY16czwZoLWYNpf9rKDO3RsTZFE7UB1FHJ6mkaXkuSwvHvDifbeHQYnZvRJxeelE8Yc8B1gAPm6uiJFXNc1laOObF+exFhFq0ImI9MJiZ90yw7czM/M4clCWpSp7L0sIxX85nA7QkSZJUBVs4JEmSpCoYoCVJkqQqGKAlSZKkKhigJWmeiIjrI6IvIrojYm9E3BkRH42Ih1dxjK0Rcf5s1ilJC50BWpLml7dkZmdmLgceB9wFfD8injXHdUnSomGAlqR5KjPvysy/Aq4G3h0Fr4mI24qz1Nsi4u8johEgIr4EbAA+EBE9EXFdcbwpIt4YEbdHxJ6I+E5EbJ67byZJ9c0ALUnz3yeBBwInANuBpwLLgHOBi4CXAmTmHwDbgJdmZkdmPrm4/5uL730KsBr4EHBtRKys5ZeQpPnCAC1J89/24uPqzPxsZv4qC34CfBR4wmQ7RkQArwb+PDPvzMyhzPwg8Fvg6bNeuSTNQ01zXYAk6YitLz7ujojnA68FHkzhv/EtwPen2HcN0AF8KSLK76zVXHZcSVIZA7QkzX/nAXcD+4F/A54NfDUz+yPiHUB5P/PwuH13Ffd7Ymb+qBbFStJ8ZwuHJM1TEXFMRLwZuBB4DYWZ5AZgJzAQEb8LvHDcbvcAx5VeZGYC/wK8IyKOKx63IyJ+PyKOnv1vIUnzTxT+2ylJqncRcT3wKKAfSGA38F3gXzLzh8X3/G/gVRRaN74FbAVOz8yzi9ufBrwbWAV8PzOfGhFNFPqgX0qhbWM/hbaPV2Vmqb9aklRkgJYkSZKqYAuHJEmSVAUDtCRJklQFA7QkSZJUBQO0JEmSVAUDtCRJklQFA7QkSZJUBQO0JEmSVAUDtCRJklQFA7QkSZJUhf8PPso4sBs0Q7AAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "get_daily_graph(click_share_direction_daily, \n", - " 'Date', 'Accepted Share (%)', **{'shrinkTicks':True, 'hue':'pyDirection'})" + "get_daily_graph(\n", + " click_share_direction_daily,\n", + " \"Date\",\n", + " \"Accepted Share (%)\",\n", + " **{\"shrinkTicks\": True, \"hue\": \"pyDirection\"},\n", + ")" ] }, { @@ -946,22 +389,14 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/QAAAG6CAYAAACm6A1XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABJSklEQVR4nO3dd5hsVZm28fsh53wMoMCAIAOOGBgDBsyKihkHARF0xog4oo4JEXPOGQcFUTEiioyOCVQExg9UFARBJAclwwFEOLzfH3s3FE13dZ1DVe+u7vt3XXV17VC7nqpTVafeWmuvlapCkiRJkiSNl+W6DiBJkiRJkpaeBb0kSZIkSWPIgl6SJEmSpDFkQS9JkiRJ0hiyoJckSZIkaQxZ0EuSJEmSNIYs6CVpHkmyW5IfdZ1DkiRJo2dBL0lTSLJrkhOTLE5ycZIfJHl417lmUlVfqaonDPu4SVZOclCSc5Ncm+R3SXactM9jk5ye5PokRyfZpGfbB5Oc2d729CR7TLrtgUn+lOSWJHsOkOd+SU5q7+ukJPfr2fbo9v6vTnLOAMd6XZJT2mxnJ3ndpO3bJ/l1u/33/V4HSQ5I8uWZ7vNOZNm0fWzXt8/j43q2vaB9Lq5JckGS9ydZoWf73u1r+sYkBw+QZb0k30lyXfvvvmvPtqckOTbJVUkuSfLfSdac4Xi7tse5LskRSdZb1mx3VpK7JDksyUXt6+RXSR48SN6Z3gtJVkryrSTnJKkkjxogT7/3znOTHNduO2aAY830Xqv2MS1uL//d51jHJPn3me5zmtsu5M+M+yX5ZXu8C5K8ZaZjStKysqCXpEmS7At8FHg3cFdgY+DTwNM7jDWj3uJtBFYAzgd2ANYG9gO+kWTT9r43AA4H3gKsB5wIfL3n9tcBO7W3fQHwsSTb92w/GXg58JuZgiRZCfgu8GVgXeAQ4Lvt+on7+gLwuqmPcMdDAnu0x3oSsHeSXdr7Wg84EvgAsA7wfuDIJOsOeOylNW2W1mHAb4H1gTcD30qyqN22GvCfwAbAg4HHAq/tue1FwDtpnptBfAr4B817YDfgM0m2abet3R5rQ+CfgY1onqOpH1Rzu88Bz2+Pdz3Ne2pZs91ZawD/D3ggzev1EOCoJGsMkLfve6F1LLA7cMlMQQZ471xB83n03gEf20zvNYBtq2qN9rJMBfsAFuRnRuurwC/ax7UD8PIkTxvw2JK0dKrKixcvXry0F5ovj4uBnfvsszLNF+yL2stHgZXbbY8CLgD+C/gbcDHwDODJwBk0X87f1HOsA4Bv0XyRvZbmy+m2PdvfAJzVbvsj8MyebXsCvwI+AlxOUxDtCRzbs08BLwXOBK6iKdLSblse+BBwGXA2sHe7/woDPle/B57dXn8xcFzPttWBG4Ctprnt94DXTLH+WGDPGe73CcCFE4+jXXce8KRJ+z0OOGcZXgMfBz7RXn8qcOqk7WcAL5ridk+iKYBval9DJ7frN2wf7xXAn4H/WMYsWwI3Amv2bP8l8NJpbrsvcOQU698JHDzD/a7ePpYte9YdCrx3mv2fBfyhz/HeDXy1Z3nz9vhrLk02mvfeVcB9etYtal9rd6H5MeP77T5XtM/PcgM+19cAD1yavFO9FyatvwB41Az3O9B7B/h34JhleD3f7r1G8x6/1wC3exewBPh7+3r+ZLt+e5ofQ65u/26/FFnm/WdGu3w9sHXP8jeBNy7tcb148eJlkIst9JJ0ew8FVgG+02efNwMPAe4HbAs8iKb1acLd2mNsBOwPfJ6mte6BwCOAtyT5p579n07zhW89mpadI5Ks2G47q73N2sDbgC8nuXvPbR8M/IWmFfFd0+R9KvCvwH2B5wJPbNf/B7Bj+zgeQPPDw0CS3JWmwDy1XbUNTYsZAFV1XZt9myluu2qb59TJ2wa0DfD7qqqedb+f6r6WVpLQPN+92TJ5N+A+k29bVT+kKQS/Xk3L57btpq/RFHYbAs8B3p3kMcuQZRvgL1V1bc9uJzP9434ky/4cbwncXFVnDOm+Jr8+zqL9wWBpQlXVjTStus/rWf1c4OdV9TfgNTTP9SKa98SbaArYvtru1yvR/OCyVHmneC8srYHfO0urz3vtF2lOlTh8Us+CW1XVm2l+ENm7fT3v3fZYOYqmgF0f+DBNz4b1B8iykD4zPgrskWTFJPem+X/lJ3f2viRpKhb0knR76wOXVdXNffbZDXh7Vf2tqi6lKbSf37P9JuBdVXUTTTG3AfCxqrq2qk6laWnftmf/k6rqW+3+H6b5MeAhAFX1zaq6qKpuqaqv07S0P6jnthdV1Seq6uaqumGavO+tqquq6jzgaJoCHppC6GNVdUFVXcmAXXrbHxu+AhxSVae3q9egabHrdTUw1XnVn6X5Iv+/g9zfFJbmvpbWATT/N36xXT4e2DDJ89ov5y+gaa1dbZCDJbkn8DDg9VX196r6HfDfNN11lzbLwI87yQuB7YAPDpJzCmvQtFgPcl+Pp+kSvf8MxxvWv9lXgd7uzbu266B5790d2KSqbqqqX04q4u4gyVo0vQ/eVlUTGQfKO817YWmN8vU81XttB2BTYCuaHkbfX4rTdZ4CnFlVh7afOYcBp9N0jZ/WAvvMgKaXyHNoehycDhxUVf9vCPclSXdgQS9Jt3c5sMEMX3A3BM7tWT63XXfrMapqSXt9osj+a8/2G2i+YE44f+JKVd3Cba25JNmjHUzqqiRX0bQMbzDVbfvoPY/3+p773nDS7Wc8VpLlaIqff9B00Z+wGFhr0u5r0Zwq0Hv7D9A8hufOVGj13GZxz2XjQe9rmmO9qedYn520bW+aQvspbUswVXU5TQ+KfWn+DZ9E09J2wSDZaZ7jKya1qp9L03ujX847ZGHw5/gZwHuAHavqskFCphn0ceJ52W0p7ushNMX0cyZa85M8oudYE62Wy/xvNoWjgdWSPLhtXb4ft/Wo+QBNK/uPkvwlyRv6Haht+T0SOKGq3tOzaca8fd4L/e5v497X86D31ed4n+053psmbZvyvVZVv6iqf1TVVcCrgH+iGQdhEJM/+2CG1/NC+8xoezH8EHg7zY+z9wSemOTlg2SXpKVlQS9Jt3c8zXnKz+izz0XAJj3LG7frltU9J660X37vAVyUZsTnz9N8CV6/qtYBTuH2XcAH+oI7jYvb+7pDjqm0XUsPounK/Oy2R8GEU+npdZBkdZqW7FN71r2Npov/E6pqcuvvtOq2wbvWaHsZnArct80z4b4M0B23qt7dc6yX9mR7Ic14BY+tqgsm3ebnVfWvVbUeTU+MrYBfT3cXk5YvAtbL7UeA35jmfN4p9clyKrDZpGNty+2f4yfRvGZ2qqo/THcfdwhdtWPP8/IVmnECVkiyRZ/7uj/Nec0vrKqf9hzrlz3HmujSPPn1sRnN+fC9XfoHzboE+AZNt/vnAd+f+MGk7QXzmqraDHgasG+Sx051nCQrA0fQ/Djzkkmb++ad4b3QL/t5va/nae7rDu+dPsd7ac/x3t1zjKV5rxV3PK2kd1uvyZ990Of1vEA/MzYDllTVl9peDBfQ9NR68qD5JWlpWNBLUo+2y+3+wKeSPCPJam1X6x2TvL/d7TBgvySL0ozUvD/N6MnL6oFJntX2CvhPmh8UTqAZJKqASwGS7MUU527fCd8AXpVkoyTrAK+fYf/P0LTk7TRF9/7vAPdJ8uwkq9A8J7+f6F6b5I00XaMf17Z6306aqb5WoSksVkyySvvjxlSOoRmsa580U2NNtPr9rD3Wcu2xVmwWs0puG836DtoW6XcDj6+qv0yx/f7ta2Atmi7s51fVdF1//wpsOpG9qs4HjgPe0+a4L/Aipnm99MvStoD/Dnhre6xn0hQl325v+xiabs3Prqo7/OCQZIX2eVkeWL49xpQ9UdrzmQ8H3p5k9SQPo+mpcGh7rPvQtEK+sqqOnOa56PUVYKe29X51mtbLwycK8aXJ1voq8G80p79MdLcnyVOT3Kst3K6meZ3cMsVzsSLNYJQ3AC9oe8YMnJf+74WJKdtWaRdXah/PdEXzTO+d5dv1KwDLtcdacZpj9X2vJdkmzZRqy6cZ0f9DNMX4adMc7q80BeqE/wG2TDOl3wpJ/g3YmqaL+VQW4mfGGe0xdm2Pezea1+rvpzueJN0pNQdG5vPixYuXuXahKRROpJnO6BKagaC2b7etQjMo1MXt5ePAKu22RwEX9BxnBZqifNOedccCu7fXD+D2o9z/FnhAz77vohmt+zKa8+t/Dvx7u21Peka0n2odk0a0Bg4G3tmTbWKE/LOBV9Ocg5wpno9N2mNNjHg9cdmtZ5/H0ZwvegPNF+hNJ+W4cdJte0f7P6bdp/fyqD7/PvcHTmrv6zfA/Xu2PWqKYx3T51hnc9vI9BOXz/ZsP4ymOLy6/Xe6S59jrd/++14J/KZddw+agucKmkG/phyVfsAsm7bP1Q3An2iKnYltRwM3T7rtD3q2HzDF83JAnyzr0bRgX0czIviuPdu+SFMo997XqdMdq73Nru1xrqOZQmy9Zc3W3ubP7XO6Us+6VwPntPdxAfCWaW67Q3sf1096DI+YKS+DvRfOmeLxbNrnsfR77+w5xbEO7nOsad9rwGPa1811NLNwHAFs0edYD6UpUK8EPt6uezjNe+/q9u/Dp7ntQv7MeAy3zQRwCU2vmdX6vZ69ePHiZVkvE1MXSZI6kOQAmoJ79zmQZUeaL6WTu9RKkiRpDrLLvSQtUElWTfLktuvsRsBb6T9dnyRJkuYQC3pJWrhCM+XelTRd/U+j/9RjkiRJmkPsci9JkiRJ0hiyhV6SJEmSpDFkQS9JWlCSHJPk3wfct5Lca9SZxlWSc5I8ruscM0myZ5Jj+2wf+DWxjPc/Fs+TJGn8WNBLkjSAJAcneWfXOUZlFopafxwZgM+TJGlpWNBLkjSHJFmh6wwajP9WkqSuWdBLkua1JI9PcnqSq5N8kmZ0/97tL0xyWpIrk/xvkk2mOMaLgd2A/0qyOMmR7fo3JDkrybVJ/pjkmcuQb88kv0rykSSXAwckWTnJB5Ocl+SvST6bZNWe2zw9ye+SXNPe/5Pa9WsnOSjJxUkuTPLOJMv33M+x7XGvTHJ2kh3bbe8CHgF8sn18n5wm6/OTnJvk8iRvnrTtQUmOT3JVe/+fTLJSu+0X7W4nt8f/tyTrJvl+kkvbPN9Pco+lff567n/tJF9qj3dukv2STPk95868JtoW9FckORM4c9yeJ0nS/GJBL0mat5JsABwO7AdsAJwFPKxn+9OBNwHPAhYBvwQOm3ycqjoQ+Arw/qpao6p2ajedRVMIr00zBeCXk9y9PfbD26JtusvDe+7iwcBfgLsC7wLeC2wJ3A+4F7AR7ZSCSR4EfAl4HbAO8EjgnPY4BwM3t7e5P/AEoLcb/YOBP7XPxfuBg5Kkqt7cPva928e39xTP5dbAZ4DnAxsC6wO9heUS4NXtsR8KPBZ4efv8PbLdZ9v2+F+n+Q7yRWATYGPgBmDKHxIG9Amaf4fNgB2APYC9pngcw3hNPIPmudx6iuPP9edJkjSPWNBLkuazJwOnVtW3quom4KPAJT3bXwq8p6pOq6qbgXcD95uqlX4qVfXNqrqoqm5pi68zgQe1246tqnX6XHoHabuoqj7RZvg78GLg1VV1RVVd2+bapd33RcAXqurH7f1eWFWnJ7lr+3j/s6quq6q/AR/puR3AuVX1+apaAhwC3J3mR4RBPAf4flX9oqpuBN4C3NLzXJxUVSdU1c1VdQ7wOZrCerrn7vKq+nZVXd8+xnf127+fthfCLsAbq+ra9v4/RFNUTzaM18R72n+bG6Y4/px9niRJ848FvSRpPtsQOH9ioaqqd5mm1fNjE63mwBU03a83GuTgSfZou75P3P4+NC2vS6s30yJgNeCknuP+sF0PcE+aVuXJNgFWBC7uud3ngLv07HNr4VpV17dX1xgw4+Tn8jrg8onlJFu23cEvSXINTSE87XORZLUkn2u7pl8D/AJYZ+IUgUn77tZ2QV+c5AdTHG4Dmsd+bs+6c5n633EYr4ne/Wc6/qw9T5KkhceCXpI0n11MUwADkCS9yzSF10smtZyvWlXHTXGs6l1oW2w/D+wNrF9V6wCn0J6PneQRPUXoVJdHTHPsy2i6VW/Tk2ntqpoovM8HNp8i3/nAjcAGPbdbq6q2melJmurxTWHyc7kaTXfyCZ8BTge2qKq1aLqt3+7c9EleA9wbeHC7/0R38zvcpqq+0nZBX6OqdpziWJcBN9EU4xM2Bi4c4HEsy2ui33PV2fMkSVp4LOglSfPZUcA2SZ6VZkTyfYC79Wz/LPDGJNvArQOr7TzNsf5Kc372hNVpCrtL29vuRdNCD0BV/bKnCJ3q8sup7qSqbqH5oeAjSe7SHnujJE9sdzkI2CvJY5Ms127bqqouBn4EfCjJWu22zZMM2j178uOb7FvAU9uxAVYC3s7tv0esCVwDLE6yFfCyGY6/Js0PF1clWQ9464A576A9heAbwLuSrNn+2LIv8OUpdh/ma2Iqc/Z5kiTNPxb0kqR5q6ouA3amGWTucmAL4Fc9278DvA/4Wtud+RRgqhZgaArprduu2EdU1R9pztM+nqYI+5feY99Jrwf+DJzQ5voJTSstVfVrmsHePgJcDfyc21qm9wBWAv4IXElTXN59wPv8GPCcdiT1j0/eWFWnAq8AvkrTCn0lcEHPLq8FdgWupflB4uuTDnEAcEj7/D2X5tz1VWla10+gOa3gznglcB3N4ILHtjm/MMXjGOZr4g7G4HmSJM0jaU4dkyRJkiRJ48QWekmSJEmSxpAFvSRJkiRJY8iCXpIkSZKkMWRBL0mSJEnSGLKglyRJkiRpDK3QdYDZsMEGG9Smm27adQxJkiRJkpbaSSeddFlVLZq8fkEU9Jtuuiknnnhi1zEkSZIkSVpqSc6dar1d7iVJkiRJGkMW9JIkSZIkjSELekmSJEmSxpAFvSRJkiRJY8iCXpIkSZKkMWRBL0mSJEnSGLKglyRJkiRpDFnQS5IkSZI0hizoJUmSJEkaQxb0kiRJkiSNoRW6DgCQZG9gT+BfgMOqas8++74aeD2wGvAt4GVVdeMsxATgSW//5mzdlRagH+6/c9cRJEmSJI2JudJCfxHwTuAL/XZK8kTgDcBjgU2AzYC3jTydJEmSJElzzJwo6Kvq8Ko6Arh8hl1fABxUVadW1ZXAO2ha9iVJkiRJWlDmREG/FLYBTu5ZPhm4a5L1O8ojSZIkSVInxq2gXwO4umd54vqak3dM8uIkJyY58dJLL52VcJIkSZIkzZZxK+gXA2v1LE9cv3byjlV1YFVtV1XbLVq0aFbCSZIkSZI0W8atoD8V2LZneVvgr1U107n3kiRJkiTNK3OioE+yQpJVgOWB5ZOskmSqKfW+BLwoydZJ1gH2Aw6evaSSJEmSJM0Nc6KgpynMb6CZkm739vp+STZOsjjJxgBV9UPg/cDRwHnAucBbu4ksSZIkSVJ3pmoFn3VVdQBwwDSb15i074eBD484kiRJkiRJc9pcaaGXJEmSJElLwYJekiRJkqQxZEEvSZIkSdIYsqCXJEmSJGkMWdBLkiRJkjSGLOglSZIkSRpDFvSSJEmSJI0hC3pJkiRJksaQBb0kSZIkSWPIgl6SJEmSpDFkQS9JkiRJ0hiyoJckSZIkaQxZ0EuSJEmSNIYs6CVJkiRJGkMW9JIkSZIkjaEVug4gae474jdndR1B89gzHrB51xEkSZLGki30kiRJkiSNIQt6SZIkSZLGkAW9JEmSJEljyIJekiRJkqQxZEEvSZIkSdIYGrigT7J6kuVHGUaSJEmSJA1m2oI+yXJJdk1yVJK/AacDFyf5Y5IPJLnX7MWUJEmSJEm9+rXQHw1sDrwRuFtV3bOq7gI8HDgBeF+S3WchoyRJkiRJmmSFPtseV1U3TV5ZVVcA3wa+nWTFkSWTJEmSJEnTmragn1zMJ1kF2B1YFfhqVV0+VcEvSZIkSZJGb2lGuf8Y8A/gSuCIkaSRJEmSJEkD6Tco3mFJNu9ZtR7wTZru9uuOOpgkSZIkSZpev3Po3wy8M8nFwDuADwLfAVYBDhh9NEmSJEmSNJ1+59D/Bdg1ycOBrwNHAU+pqiWzFU6SJEmSJE2tX5f7dZO8Atga2Jnm3Pn/TbLTbIWTJEmSJElT6zco3hHAVUABh1bVocBOwP2THDn6aJIkSZIkaTr9zqFfH/gWzTR1LwGoqhuAtye5+yxkkyRJkiRJ0+hX0L8V+CGwBHhD74aquniUoSRJkiRJUn/9BsX7Ns0UdZIkSZIkaY7pNyje55PcZ5ptqyd5YZLdRhdNkiRJkiRNp1+X+08B+yf5F+AU4FKaOei3ANYCvgB8ZeQJJUmSJEnSHfTrcv874LlJ1gC2A+4O3ACcVlV/mp14kiRJkiRpKv1a6AGoqsXAMaOPIkmSJEmSBtVvHnpJkiRJkjRHWdBLkiRJkjSGBi7ok6w2yiCSJEmSJGlwMxb0SbZP8kfg9HZ52ySfHnkySZIkSZI0rUFa6D8CPBG4HKCqTgYeOcpQkiRJkiSpv4G63FfV+ZNWLRlBFkmSJEmSNKAZp60Dzk+yPVBJVgReBZw22liSJEmSJKmfQVroXwq8AtgIuBC4X7s8NEnWS/KdJNclOTfJrtPsd0CSm5Is7rlsNswskiRJkiSNgxlb6KvqMmC3Eef4FPAP4K40PxgcleTkqjp1in2/XlW7jziPJEmSJElz2iCj3B+SZJ2e5XWTfGFYAZKsDjwbeEtVLa6qY4HvAc8f1n1IkiRJkjTfDNLl/r5VddXEQlVdCdx/iBm2BG6uqjN61p0MbDPN/jsluSLJqUleNsQckiRJkiSNjUEK+uWSrDuxkGQ9BhtMb1BrANdMWnc1sOYU+34D+GdgEfAfwP5JnjfVQZO8OMmJSU689NJLhxhXkiRJkqTuDVLQfwg4Psk7krwTOA54/xAzLAbWmrRuLeDayTtW1R+r6qKqWlJVxwEfA54z1UGr6sCq2q6qtlu0aNEQ40qSJEmS1L1BBsX7UpKTgEe3q55VVX8cYoYzgBWSbFFVZ7brtgWmGhDvDvGADDGLJEmSJEljYZAWeoDTgcNpBqtbnGTjYQWoquvaY789yepJHgY8HTh08r5Jnt4OypckDwL2Ab47rCySJEmSJI2LGVvok7wSeCvwV2AJTYt4AfcdYo6XA18A/gZcDrysqk5N8gjgB1W1RrvfLu1+KwMXAO+rqkOGmEOSJEmSpLEwyOB2rwLuXVWXjypEVV0BPGOK9b+kGTRvYnnKAfAkSZIkSVpoBulyfz7NqPOSJEmSJGmOGKSF/i/AMUmOAm6cWFlVHx5ZKkmSJEmS1NcgBf157WWl9iJJkiRJkjo2yLR1bwNIslpVXT/6SJIkSZIkaSYznkOf5KFJ/kgzdR1Jtk3y6ZEnkyRJkiRJ0xpkULyPAk+kmU6OqjoZeOQIM0mSJEmSpBkMUtBTVedPWrVkBFkkSZIkSdKABhkU7/wk2wOVZEWaeelPG20sSZIkSZLUzyAt9C8FXgFsBFwI3K9dliRJkiRJHenbQp9keeBjVbXbLOWRJEmSJEkD6NtCX1VLgE2SOP+8JEmSJElzyCDn0P8F+FWS7wHXTaysqg+PLJUkSZIkSeprkIL+rPayHLDmaONIkiRJkqRBzFjQV9XbAJKsVlXXjz6SJEmSJEmayYyj3Cd5aJI/Aqe3y9sm+fTIk0mSJEmSpGkNMm3dR4EnApcDVNXJwCNHmEmSJEmSJM1gkIKeqjp/0qolI8giSZIkSZIGNMigeOcn2R6oJCsCrwJOG20sSZIkSZLUzyAt9C8FXgFsBFwI3K9dliRJkiRJHZm2hT7J+6rq9cCjq2q3WcwkSZIkSZJm0K+F/slJArxxtsJIkiRJkqTB9DuH/ofAlcAaSa4BAtTE36paaxbySZIkSZKkKfRrod+vqtYBjqqqtapqzd6/s5RPkiRJkiRNoV9Bf3z795rZCCJJkiRJkgbXr8v9Skl2BbZP8qzJG6vq8NHFkiRJkiRJ/fQr6F8K7AasA+w0aVsBFvSSJEmSJHVk2oK+qo4Fjk1yYlUdNIuZJEmSJEnSDPrNQ/+YqvoZcKVd7iVJkiRJmlv6dbnfAfgZd+xuD3a5lyRJkiSpU/263L+1/bvX7MWRJEmSJEmD6NdCT5J7Ay8GtmpXnQYcWFVnjDqYJEmSJEma3rTz0Cd5KHAMsBg4EPg8cB1wTJKHzEo6SZIkSZI0pX4t9PsDz6uqY3rWHZHkZ8BbgR1HGUySJEmSJE1v2hZ6YPNJxTwAVfVzYLORJZIkSZIkSTPqV9Bf22fbdcMOIkmSJEmSBtevy/09k3x8ivUBNhpRHkmSJEmSNIB+Bf3r+mw7cdhBJEmSJEnS4PrNQ3/IbAaRJEmSJEmD63cOvSRJkiRJmqMs6CVJkiRJGkMW9JIkSZIkjaEZC/okWyb5aZJT2uX7Jtlv9NEkSZIkSdJ0Bmmh/zzwRuAmgKr6PbDLKENJkiRJkqT+BinoV6uqX09ad/MowkiSJEmSpMEMUtBflmRzoACSPAe4eKSpJEmSJElSX9POQ9/jFcCBwFZJLgTOBnYbaSpJkiRJktRX3xb6JMsDL6+qxwGLgK2q6uFVde4wQyRZL8l3klyX5Nwku06zX5K8L8nl7eV9STLMLJIkSZIkjYO+LfRVtSTJw9vr140wx6eAfwB3Be4HHJXk5Ko6ddJ+LwaeAWxLcwrAj2l6DHx2hNkkSZIkSZpzBuly/9sk3wO+Cdxa1FfV4cMIkGR14NnAfapqMXBse3/PB94wafcXAB+qqgva234I+A8s6CVJkiRJC8wgBf0qwOXAY3rWFTCUgh7YEri5qs7oWXcysMMU+27Tbuvdb5sh5ZAkSZIkaWzMWNBX1V4jzrAGcM2kdVcDa06z79WT9lsjSaqqendM8mKaLvpsvPHGQwv7w/13HtqxpHHxjAds3nUEaVb9/YYbuo6geWyVVVftOsKULvvJV7uOoHlsg8dNOURW5/73357ddQTNY0/8+rdHfh8zFvRJVgFeRNMSvsrE+qp64ZAyLAbWmrRuLeDaAfZdC1g8uZhv8x1IMzo/22233R22S5IkSZI0zgaZh/5Q4G7AE4GfA/dg6mJ7WZ0BrJBki5512wKTB8SjXbftAPtJkiRJkjSvDVLQ36uq3gJcV1WHAE8BHjysAO3o+YcDb0+yepKHAU+n+SFhsi8B+ybZKMmGwGuAg4eVRZIkSZKkcTFIQX9T+/eqJPcB1gbuMuQcLwdWBf4GHAa8rKpOTfKIJIt79vsccCTwB+AU4Kh2nSRJkiRJC8ogo9wfmGRd4C3A92gGptt/mCGq6gqa+eUnr/9le38TywX8V3uRJEmSJGnBGmSU+/9ur/4c2Gy0cSRJkiRJ0iAGGeV+ZeDZwKa9+1fV20cXS5IkSZIk9TNIl/vv0sz3fhJw42jjSJIkSZKkQQxS0N+jqp408iSSJEmSJGlgg4xyf1ySfxl5EkmSJEmSNLBpW+iT/AGodp+9kvyFpst9aAacv+/sRJQkSZIkSZP163L/1FlLIUmSJEmSlkq/gv5S4Kaqugkgyb2BJwPnVtXhsxFOkiRJkiRNrd859D+kmaqOJPcCjqeZh/4VSd4z+miSJEmSJGk6/Qr6davqzPb6C4DDquqVwI7YHV+SJEmSpE71K+ir5/pjgB8DVNU/gFtGGUqSJEmSJPXX7xz63yf5IHAhcC/gRwBJ1pmFXJIkSZIkqY9+LfT/AVxGcx79E6rq+nb91sAHR5xLkiRJkiT1MW0LfVXdALx3ivXHAceNMpQkSZIkSeqvX5d7SZIWpFVWXbXrCJIkSTPq1+VekiRJkiTNURb0kiRJkiSNoWm73Cc5kttPXXc7VfW0kSSSJEmSJEkz6ncO/cRI9s8C7gZ8uV1+HvDXUYaSJEmSJEn99Rvl/ucAST5UVdv1bDoyyYkjTyZJkiRJkqY1yDn0qyfZbGIhyT8Bq48ukiRJkiRJmskg09a9GjgmyV+AAJsALxlpKkmSJEmS1NeMBX1V/TDJFsBW7arTq+rG0caSJEmSJEn9zNjlPslqwOuAvavqZGDjJE8deTJJkiRJkjStQc6h/yLwD+Ch7fKFwDtHlkiSJEmSJM1okIJ+86p6P3ATQFVdT3MuvSRJkiRJ6sggBf0/kqwKFECSzQHPoZckSZIkqUODjHL/VuCHwD2TfAV4GLDnKENJkiRJkqT+BinoTwKeBTyEpqv9q4A1RxlKkiRJkiT1N0iX+yOBm6rqqKr6PrCoXSdJkiRJkjoySEH/buDIJKsneSDwLWD30caSJEmSJEn9zNjlvqqOSrIi8GOarvbPrKozRp5MkiRJkiRNa9qCPsknaEe2b60NnAXsnYSq2mfU4SRJkiRJ0tT6tdCfOGn5pFEGkSRJkiRJg5u2oK+qQwCSrA78vaqWtMvLAyvPTjxJkiRJkjSVQQbF+ymwas/yqsBPRhNHkiRJkiQNYpCCfpWqWjyx0F5fbXSRJEmSJEnSTAYp6K9L8oCJhXbquhtGF0mSJEmSJM1kxmnrgP8EvpnkIiDA3YB/G2UoSZIkSZLU3yDz0P+/JFsB925X/amqbhptLEmSJEmS1E+/eegfU1U/S/KsSZu2bOehP3zE2SRJkiRJ0jT6tdDvAPwM2GmKbQVY0EuSJEmS1JF+89C/tf271+zFkSRJkiRJg+jX5X7ffjesqg8PP44kSZIkSRpEvy73a85aCkmSJEmStFT6dbl/22wGkSRJkiRJg1tuug1JPpDkJVOsf0mS9w4rQJL1knwnyXVJzk2ya599D0hyU5LFPZfNhpVFkiRJkqRx0a/L/WOA/5pi/eeB3wNvGFKGTwH/AO4K3A84KsnJVXXqNPt/vap2H9J9S5IkCdjgcdO2qUiS5qhpW+iBlauqJq+sqluADOPOk6wOPBt4S1Utrqpjge8Bzx/G8SVJkiRJmq/6FfQ3JNli8sp23Q1Duv8tgZur6oyedScD2/S5zU5JrkhyapKXDSmHJEmSJEljpV+X+/2BHyR5J3BSu2474I3Afw7p/tcArpm07mqmH2H/G8CBwF+BBwPfTnJVVR02ecckLwZeDLDxxhsPKa4kSZIkSXPDtC30VfUD4BnAo4GD28ujgGdX1f8McvAkxySpaS7HAouBtSbdbC3g2mky/bGqLqqqJVV1HPAx4DnT7HtgVW1XVdstWrRokLiSJEmSJI2Nfi30VNUpwAuW9eBV9ah+29tz6FdIskVVndmu3haYbkC8O9wFQzqfX5IkSZKkcdLvHPqRq6rrgMOBtydZPcnDgKcDh061f5KnJ1k3jQcB+wDfnb3EkiRJkiTNDZ0W9K2XA6sCfwMOA142MWVdkkckWdyz7y7An2m65H8JeF9VHTLLeSVJkiRJ6lzfLvezoaquoDlXf6ptv6QZOG9i+XmzFEuSJEmSpDltxhb6JFsm+WmSU9rl+ybZb/TRJEmSJEnSdAbpcv95mqnqbgKoqt/TdH2XJEmSJEkdGaSgX62qfj1p3c2jCCNJkiRJkgYzSEF/WZLNaaaII8lzgItHmkqSJEmSJPU1yKB4rwAOBLZKciFwNrD7SFNJkiRJkqS+Zizoq+ovwOOSrA4sV1XXjj6WJEmSJEnqZ8aCPsk6wB7ApsAKSQCoqn1GGUySJEmSJE1vkC73/wOcAPwBuGW0cSRJkiRJ0iAGKehXqap9R55EkiRJkiQNbJBR7g9N8h9J7p5kvYnLyJNJkiRJkqRpDdJC/w/gA8Cbaaeua/9uNqpQkiRJkiSpv0EK+tcA96qqy0YdRpIkSZIkDWaQLvd/Bq4fdRBJkiRJkjS4QVrorwN+l+Ro4MaJlU5bJ0mSJElSdwYp6I9oL5IkSZIkaY6YsaCvqkNmI4gkSZIkSRrcjAV9ki2A9wBbA6tMrK8qR7mXJEmSJKkjgwyK90XgM8DNwKOBLwFfHmUoSZIkSZLU3yAF/apV9VMgVXVuVR0APGW0sSRJkiRJUj+DDIp3Y5LlgDOT7A1cCKwx2liSJEmSJKmfQVroXwWsBuwDPBB4PvCCUYaSJEmSJEn9DTLK/f9rry4G9hptHEmSJEmSNIhpC/okXwRqms1VVS8aTSRJkiRJkjSTfi30359i3T2BVwPLjyaOJEmSJEkaxLQFfVV9e+J6ks2ANwGPBN4LHDT6aJIkSZIkaTp9B8VLslWSLwNHAscCW1fVZ6rqH7OSTpIkSZIkTanfOfTfpBnV/kM03eyXAGslAaCqrpiNgJIkSZIk6Y76nUP/rzSD4r0WeE27Lu3fAjYbYS5JkiRJktRHv3PoN53FHJIkSZIkaSn0PYceIMm3kzw5yYz7SpIkSZKk2TFIkf4ZYDfgzCTvTXLvEWeSJEmSJEkzmLGgr6qfVNVuwAOAc4CfJDkuyV5JVhx1QEmSJEmSdEcDdaNPsj6wJ/DvwG+Bj9EU+D8eWTJJkiRJkjStfqPcA5DkO8C9gUOBnarq4nbT15OcOMpwkiRJkiRpajMW9MDHq+roqTZU1XZDziNJkiRJkgYwSEF/fJJ9gYfTzD9/LPCZqvr7SJNJkiRJkqRpDVLQfwm4FvhEu7wrTff7nUcVSpIkSZIk9TdIQX+fqtq6Z/noJH8cVSBJkiRJkjSzQUa5/02Sh0wsJHkw4GB4kiRJkiR1aJAW+gcCxyU5r13eGPhTkj8AVVX3HVk6SZIkSZI0pUEK+ieNPIUkSZIkSVoqMxb0VXXubASRJEmSJEmDG+QcekmSJEmSNMdY0EuSJEmSNIYs6CVJkiRJGkMW9JIkSZIkjaFOC/okeyc5McmNSQ4eYP9XJ7kkyTVJvpBk5VmIKUmSJEnSnNN1C/1FwDuBL8y0Y5InAm8AHgtsAmwGvG2k6SRJkiRJmqM6Leir6vCqOgK4fIDdXwAcVFWnVtWVwDuAPUcYT5IkSZKkOavrFvqlsQ1wcs/yycBdk6zfUR5JkiRJkjozTgX9GsDVPcsT19ecauckL27Pzz/x0ksvHXk4SZIkSZJm08gK+iTHJKlpLscuwyEXA2v1LE9cv3aqnavqwKrarqq2W7Ro0TLcnSRJkiRJc9cKozpwVT1qyIc8FdgW+Ea7vC3w16oa5Px7SZIkSZLmla6nrVshySrA8sDySVZJMt2PDF8CXpRk6yTrAPsBB89OUkmSJEmS5pauz6HfD7iBZjq63dvr+wEk2TjJ4iQbA1TVD4H3A0cD5wHnAm/tIrQkSZIkSV0bWZf7QVTVAcAB02w7j2YgvN51HwY+PPJgkiRJkiTNcV230EuSJEmSpGVgQS9JkiRJ0hiyoJckSZIkaQxZ0EuSJEmSNIYs6CVJkiRJGkMW9JIkSZIkjSELekmSJEmSxpAFvSRJkiRJY8iCXpIkSZKkMWRBL0mSJEnSGLKglyRJkiRpDFnQS5IkSZI0hizoJUmSJEkaQxb0kiRJkiSNoRW6DiBJkiRJXXji17/ddQTpTrGFXpIkSZKkMWRBL0mSJEnSGLKglyRJkiRpDFnQS5IkSZI0hizoJUmSJEkaQxb0kiRJkiSNIQt6SZIkSZLGkAW9JEmSJEljyIJekiRJkqQxZEEvSZIkSdIYsqCXJEmSJGkMpaq6zjBySS4Fzu06xwK1AXBZ1yGkWebrXguNr3ktRL7utRD5uu/OJlW1aPLKBVHQqztJTqyq7brOIc0mX/daaHzNayHyda+FyNf93GOXe0mSJEmSxpAFvSRJkiRJY8iCXqN2YNcBpA74utdC42teC5Gvey1Evu7nGM+hlyRJkiRpDNlCL0mSJEnSGLKglyRJkiRpDFnQS5IkSZI0hlboOoDGX5LHDLJfVf1s1FkkSaOT5IWD7FdVXxh1FknS8Pk5P34cFE93WpKzB9itqmqzkYeRJI1MkqN7F4GHAZcA5wP3BO4K/KqqHt1BPEnSnTTpc346VVUDNehp9CzoJWkZJDkfmPEDtKo2noU40qxL8gngrKr6aM+6VwGbV9U+nQWThizJLxns8/6RsxBHkm7Hgl6SlkGSHXoW/xV4AfBx4FxgE2Bv4EtV9aEO4kkjl+RKYIOqWtKzbnngsqpat7tk0nAleUHP4ubAC4FDaD7vN6b5/P9CVb21g3jSrEgSmp5ZAFTVLR3GUQ8Leg1VkrWAA4AdgA24/RvflkrNS0lOAZ5YVRf2rLsH8MOquk93yaTRSXIa8Kaq+k7PumcA76uqe3cWTBqhJCcAL6qqU3vWbU1T0D+ku2TS8CXZCPgk8Ehgnd5tVbV8F5l0R45yr2H7NPAA4O3AesArgfOAj3QZShqxDYHFk9YtBjbqIIs0W/YBDklyXJKvJzmeptXylR3nkkbpn4GzJq07G9iqgyzSqH0W+AfwWJrvNQ8Avge8tMtQuj1b6DVUSf4G/HNVXZ7kqqpap/1178iqekDX+aRRSHIw8E/AO4ELaAYHeyNwXlW9oM9NpbGWZANgR5oftS4Gjqqqy7tNJY1Oku8B1wNv4bbP+wOANatqpw6jSUOX5HJg46q6rud7/XrAcVXlj1hzhAW9hirJZcDdqurmJBcA2wDXAldV1VrdppNGI8kqNF/odua2wuYbwNuq6oYOo0mShqgtZj4NPAtYHrgZOBx4ZVVd1mU2adjahrp7VtWNSc6hGTPoGpqxUtbsNJxuZUGvoUryU+DdVfXTJIcBt9B00XlgVW3XbTpJ0p3haN9SI8lywCLgUgcH03yV5Eia8SG+k+RzwBbADcBqTk86d1jQa6iSbEbzujoryV2AdwNr0rRU/rHbdNLwJBlo/tWq+tmos0izZdJo39OqqkNGnUWaLe13mxlV1V9GnUWaTUnWAZarqiuSrAq8FlgD+GhVXdxpON3Kgl6SlkGSswfYrapqoC+CkqS5KcktND1T0me3ctRvSV2woNfQJXkCcD+aX/BuVVX7dxJIkjQSSfYCnk8zo8OFwKFV9cVuU0mShiHJSsCeTP29fo8OImkKK3QdQPNLkk8CzwWOphkFVloQkqwAbE9T2FwAHF9VN3ebShqdJG8G9gA+BJwLbAL8V5INq+pdnYaTRiDJ8sAZwNZVdWPXeaRZcAiwLXAk8NeOs2gattBrqJJcAWxbVed3nUWaLUm2ovnPblXgfJppjP4O7FRVp3WZTRqV9rSTR1XVuT3rNgF+UVWbdJdMGp0kZwAPqqqrus4ijVqSK4F/8vU+ty3XdQDNO5cBV3UdQpplnwYOpJna5aFVdQ/gs+16ab5aHbh00rrLaX7YkuarjwJfT7JDks2TbDZx6TqYNALnASt3HUL92UKvoUryEuApwHuY1DXH0V81X7U9UxZV1ZKedSvQTGe0bnfJpNFJ8iWaWUzeQPOlbxPgXcD1VfX8LrNJo9IOkDcVB8XTvJPkNcDOwMe44/d6Z/GZIyzoNVT+R6eFKMkpwD69/7kleTTwyaraprtk0ugkWQv4JPBvNGPy3AR8E3il3TMlafz1mdHHWXzmEAt6SbqTkjwN+CrwfW4bHOwpwO5V9d0us0mjlmQ5YAPgsqqa7kddaV5JsjHtIKiOGySpSxb0Ggn/o9NCk2RLmhkeNgQuAr5RVWd0m0oajSQrVtVN7fWHc/sxeY5zhgfNV0nuDnwNeCjNmBHrAycAu1TVRV1mk0bBWXzmPgt6DZX/0UnS/JbkZcD2E+fJJ7meZkDUAKsB/1VVB3UYURqZJEfQjBnxxqq6LsnqwLtpRgJ/WqfhpCFzFp/xYEGvofI/Oi0kbav846vqU+3yD4GVenZ5WVX9qZNw0ogkOR54aVWd3C5fOTH4Y5L7AZ+pqod2GFEamSSXAXef6KHSrlsZuLCqNugumTR8SX4G/AD4YLVFY5LXAk+pqkd3Gk63sqDXUPkfnRaSJF8Afl5Vh7TL1wCvbjffD1ijqvbqKJ40Ekkuqaq79Sz/qqoe1l4PcElV3bWzgNIIJTkTeM7ED1rtuvsCh1fVvbpLJg2fs/iMhxW6DqB550pga+DknnX3xrnpNT89EvjPnuUlE12Nk6wJ/KaLUNKIrZFk9aq6DmCimG+t3l6k+er9wE+SHMRtg6DuBbyl01TSaFwE7AD0TlH3iHa95ggLeg2b/9FpIblLVV3Ts7zHxJWqujaJrZSaj04BngB8Z4ptTwROnd040uypqs8nOQvYFbgvTWGza1X9tNtk0ki8CfhekolZfDYFngzs3mUo3Z5d7jV0SR5D8x/dxGjfh/kfneajJBcCD6uqc6bYthlwbFVtOOvBpBFKsgvwEeBlwPeq6pZ26rqnA58G9q2qw7rMKA1bktOAnwO/AH5RVRd0HEmaFc7iM/dZ0EvSMkpyILDqxGjfk7YdCvy9qv5j9pNJo5XkNcDbaAaBvIxmHvobgbdX1Qe6zCaNQpLdaLoaPwL4Z+Bs2uKepsA/q8N40qxIsipwS1Xd2HUW3caCXkOVZF/gZ1X1uyQPBr4JLKHpjnZ8t+mk4UpyN+A44Gqa7seXAHcHngGsCzykqi7pLKA0QknWopmbeH2aaUqPr6qru00ljV6SDYCH04yj8ghgW+CvVXXPToNJQ5bkgzQt8r9O8hTgW0AB/1ZVR3abThMs6DVUSc4H7lNVVyc5GvgucC3w4qp6cLfppOFLsh6wL/BYmlbKy4GfAh+uqsu7zCbNlrbL/a2q6pauskizIcm/0AwW9kjgUTSjfm/TaShpyJJcDGxeVdcn+T+asbKuBj5SVf/SbTpNsKDXUCW5pqrWakf4Ppd2qoskV1XVOh3HkyQNSZIHAJ+iGRhslYnVQFXV8p0Fk0Ygyb/SFO87AA8C/gL8Cvgl8Ct/wNV8lOTqqlo7yfrA6VW1qF1/TVWt1XE8tRzlXsN2fpLtgW1ozilb0nbLXDLD7aSxkuSFg+xXVV8YdRapI4cARwIvBK7vOIs0av8HnAa8D3huVf294zzSbDijHT/iXsCP4dZTTm7oNJVuxxZ6DVWSHYGDgH8Az66qk5LsCjy/qnbsNp00PO0pJbcuAg+jOYf+fOCewF1pWm0e3UE8aeSSXAOsXX6R0AIwaVC81YBjaVrnf1lVp3WZTRqVtmfKx2i+17+oqs5q3wtPmmpAYHXDgl5D055D+SiaIubGnvUrAlTVTR1Fk0YqySeAs6rqoz3rXkVz3tk+nQWTRijJIcBXq+p/u84izaZJg+I9nOZH3BOq6pmdBpOGKMnywAtoPuftkTKHWdBrqJJcW1Vrdp1Dmk1JrgQ2qKolPeuWBy6rqnW7SyaNTpKvAzvRtFTebjaHqtqjk1DSLJk0KN6jgbWqauVuU0nD5RhY48Fz6DVsv0jykKo6oesg0iy6BHgazdR1E3YC/tZNHGlW/LG9SPNez6B4j6Q5xWo14Nc03e4/TzOFqTTfHJlkJ6eom9tsoddQJfk08Dya6erOp5mrEoCq2r+rXNIoJXk88G3gFJrX/cbA1sDOVfWjLrNJku68JNcCxwO/aC//13t6oTQfJfkmTYPF8dzxe709seYIW+g1bKsCR7TX79FhDmlWJAlwFrAZsCOwIXAUcJTTGGm+SfLIqvpFe/0x0+1XVT+bvVTSrFin97QqaYE4pb1oDrOFXpLupCTXAWtW1S1dZ5FGKckpVXWf9vrZ0+xWVbXZLMaSRsppSiXNZRb0GrokWwE7A3etqr2T3BtYuap+33E0aSSSHAv8e1Wd3nUWadSSPLiq/q/rHNJscZpSLWTtaYW7AHepqp2SbEczCKQ9seaI5boOoPklyc40A8RsBEycW7Mm8OHOQkmjdwzwwyQHJHlRkhdOXLoOJo3Aj3sXkpzYVRBpNlTVoycuwB+A11XVPatq+6q6J/C6dr00ryR5JfAZ4EyaASEBbgDe2Vko3YEt9BqqJKcBu1TVyUmurKp123noL6qqRV3nk0ZhUutNr6qqac8zlsbR5OlJJz7ru8wkzRanKdVCkuQs4LFVdU7P9/rlgb9V1fpd51PDQfE0bHcBJrrWV89ffznSvGU3Sy0wkz/P/XzXQuI0pVpI1qQ5tQRu+6xfEfhHN3E0FQt6DdtJwPOBL/Ws24VmrlZp3mtHvc/EsgPlaR5aMcle3PY6X2ny6SUODqZ5bB/g20lex6RpSjtNJY3GL4A3AO/qWbcPMF3PRHXALvcaqnZAvB8BZwMPoTm3eEvgCVV1ZofRpJFJshHwSZrzy9bp3VZVy3eRSRqVJMfQv1XeU000L7U/2P4TcA23TVN6MU5TqnkqyYbA94ANaMbH+gtwLfDUqrqky2y6jQW9hi7JasBTgU1ofr3+flUt7jaVNDpJjgSuB94D/JymsD8A+J+q+nyH0SRJQ+Q0pVpo2h+y/pXbvtf/2tf/3GJBr6FIclfgI8B9gN8Ar/HXai0USS4HNq6q65JcVVXrJFkPOK6qtuo6nzQKSRYBN1TV4naQpD2AJcCX/bKn+cppSrUQtFNOf5HbvtfvVVVnd5tK07Gg11Ak+TZNV+NvA88GLqyqPfreSJonkvwNuGdV3ZjkHJpfsq+hGfV4zb43lsZUkv8DXlpVv03yPpqeWTcBR1fVq7tNJ41GkncCuwMH07RW3vpF2rEjNF8k+RHNQI9fBXYDVq+qZ3QaStOyoNdQtAXNllV1VZINgN+2c7NK817b5f4LVfWdJJ8DtqCZp3U1R8DXfNVO37VeVVWSC4DtgcXAqVV1927TSaPhNKVaCJJcBtyjqv6eZA3gjKrasOtcmpoFvYYiyTVVtVbP8hVVtV6XmaTZkmQdYLmquiLJqsBraKZ6+WhVXdxpOGlE2i98G9EMfPq1qtomyXLA1fZMkaTx5ff68eK0dRqWydMYrew0Rlooquqqnus3AO/sLo00a34AfANYH/hau25r4MLOEkmzyGlKNY+tnOTtPcurTlqmqvaf5Uyahi30GgqnMdJClGRP4ElVtcsU2w6jmcroy7MeTJoFSVYGXkBz3vyhVXVzkkcBd6uqr/W7rTSunKZUC0GSg+n/vZ6q2mt20mgmFvSStIySnAC8rKp+O8W2bYHPVtVDZz+ZJGkUnKZU0lxjQa+hchojLSRJLquqDZZ1uzTOkhzKNC04znKi+cppSrUQJHlWVR0+zbYVgf2r6i2zHEvTWK7rAJp3vk8zwjfAu4HXAvsCH+oskTQ6y7df5O6gXW/3S81nfwbO6rlcB+wIXNFlKGnElgA3t9evahsyrqMZIFKaLz6S5Jvt6/tWSR4G/B6w9+EcYgu9hsppjLSQJDmKZs7tD06x7TXAY6vqybOfTOpGku2At1bVTl1nkUbBaUq1ELRT1X0AeA7NzD3fAd4P7Ay8vqoO6jCeJrGg11A5jZEWkiQPAn4KfBH4NnAxcHfg2cCewGOq6sTOAkqzLMkKwBW90x1J84nTlGohSbID8C1gVeAnwEur6pJuU2kyp63TsDmNkRaMqvp1kifQ/Gr9cprTmG4BjgeeaDGv+SzJ5JlLVgN2Af7YQRxpVjhNqRaKJOsDL6GZyeR3NN/ntwQs6OcYW+g1VE5jpIWm7YHyKJoifl3gyvZLnjSvJTl70qrraL70vaWqJm+TxprTlGohSfI84GM0Y2PtW1VXJdkd+DBwOPBfVXVNlxl1Gwt6SbqTklzrKSWSNH85TakWkvYH25dU1Y8mrb8L8Elg+6q6RyfhdAcW9BoqpzHSQtQOjveOqjqh6yzSKLU9UmbkNKWab5ymVAtJktWr6ro+259eVd+dzUyanufQa9j+PGn5bjQjZH6lgyzSbDkX+EGS7wLn0/OjVlXt31kqafhuZpofbVtptztlo+ab5ZOsV1V3mJbRaUo1D61JcxrVdC6YrSCamQW9hqqq3jZ5XZKDgLd2EEeaLasCR7TX7YKm+eyfug4gdeQ44IXAHaYpBfaiGUdFmi/OAG6drSTJmVW1Rc/2o3u3q1t2udfIOY2RJM0vSbaiGfH491U1uWeWNO84TakWksljAyW5sqrWnW67umULvYbKaYy0ULUFzs7AXatq7yT3Blauqt93HE0aqna0788DVwJrJ9m9qr7ZbSpptJymVAvM5BbfmZbVIVvoNVROY6SFKMnOwKdpWm12raq1kmwHvLeqHtdtOmm4kpwGvKGqvpvkmTSf7w/oOpc0ak5TqoUiyTW9PWuTXFFV6023Xd2yoJekO6ktcHapqpMnuqUlWRG4qKoWdZ1PGqYkV1fV2u31AJc6urcWCrsaayFIcjPNuBETHspt40QEeEhVrTjrwTQlu9zrTnMaI4m7ABNd66vnr7+Yaj7KxJWqqkH/D5DmiV8keYjTlGqee9Gk5YMmLf/3bAXRzCzoNQxOY6SF7iTg+cCXetbtAvy6mzjSSK2e5Lye5bUnLVNVG89yJmm2OE2p5r2qOqTrDBqcBb2GwWmMtNDtA/woyYtoip3/BbYEntBtLGkkJg9+Ki0kTlOqeS/JHjPtU1VfmmkfzQ7PodfQOI2RFrIkqwFPBTahabX5flUt7jaVJEnS0klyC/Bn4BJ6TrPqUVX1yNlNpelY0GsoJk9jBDiNkRaMJB+vqn2mWP/RqvrPDiJJI5NkqlaZm2i6In+rqpymVPOa05RqvkvyEZrX+Mk0pxMeUVU3dptK03EgGw3L64HnVNVdaM4dfmPHeaTZtOc0658/myGkWXLWFJeLgXsDxyd5SofZpJFqpyn9JbARMNEteU3gw52Fkoasql5N0+Pw08CzgHOSfD7Jw7tNpqnYQq+hcBojLURJXthe/SSw96TNmwE7V9W9ZzeV1J0kjwPeW1XbdZ1FGgWnKdVClGRt4M3AvsDjq+rojiOph4PiaVicxkgL0UQL/ErcvjW+gL8CL5j1RFK3fgps3nUIaYScplQLRlvI70LzfWYR8A7gd11m0h1Z0GtYnMZIC05VPRogyTurar+u80hzwIbAVV2HkEbIaUo17yXZieaUkocD3wVeV1W/6jaVpmOXew1Fkh1m2qeqfj4bWaTZlmQRcENVLU6yPM1/gkuAL1fVLd2mk4YryWZTrF4R2BTYDziuql4/q6GkWdIOiPcj4GzgIcAxtNOUVtWZHUaThqYd5f5PwPeBG6bap6r2n9VQmpYFvSTdSUn+D3hpVf02yftopq+7CTi6HVhGmjfaL3rF7acyWgKcB3wdeHtV/b2LbNJscJpSzXdJDqb/aSTLV9WMc9VrdljQayicxkgLWZIrgfXa8SMuALYHFgOnVtXdu00nSRoWpynVQpbkvjS9EHetqg27zqOGA5dpWJzGSAvZEmClJP8CXF1V59GcR7xGp6kkScO25zTrnaZU81KSRUleleQ3wG+B7YBXdRxLPRwUT0NRVW+bbtvENEbAUbOXSJpVPwC+AawPfK1dtzVwYWeJJElD0zNN6Qo91ydsBlw2y5GkkWmnYnwazQ9YTwT+DBxGM1bKc6vqb52F0x3Y5V4j185Lf0VVrdt1FmkUkqxMM6XLTcChVXVzkkcBd6uqr/W7rSRp7ksyMe/2I4Bf9myamKb0Y1V1wqwHk0YgyRXALcDBwFer6jft+ouBbS3o5xZb6DUbnMZI81pV3QgcOGndMd2kkSQNm9OUaoH5Pc2UdQ8GzkxydlVd2XEmTcMWeg2F0xhpoUvyNGAHYAN6Rv92FFhJmj+cplQLRZJNaF7fewAb00zXuAPwz1XlKYVziIPiaVj+DJzZ/p24nAJ8mqZr2lu7iyaNVpK3Ap+j+UzdGbic5pyzqzqMJUkavu8DW7TX3w28FtgX+FBniaQRqKpzq+odVbUF8Fiawa5vAU5O8v5u06mXLfSSdCclORd4SlWdkuSqqlonyYOA/arqaV3nkyQNh9OUaiFLsgrwTGCPqtqx6zxqWNBL0p2U5OqqWru9/jdgo6q6qXe9JGn8JbkM2AjYEvhaVW2TZDmaKUvX7DadpIXIQfEk6c47K8k2VXUqzakmL2tbcRxARpLmF6cplTSn2EIvSXdSkicDi6vqF0keDHwFWAN4eVUd3m06SdKwOE2ppLnGgl6SllGSjWfap6rOm40skiRJWngs6CVpGSW5BZj4EE3PpmqXq6qWn/VgkqSRcZpSSXOJ09ZJ0rI7mWa6xv2ATYAV28tKPX8lSfOE05RKmmss6CVpGVXV/YHnAOsBvwL+B9gFWKmqllTVki7zSZKG7oXA46vq1cA/2r87AZt2mkrSgmWXe0kagnbaoscDewI7Ao+pqt90GkqSNFROUypprnHaOkkaji1ozql8KPBbnLJOkuYjpymVNKdY0EvSMkqyHvA8mimM1gQOBR7pyPaSNG/tRzMHPcAb6ZmmtLNEkhY0u9xL0jJK8nfgbJpC/oSp9qmqn81qKEnS0DlNqaS5yoJekpZRknO4bdq6qVRVbTZLcSRJI+I0pZLmKgt6SZIkqY8kvwVWBQ4BvgxcNHkfZzaR1AWnrZMkSZL6cJpSSXOVLfSSJEnSgJymVNJcYgu9JEmSNDinKZU0ZzhtnSRJktSH05RKmqvsci9JkiT14TSlkuYqC3pJkiSpD6cplTRXWdBLkiRJkjSGHBRPkiRJkqQxZEEvSZIkSdIYsqCXJEkzSnJOkm/3LD8nycEdRpIkacGzoJckSYN6YJKtuw4hSZIaFvSSJC1ASTZNcnqSryQ5Lcm3kjw5yRE9+zw+yXd6bvYh4M1THOtBSY5P8tskxyW5d7t+zyRHJPlx28K/d5J92/1OaOf2JsnmSX6Y5KQkv0yy1YgfviRJ84IFvSRJC9e9gU9X1T8D1wDbAFslWdRu3wv4Qs/+3wAekORek45zOvCIqro/sD/w7p5t9wGeBfwr8C7g+na/44E92n0OBF5ZVQ8EXgt8ekiPT5KkeW2FrgNIkqTOnF9Vv2qvfxnYBzgU2D3JF4GHclvRDbAE+ADwRuAHPevXBg5JsgXNXN0r9mw7uqquBa5NcjVwZLv+D8B9k6wBbA98M8nEbVYe0uOTJGles6CXJGnhqimWv0hTdP8d+GZV3Txpn0NpCvpTeta9g6Zwf2aSTYFjerbd2HP9lp7lW2i+hywHXFVV91vmRyFJ0gJll3tJkhaujZM8tL2+K3BsVV0EXATsR1Pc305V3QR8BHh1z+q1gQvb63suTYCqugY4O8nOAGlsuzTHkCRpobKglyRp4foT8IokpwHrAp9p13+Fpjv+adPc7iBu38vv/cB7kvyWZev9txvwoiQnA6cCT1+GY0iStOCkanJvO0mSNN+1XeO/X1X3mWLbJ4HfVtVBsx5MkiQNzHPoJUnSrZKcBFwHvKbrLJIkqT9b6CVJkiRJGkOeQy9JkiRJ0hiyoJckSZIkaQxZ0EuSJEmSNIYs6CVJkiRJGkMW9JIkSZIkjSELekmSJEmSxtD/B/7LAehAQM7ZAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "plot_share_delta_graph(df[df['pyChannel']=='SMS'].reset_index(drop=True), 'Clicked', 'pyName', dates=4)" + "plot_share_delta_graph(\n", + " df[df[\"pyChannel\"] == \"SMS\"].reset_index(drop=True), \"Clicked\", \"pyName\", dates=4\n", + ")\n", + "\n", + "# A delta view of actions vs a condition (eg experiment) could be neat, showing a delta of offer counts\n", + "# maybe a variant of the regular bar charts" ] }, { @@ -977,6 +412,15 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Performance\n", + "\n", + "Model performance over time\n" + ] } ], "metadata": { diff --git a/python/pdstools/ih/Aggregates.py b/python/pdstools/ih/Aggregates.py index b22b968e..8bd41842 100644 --- a/python/pdstools/ih/Aggregates.py +++ b/python/pdstools/ih/Aggregates.py @@ -1,8 +1,10 @@ +from datetime import timedelta from typing import TYPE_CHECKING, List, Optional, Union import polars as pl from ..utils.namespaces import LazyNamespace -from ..utils.cdh_utils import safe_flatten_list +from ..utils import cdh_utils +from ..utils.types import QUERY if TYPE_CHECKING: from .IH import IH as IH_Class @@ -17,7 +19,8 @@ def __init__(self, ih: "IH_Class"): def summary_success_rates( self, by: Optional[Union[str, List[str]]] = None, - every: Optional[str] = None, + every: Optional[Union[str, timedelta]] = None, + query: Optional[QUERY] = None, ) -> pl.LazyFrame: """Groups the IH data summarizing into success rates (SuccessRate) and standard error (StdErr). @@ -49,17 +52,13 @@ def summary_success_rates( else: source = self.ih.data - group_by_clause = safe_flatten_list( + group_by_clause = cdh_utils.safe_flatten_list( [by] + (["OutcomeTime"] if every is not None else []) ) - # TODO filter out nulls for the by arguments - # source.filter( - # pl.col.ExperimentGroup.is_not_null() & (pl.col.ExperimentGroup != "") - # ) - summary = ( - source.group_by( + cdh_utils._apply_query(source, query) + .group_by( (group_by_clause + ["InteractionID"]) if group_by_clause is not None else ["InteractionID"] @@ -143,3 +142,26 @@ def summary_success_rates( summary = summary.sort(group_by_clause) return summary + + def summary_outcomes( + self, + by: Optional[Union[str, List[str]]] = None, + every: Optional[Union[str, timedelta]] = None, + query: Optional[QUERY] = None, + ): + + if every is not None: + source = self.ih.data.with_columns(pl.col.OutcomeTime.dt.truncate(every)) + else: + source = self.ih.data + + group_by_clause = cdh_utils.safe_flatten_list( + ["Outcome"] + [by] + (["OutcomeTime"] if every is not None else []) + ) + + summary = ( + cdh_utils._apply_query(source, query) + .group_by(group_by_clause) + .agg(Count=pl.len()) + ) + return summary diff --git a/python/pdstools/ih/Plots.py b/python/pdstools/ih/Plots.py index 4951001a..22fed280 100644 --- a/python/pdstools/ih/Plots.py +++ b/python/pdstools/ih/Plots.py @@ -1,10 +1,13 @@ -from typing import TYPE_CHECKING, Dict, List, Optional +from datetime import timedelta +from typing import TYPE_CHECKING, Dict, List, Optional, Union import polars as pl import plotly as plotly import plotly.express as px import plotly.graph_objs as go from plotly.subplots import make_subplots +from ..utils.types import QUERY +from ..utils import cdh_utils from ..utils.namespaces import LazyNamespace if TYPE_CHECKING: @@ -18,15 +21,16 @@ def __init__(self, ih: "IH_Class"): def overall_gauges( self, - metric: str, - experiment_field: str, + condition: Union[str, pl.Expr], + metric: Optional[str] = "Engagement", by: Optional[str] = "Channel", reference_values: Optional[Dict[str, float]] = None, title: Optional[str] = None, + query: Optional[QUERY] = None, return_df: Optional[bool] = False, ): plot_data = self.ih.aggregates.summary_success_rates( - by=[experiment_field, by], + by=[condition, by], query=query ) if return_df: @@ -38,7 +42,9 @@ def overall_gauges( plot_data = plot_data.collect() cols = plot_data[by].unique().shape[0] # TODO can be None - rows = plot_data[experiment_field].unique().shape[0] + rows = ( + plot_data[condition].unique().shape[0] + ) # TODO generalize to support pl expression fig = make_subplots( rows=rows, @@ -87,7 +93,7 @@ def overall_gauges( number={"valueformat": ",.2%"}, value=row[f"SuccessRate_{metric}"], delta={"reference": ref_value, "valueformat": ",.2%"}, - title={"text": f"{row[by]}: {row[experiment_field]}"}, + title={"text": f"{row[by]}: {row[condition]}"}, gauge=gauge, ) r, c = divmod(index, cols) @@ -96,45 +102,31 @@ def overall_gauges( return fig - def conversion_overall_gauges( + def outcomes_tree_map( self, - experiment_field: str, - by: Optional[str] = "Channel", - reference_values: Optional[Dict[str, float]] = None, - title: Optional[str] = None, + by: Optional[List[str]] = None, + query: Optional[QUERY] = None, return_df: Optional[bool] = False, ): - return self.overall_gauges( - metric="Conversion", - experiment_field=experiment_field, - by=by, - reference_values=reference_values, - title=title, - return_df=return_df, - ) + if by is None: + by = [ + f + for f in ["Outcome", "Direction", "Channel", "Issue", "Group", "Name"] + if f in self.ih.data.collect_schema().names() + ] - def egagement_overall_gauges( - self, - experiment_field: str, - by: Optional[str] = "Channel", - reference_values: Optional[Dict[str, float]] = None, - title: Optional[str] = None, - return_df: Optional[bool] = False, - ): - return self.overall_gauges( - metric="Engagement", - experiment_field=experiment_field, - by=by, - reference_values=reference_values, - title=title, - return_df=return_df, - ) + plot_data = self.ih.aggregates.summary_outcomes(by=by, query=query) + + if return_df: + return plot_data + pass def success_rates_tree_map( self, - metric: str, + metric: Optional[str] = "Engagement", by: Optional[List[str]] = None, title: Optional[str] = None, + query: Optional[QUERY] = None, return_df: Optional[bool] = False, ): if by is None: @@ -144,9 +136,7 @@ def success_rates_tree_map( if f in self.ih.data.collect_schema().names() ] - plot_data = self.ih.aggregates.summary_success_rates( - by=by, - ) + plot_data = self.ih.aggregates.summary_success_rates(by=by, query=query) if return_df: return plot_data @@ -179,46 +169,23 @@ def success_rates_tree_map( return fig - def conversion_success_rates_tree_map( - self, - by: Optional[List[str]] = None, - title: Optional[str] = None, - return_df: Optional[bool] = False, - ): - return self.success_rates_tree_map( - metric="Conversion", - by=by, - title=title, - return_df=return_df, - ) - - def engagement_success_rates_tree_map( - self, - by: Optional[List[str]] = None, - title: Optional[str] = None, - return_df: Optional[bool] = False, - ): - return self.success_rates_tree_map( - metric="Engagement", - by=by, - title=title, - return_df=return_df, - ) - def success_rates_trend_bar( self, - metric: str, - experiment_field: str, - every: str = "1d", + condition: Union[str, pl.Expr], + metric: Optional[str] = "Engagement", + every: Union[str, timedelta] = "1d", by: Optional[str] = None, title: Optional[str] = None, + query: Optional[QUERY] = None, return_df: Optional[bool] = False, ): plot_data = self.ih.aggregates.summary_success_rates( every=every, - by=[experiment_field] + [by], + by=[condition] + [by], # TODO generalize to support pl expression + query=query, ) + if return_df: return plot_data @@ -229,63 +196,30 @@ def success_rates_trend_bar( plot_data.collect(), x="OutcomeTime", y=f"SuccessRate_{metric}", - color=experiment_field, + color=condition, error_y=f"StdErr_{metric}", facet_row=by, barmode="group", - custom_data=[experiment_field], + custom_data=[condition], template="pega", title=title, ) fig.update_yaxes(tickformat=",.3%").update_layout(xaxis_title=None) return fig - def conversion_success_rates_trend_bar( - self, - experiment_field: str, - every: str = "1d", - by: Optional[str] = None, - title: Optional[str] = None, - return_df: Optional[bool] = False, - ): - return self.success_rates_trend_bar( - metric="Conversion", - experiment_field=experiment_field, - every=every, - by=by, - title=title, - return_df=return_df, - ) - - def engagement_success_rates_trend_bar( - self, - experiment_field: str, - every: str = "1d", - by: Optional[str] = None, - title: Optional[str] = None, - return_df: Optional[bool] = False, - ): - return self.success_rates_trend_bar( - metric="Engagement", - experiment_field=experiment_field, - every=every, - by=by, - title=title, - return_df=return_df, - ) - def success_rates_trend_line( self, - metric: str, - every: Optional[str] = "1d", + metric: Optional[str] = "Engagement", + every: Union[str, timedelta] = "1d", by: Optional[str] = None, title: Optional[str] = None, + query: Optional[QUERY] = None, return_df: Optional[bool] = False, ): plot_data = self.ih.aggregates.summary_success_rates( - every=every, - by=by, + every=every, by=by, query=query ) + if return_df: return plot_data @@ -302,33 +236,3 @@ def success_rates_trend_line( fig.update_yaxes(tickformat=",.3%").update_layout(xaxis_title=None) return fig - - def conversion_success_rates_trend_line( - self, - every: Optional[str] = "1d", - by: Optional[str] = None, - title: Optional[str] = None, - return_df: Optional[bool] = False, - ): - return self.success_rates_trend_line( - metric="Conversion", - every=every, - by=by, - title=title, - return_df=return_df, - ) - - def engagement_success_rates_trend_line( - self, - every: Optional[str] = "1d", - by: Optional[str] = None, - title: Optional[str] = None, - return_df: Optional[bool] = False, - ): - return self.success_rates_trend_line( - metric="Engagement", - every=every, - by=by, - title=title, - return_df=return_df, - ) From c7fb73820b841353883a49b81b13ef371a4cd634 Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Sun, 22 Dec 2024 20:11:51 +0100 Subject: [PATCH 19/21] Reworked old IH example article --- examples/ih/Example_IH_Analysis.ipynb | 398 ++++++++++---------------- python/pdstools/ih/Aggregates.py | 70 +++-- 2 files changed, 198 insertions(+), 270 deletions(-) diff --git a/examples/ih/Example_IH_Analysis.ipynb b/examples/ih/Example_IH_Analysis.ipynb index c0fc2cf0..89d27fc8 100644 --- a/examples/ih/Example_IH_Analysis.ipynb +++ b/examples/ih/Example_IH_Analysis.ipynb @@ -7,7 +7,9 @@ "outputs": [], "source": [ "from pdstools import IH\n", + "from pdstools.utils import cdh_utils\n", "\n", + "import polars as pl\n", "import plotly.io as pio\n", "import plotly as plotly\n", "\n", @@ -19,7 +21,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# IH Example Analysis\n", + "# Example IH Analysis\n", + "\n", + "Interaction History (IH) is a rich source of data at the level of individual interactions. It contains the time of the interaction, the channel, the actions/treatments, the customer ID and is used to track different types of outcomes (decisions, sends, opens, clicks, etc). It does **not** contain customer attributes - only the IDs.\n", + "\n", + "This notebook gives some examples of data analysis on IH. It uses plotly (visualizations) and polars (dataframe) but the purpose is more to serve example analyses than re-usable code. All of the analyses should be able to be replicated easily in other analytical BI environments - except perhaps the analysis of model performance / AUC.\n", "\n", "This notebook uses sample data shipped with PDStools. Replace with actual IH data." ] @@ -39,7 +45,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "At first, take a look into the IH dataframe, explore the columns, outcome types and business structure" + "Preview of the raw IH data" ] }, { @@ -51,6 +57,31 @@ "ih.data.head().collect()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The same interaction can occur multiple times: once when the first decision is made, then later when responses are captured (accepted, sent, clicked, etc.). For some of the analyses it makes more sense to group by interaction first. This is how that data looks like:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ih.aggregates._summary_interactions(by=[\"Channel\"]).head().collect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Distribution Analysis\n", + "\n", + "A distribution of the offers (actions/treatments) is often the most obvious type of analysis. You can do an action distribution for specific outcomes (what is offered, what is accepted), view it conditionally (what got offered last month vs this month) - possibly with a delta view, or over time." + ] + }, { "cell_type": "code", "execution_count": null, @@ -90,53 +121,29 @@ "fig" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use \"plot_daily_accept_rate\" to plot accept rate per day to understand how accept rates changed over time. To define accept rate, enter the positive (here: Accepted) and negative (here: Rejected) behaviour in the function. use kwargs to customize the graph. If the time ticks on the x axis are too many, shrink them using 'shrinkTicks'. If data is missing in certain days, force the graph make gaps for the missing days by setting 'allTime':True. you can also define hue" - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# plot_daily_accept_rate(\n", - "# df,\n", - "# \"Accepted\",\n", - "# \"Rejected\",\n", - "# **{\"hue\": [\"pyChannel\"], \"allTime\": True, \"shrinkTicks\": True},\n", - "# )\n", - "# TODO more or less fits the engagement trend charts\n", - "ih.plots.success_rates_trend_line(by=\"Channel\")" + "plot_data=ih.data.group_by([\"Name\"]).agg(\n", + " pl.col.Name.filter(pl.col.Outcome==\"Clicked\").len().alias(\"Count\")\n", + ").filter(pl.col.Count.is_not_null() & pl.col.Count != 0).sort(\"Count\")\n", + "px.bar(plot_data.collect(),\n", + " x=\"Count\",\n", + " y=\"Name\",\n", + " template=\"pega\",\n", + " title=\"Action Distribution\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The above graph provides detailed metric behavior over time. Instead of Accept, you can use other outcome types. To get a rolled up view, plot the accept rate graph based on a weekly axis. The week values are calculated based on the starting date of the IH file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# plot_weekly_accept_rate(\n", - "# df, \"Accepted\", \"Rejected\", **{\"showOutlier\": True, \"hue\": \"pyDirection\"}\n", - "# )\n", - "# Same" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The above graphs provide insight into the accept rates on daily or weekly basis. ADM models however, take all time data every update cycle, to generate bubble charts. To view the historical cumulative accept rate, use the function below. If choosing a single model, this graph will be as if you had ADM success rate captured over time. Set 'showOutlier' to True to view outlier values" + "# Responses\n", + "\n", + "A simple view of the responses over time." ] }, { @@ -145,46 +152,48 @@ "metadata": {}, "outputs": [], "source": [ - "plot_daily_cumulative_accept_rate(\n", - " df[df[\"pyName\"] == \"UPlusPersonal\"],\n", - " \"Accepted\",\n", - " \"Rejected\",\n", - " **{\n", - " \"allTime\": True,\n", - " \"shrinkTicks\": True,\n", - " \"showOutlier\": True,\n", - " \"title\": \"Proposition: UPlusPersonal\",\n", - " },\n", + "outcomes = [\n", + " c\n", + " for c in ih.data.select(pl.col.Outcome.unique().sort())\n", + " .collect()[\"Outcome\"]\n", + " .to_list()\n", + " if c is not None and c != \"\"\n", + "]\n", + "plot_data = (\n", + " ih.data.with_columns(pl.col.OutcomeTime.dt.truncate(\"1d\"))\n", + " .group_by([\"OutcomeTime\", \"Channel\"])\n", + " .agg([(pl.col.Outcome == o).sum().alias(o) for o in outcomes])\n", + " .collect()\n", + " .unpivot(\n", + " index=[\"OutcomeTime\", \"Channel\"], variable_name=\"Outcome\", value_name=\"Count\"\n", + " )\n", ")\n", - "# Not sure how useful that really is" + "\n", + "px.bar(\n", + " plot_data,\n", + " x=\"OutcomeTime\",\n", + " y=\"Count\",\n", + " color=\"Outcome\",\n", + " template=\"pega\",\n", + " title=\"Daily Responses\",\n", + " facet_row=\"Channel\",\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The above graph can be done in various granularity level. For example the below graph shows the cumulative accept rate over time across all the offers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plot_daily_cumulative_accept_rate(\n", - " df,\n", - " \"Accepted\",\n", - " \"Rejected\",\n", - " **{\"allTime\": True, \"shrinkTicks\": True, \"showOutlier\": True},\n", - ")" + "# Success Rates\n", + "\n", + "Success rates (accept rate, open rate, conversion rate) are interesting to track over time. In addition you may want to split by e.g. Channel, or contrast the rates for different experimental setups in an A-B testing set-up." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Below graph shows the cumulative accept rate per pyGroup, pyDirection and pyChannel" + "Use \"plot_daily_accept_rate\" to plot accept rate per day to understand how accept rates changed over time. To define accept rate, enter the positive (here: Accepted) and negative (here: Rejected) behaviour in the function. use kwargs to customize the graph. If the time ticks on the x axis are too many, shrink them using 'shrinkTicks'. If data is missing in certain days, force the graph make gaps for the missing days by setting 'allTime':True. you can also define hue" ] }, { @@ -193,15 +202,8 @@ "metadata": {}, "outputs": [], "source": [ - "plot_daily_cumulative_accept_rate(\n", - " df,\n", - " \"Clicked\",\n", - " \"NoResponse\",\n", - " **{\n", - " \"hue\": [\"pyGroup\", \"pyDirection\", \"pyChannel\"],\n", - " \"allTime\": True,\n", - " \"shrinkTicks\": True,\n", - " },\n", + "ih.plots.success_rates_trend_line(\n", + " by=\"Channel\", query=pl.col.Channel.is_not_null() & (pl.col.Channel != \"\")\n", ")" ] }, @@ -209,7 +211,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In addition to accept rate, it is important to track other outcome values over time. Use 'daily' or 'weekly' to set the granularity of time axis. Instead of 'Accepted', other outcome labels can be explored over time" + "# Model Performance\n", + "\n", + "Similar to Success Rates: typically viewed over time, likely split by channel, conditioned on variations, e.g. NB vs AGB models." ] }, { @@ -218,51 +222,41 @@ "metadata": {}, "outputs": [], "source": [ - "plot_outcome_count_time(\n", - " df, \"Accepted\", \"weekly\", **{\"hue\": \"pyIssue\", \"allTime\": True, \"shrinkTicks\": True}\n", + "plot_data = (\n", + " ih.aggregates._summary_interactions(every=\"1d\", by=\"Channel\")\n", + " .filter(\n", + " pl.col.Propensity.is_not_null()\n", + " & pl.col.Interaction_Outcome_Engagement.is_not_null()\n", + " )\n", + " .group_by([\"OutcomeTime\", \"Channel\"])\n", + " .agg(\n", + " pl.map_groups(\n", + " exprs=[\"Interaction_Outcome_Engagement\", \"Propensity\"],\n", + " function=lambda data: cdh_utils.auc_from_probs(data[0], data[1]),\n", + " return_dtype=pl.Float64,\n", + " ).alias(\"Performance\")\n", + " )\n", ")\n", - "\n", - "# A count trend seems useful to, not just the CTR" + "fig = px.line(\n", + " plot_data.collect().sort([\"OutcomeTime\"]),\n", + " y = \"Performance\",\n", + " x=\"OutcomeTime\",\n", + " color=\"Channel\",\n", + " template=\"pega\",\n", + " title=\"Model Performance over Time\"\n", + ")\n", + "fig" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "below graphs puts a couple of graphs together to provide better insight at the offer level to be able to compare the accept rate, accept count and total responses per model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plot_df = get_accept_rate(\n", - " df[df[\"pyDirection\"] == \"Inbound\"], \"Accepted\", \"Rejected\", \"pyName\"\n", - ")\n", + "# Propensity Distribution\n", "\n", - "fig, ax = plt.subplots(2, 1, figsize=(13, 9), sharex=True, gridspec_kw={\"hspace\": 0.05})\n", - "sort = plot_df.sort_values(\"Accept Rate (%)\", ascending=False)[\"pyName\"].tolist()\n", - "sns.barplot(x=\"pyName\", y=\"Accept Rate (%)\", data=plot_df, ax=ax[0], order=sort)\n", - "sns.barplot(x=\"pyName\", y=\"Accepted\", data=plot_df, ax=ax[1], order=sort)\n", - "sns.pointplot(x=\"pyName\", y=\"Total\", data=plot_df, ax=ax[1], order=sort)\n", - "for x in ax[1].get_xmajorticklabels():\n", - " x.set_rotation(90)\n", - "ax[0].set_xlabel(\"\")\n", - "ax[1].text(2, 2000, \"The bars show the accepts\\nThe line shows accept+reject\")\n", - "ax[0].set_ylabel(\"Accept Rate (%)\", fontsize=13)\n", - "ax[1].set_ylabel(\"Accepts\", fontsize=13)\n", - "ax[0].set_title(\"Offers within Inbound direction\")\n", + "IH also contains information about the factors that determine the prioritization of the offers: lever values, propensities etc.\n", "\n", - "# Not sure about this" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Another insightful graph is to see what share of a given outcome label, each offer(or direction or channel) has. For example the below graph shows that of all the historical 'Accepted' labels, 'UPlusGold' proposition has a little over 50% of all the 'Accepted' outcomes. 'UPlusFinPersonal' has roughly 10% of all time Accepted outcomes. instead of proposition level, you can set other levels (channel, direction etc)." + "Here we show the distribution of the propensities of the offers made. \n" ] }, { @@ -271,139 +265,40 @@ "metadata": {}, "outputs": [], "source": [ - "plot_outcome_share_graph(df[df[\"pyChannel\"] == \"Web\"], \"Accepted\", \"pyName\", \"pyGroup\")\n", + "import plotly.figure_factory as ff\n", "\n", - "# Wouldn't the tree show this?" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It is also possible to see how the outcome share of a given proposition (or channel etc.) changed over time" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "click_share_name_daily = get_outcome_share_time(\n", - " df[df[\"pyChannel\"] == \"Web\"], \"Clicked\", \"pyName\", time=\"daily\"\n", - ")\n", - "click_share_name_weekly = get_outcome_share_time(\n", - " df[df[\"pyChannel\"] == \"Web\"], \"Clicked\", \"pyName\", time=\"weekly\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The graph below shows among offer within Web channel, what share of Clicked outcome labels belonged to UPlusGold proposition every day. It can be seen that the value dropped significantly on 12-23" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "get_daily_graph(\n", - " click_share_name_daily[click_share_name_daily[\"pyName\"] == \"UPlusGold\"],\n", - " \"Date\",\n", - " \"Clicked Share (%)\",\n", - " **{\"shrinkTicks\": True},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "same graph can be viewed on a weekly basis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "get_daily_graph(\n", - " click_share_name_weekly[click_share_name_weekly[\"pyName\"] == \"UPlusGold\"],\n", - " \"Week\",\n", - " \"Clicked Share (%)\",\n", - " **{\"shrinkTicks\": True},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The graph below shows the Accepted share between two directions: Inbound/Outbound. Of course in this case because there are only 2 directions, when one graph goes up, the other has to go down so the sum of the two per day would be 100%" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "click_share_direction_daily = get_outcome_share_time(\n", - " df, \"Accepted\", \"pyDirection\", time=\"daily\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "get_daily_graph(\n", - " click_share_direction_daily,\n", - " \"Date\",\n", - " \"Accepted Share (%)\",\n", - " **{\"shrinkTicks\": True, \"hue\": \"pyDirection\"},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The above graph can help identify how things evolve as a whole. It helps identify when the share for one direction (or channel etc.) goes down, which channel takes over goes down, which other " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "same graph can be done for pyName as well. However, since there are usually so many pyNames, it would be hard to follow up and identify which offer had the highest share over time, and when an offer's share drops, which other offer takes over. So instead of looking over time, the below graph calculates a delta between the share percentage across two time frames. This is significanlty helpful when things in the strategy changes (priotitization, eligibility etc.) it helps identify how the system reacts once there is a change introduced." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plot_share_delta_graph(\n", - " df[df[\"pyChannel\"] == \"SMS\"].reset_index(drop=True), \"Clicked\", \"pyName\", dates=4\n", - ")\n", + "channels = [\n", + " c\n", + " for c in ih.data.select(pl.col.Channel.unique().sort())\n", + " .collect()[\"Channel\"]\n", + " .to_list()\n", + " if c is not None and c != \"\"\n", + " # if c == \"Web\"\n", + "]\n", + "\n", + "plot_data = [\n", + " ih.data.filter(pl.col.Channel == c)\n", + " .select([\"Propensity\"])\n", + " .collect()[\"Propensity\"]\n", + " .sample(fraction=0.1)\n", + " .to_list()\n", + " for c in channels\n", + "]\n", "\n", - "# A delta view of actions vs a condition (eg experiment) could be neat, showing a delta of offer counts\n", - "# maybe a variant of the regular bar charts" + "fig = ff.create_distplot(plot_data, group_labels=channels, show_hist=False)\n", + "fig.update_layout(title=\"Propensity Distribution\")\n", + "fig" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "In the above graph, the clicked outcome shares for MasterCardGold has increased by 5% recently. The time range can be specified either by defining a lookback window (in that case only enter an integer) or by a list of two tuples where the first tuple represents the earlier time range and the second tuple represent the recent time range" + "# Response Analysis\n", + "\n", + "Time is one of the dimensions in IH. Here we take a look at how subsequent responses relate to the original decision. It shows, for example, how much time there typically is between the moment of decision and the click.\n", + "\n", + "This type of analysis is usually part of attribution analysis when considering conversion modeling.\n" ] }, { @@ -411,15 +306,32 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, "source": [ - "# Performance\n", - "\n", - "Model performance over time\n" + "outcomes = [\n", + " c\n", + " for c in ih.data.select(pl.col.Outcome.unique().sort())\n", + " .collect()[\"Outcome\"]\n", + " .to_list()\n", + " if c is not None and c != \"\"\n", + "]\n", + "plot_data=ih.data.filter(pl.col.OutcomeTime.is_not_null()).group_by(\"InteractionID\").agg(\n", + " [pl.col.OutcomeTime.min().alias(\"Decision_Time\")]+\n", + " [pl.col.OutcomeTime.filter(pl.col.Outcome == o).max().alias(o) for o in outcomes],\n", + ").collect().unpivot(\n", + " index=[\"InteractionID\", \"Decision_Time\"],\n", + " variable_name=\"Outcome\",\n", + " value_name=\"Time\",\n", + ").with_columns(\n", + " Duration = (pl.col.Time - pl.col.Decision_Time).dt.total_seconds()\n", + ").filter(pl.col.Duration > 0)\n", + "fig = px.box(\n", + " plot_data,\n", + " x=\"Duration\",\n", + " y=\"Outcome\",\n", + " color=\"Outcome\",\n", + " template=\"pega\"\n", + ")\n", + "fig" ] } ], diff --git a/python/pdstools/ih/Aggregates.py b/python/pdstools/ih/Aggregates.py index 8bd41842..c39255ac 100644 --- a/python/pdstools/ih/Aggregates.py +++ b/python/pdstools/ih/Aggregates.py @@ -16,37 +16,12 @@ def __init__(self, ih: "IH_Class"): super().__init__() self.ih = ih - def summary_success_rates( + def _summary_interactions( self, by: Optional[Union[str, List[str]]] = None, every: Optional[Union[str, timedelta]] = None, query: Optional[QUERY] = None, ) -> pl.LazyFrame: - """Groups the IH data summarizing into success rates (SuccessRate) and standard error (StdErr). - - It optionally groups by one or more dimensions (e.g. Experiment, Channel, Issue etc). When - given, the 'every' argument is used to divide the timerange into buckets. It uses the same string - language as Polars. - - Every interaction is considered to have only one outcome: positive, negative or none. When any - outcome in the interaction is in the positive labels, the outcome is considered positive. Next, - when any is in the negative labels, the outcome of the interaction is considered negative. Otherwise - there is no defined outcome and the interaction is ignored in calculations of success rate or error. - - Parameters - ---------- - by : Optional[Union[str, List[str]]], optional - Grouping keys, by default None - every : Optional[str], optional - Every interval start and period length, by default None - - Returns - ------- - pl.LazyFrame - A polars frame with the grouping keys and columns for the total number of Positives, Negatives, - number of Interactions, success rate (SuccessRate) and standard error (StdErr). - """ - if every is not None: source = self.ih.data.with_columns(pl.col.OutcomeTime.dt.truncate(every)) else: @@ -56,7 +31,7 @@ def summary_success_rates( [by] + (["OutcomeTime"] if every is not None else []) ) - summary = ( + interactions = ( cdh_utils._apply_query(source, query) .group_by( (group_by_clause + ["InteractionID"]) @@ -81,8 +56,49 @@ def summary_success_rates( .alias(f"Interaction_Outcome_{metric}") for metric in self.ih.positive_outcome_labels.keys() ], + Propensity=pl.col.Propensity.last(), Outcomes=pl.col.Outcome.unique().sort(), # for debugging ) + ) + return interactions + + def summary_success_rates( + self, + by: Optional[Union[str, List[str]]] = None, + every: Optional[Union[str, timedelta]] = None, + query: Optional[QUERY] = None, + ) -> pl.LazyFrame: + """Groups the IH data summarizing into success rates (SuccessRate) and standard error (StdErr). + + It optionally groups by one or more dimensions (e.g. Experiment, Channel, Issue etc). When + given, the 'every' argument is used to divide the timerange into buckets. It uses the same string + language as Polars. + + Every interaction is considered to have only one outcome: positive, negative or none. When any + outcome in the interaction is in the positive labels, the outcome is considered positive. Next, + when any is in the negative labels, the outcome of the interaction is considered negative. Otherwise + there is no defined outcome and the interaction is ignored in calculations of success rate or error. + + Parameters + ---------- + by : Optional[Union[str, List[str]]], optional + Grouping keys, by default None + every : Optional[str], optional + Every interval start and period length, by default None + + Returns + ------- + pl.LazyFrame + A polars frame with the grouping keys and columns for the total number of Positives, Negatives, + number of Interactions, success rate (SuccessRate) and standard error (StdErr). + """ + + group_by_clause = cdh_utils.safe_flatten_list( + [by] + (["OutcomeTime"] if every is not None else []) + ) + + summary = ( + self._summary_interactions(by, every, query) .group_by(group_by_clause) .agg( [ From 2e50c51bd132c2f8992f2dbbeea4e1aa95a6f3d6 Mon Sep 17 00:00:00 2001 From: Otto Perdeck Date: Sun, 22 Dec 2024 22:56:09 +0100 Subject: [PATCH 20/21] Reworked old IH example article --- examples/ih/Conversion_Reporting.ipynb | 4 +- examples/ih/Example_IH_Analysis.ipynb | 120 ++++++----------------- python/pdstools/ih/Aggregates.py | 3 +- python/pdstools/ih/Plots.py | 126 +++++++++++++++++++++++-- 4 files changed, 152 insertions(+), 101 deletions(-) diff --git a/examples/ih/Conversion_Reporting.ipynb b/examples/ih/Conversion_Reporting.ipynb index 1ef101cd..91c84afd 100644 --- a/examples/ih/Conversion_Reporting.ipynb +++ b/examples/ih/Conversion_Reporting.ipynb @@ -110,7 +110,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.success_rates_trend_line(metric=\"Conversion\", every=\"1d\")" + "ih.plots.success_rates_trend(metric=\"Conversion\", every=\"1d\")" ] }, { @@ -139,7 +139,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.success_rates_trend_line(\n", + "ih.plots.success_rates_trend(\n", " by=\"Channel\"\n", ")" ] diff --git a/examples/ih/Example_IH_Analysis.ipynb b/examples/ih/Example_IH_Analysis.ipynb index 89d27fc8..5fad5087 100644 --- a/examples/ih/Example_IH_Analysis.ipynb +++ b/examples/ih/Example_IH_Analysis.ipynb @@ -23,11 +23,11 @@ "source": [ "# Example IH Analysis\n", "\n", - "Interaction History (IH) is a rich source of data at the level of individual interactions. It contains the time of the interaction, the channel, the actions/treatments, the customer ID and is used to track different types of outcomes (decisions, sends, opens, clicks, etc). It does **not** contain customer attributes - only the IDs.\n", + "Interaction History (IH) is a rich source of data at the level of individual interactions from Pega DSM applications. It contains the time of the interaction, the channel, the actions/treatments, the customer ID and is used to track different types of outcomes (decisions, sends, opens, clicks, etc). It does **not** contain customer attributes - only the IDs.\n", "\n", - "This notebook gives some examples of data analysis on IH. It uses plotly (visualizations) and polars (dataframe) but the purpose is more to serve example analyses than re-usable code. All of the analyses should be able to be replicated easily in other analytical BI environments - except perhaps the analysis of model performance / AUC.\n", + "This notebook gives some examples of data analysis on IH. Like most of PDSTools, it uses [plotly](https://plotly.com/python/) for visualization and [polars](https://docs.pola.rs/) (dataframe) but the purpose of this Notebook is more to serve example analyses than re-usable code, although of course we do try to provide some generic, re-usable functions. All of the analyses should be able to be replicated easily in other analytical BI environments - except perhaps the analysis of model performance / AUC.\n", "\n", - "This notebook uses sample data shipped with PDStools. Replace with actual IH data." + "This notebook uses sample data shipped with PDStools. Replace it with your own actual IH data and modify the analyses as appropriate." ] }, { @@ -88,37 +88,7 @@ "metadata": {}, "outputs": [], "source": [ - "# df.groupby(['pyIssue', 'pyGroup', 'pyDirection', 'pyChannel', 'pyName', 'pyOutcome']).count()[[\n", - "# 'pxInteractionID']].rename(columns={'pxInteractionID':'Count'})\n", - "\n", - "# TODO tree map\n", - "import plotly.express as px\n", - "\n", - "plot_data = ih.aggregates.summary_outcomes(\n", - " by=[\"Issue\", \"Group\", \"Direction\", \"Channel\", \"Name\"]\n", - ").collect()\n", - "fig = px.treemap(\n", - " plot_data,\n", - " path=[px.Constant(\"ALL\")]\n", - " + [\"Outcome\"]\n", - " + [\"Issue\", \"Group\", \"Direction\", \"Channel\", \"Name\"],\n", - " values=\"Count\",\n", - " color=\"Count\",\n", - " branchvalues=\"total\",\n", - " # color_continuous_scale=px.colors.sequential.RdBu_r,\n", - " # title=title,\n", - " # hover_data=[\n", - " # f\"StdErr_{metric}\",\n", - " # f\"Positives_{metric}\",\n", - " # f\"Negatives_{metric}\",\n", - " # ],\n", - " height=640,\n", - " template=\"pega\",\n", - ")\n", - "fig.update_coloraxes(showscale=False)\n", - "fig.update_traces(textinfo=\"label+value+percent parent\")\n", - "fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))\n", - "fig" + "ih.plots.response_count_tree_map()\n" ] }, { @@ -127,14 +97,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_data=ih.data.group_by([\"Name\"]).agg(\n", - " pl.col.Name.filter(pl.col.Outcome==\"Clicked\").len().alias(\"Count\")\n", - ").filter(pl.col.Count.is_not_null() & pl.col.Count != 0).sort(\"Count\")\n", - "px.bar(plot_data.collect(),\n", - " x=\"Count\",\n", - " y=\"Name\",\n", - " template=\"pega\",\n", - " title=\"Action Distribution\")" + "ih.plots.action_distribution(query=pl.col.Outcome == \"Clicked\")" ] }, { @@ -152,31 +115,25 @@ "metadata": {}, "outputs": [], "source": [ - "outcomes = [\n", - " c\n", - " for c in ih.data.select(pl.col.Outcome.unique().sort())\n", - " .collect()[\"Outcome\"]\n", - " .to_list()\n", - " if c is not None and c != \"\"\n", - "]\n", - "plot_data = (\n", - " ih.data.with_columns(pl.col.OutcomeTime.dt.truncate(\"1d\"))\n", - " .group_by([\"OutcomeTime\", \"Channel\"])\n", - " .agg([(pl.col.Outcome == o).sum().alias(o) for o in outcomes])\n", - " .collect()\n", - " .unpivot(\n", - " index=[\"OutcomeTime\", \"Channel\"], variable_name=\"Outcome\", value_name=\"Count\"\n", - " )\n", - ")\n", - "\n", - "px.bar(\n", - " plot_data,\n", - " x=\"OutcomeTime\",\n", - " y=\"Count\",\n", - " color=\"Outcome\",\n", - " template=\"pega\",\n", - " title=\"Daily Responses\",\n", - " facet_row=\"Channel\",\n", + "ih.plots.response_counts(every=\"1d\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Which could be viewed per channel as well:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ih.plots.response_counts(\n", + " by=\"Channel\",\n", + " query=pl.col.Channel != \"\",\n", ")" ] }, @@ -202,7 +159,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.success_rates_trend_line(\n", + "ih.plots.success_rates_trend(\n", " by=\"Channel\", query=pl.col.Channel.is_not_null() & (pl.col.Channel != \"\")\n", ")" ] @@ -222,30 +179,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_data = (\n", - " ih.aggregates._summary_interactions(every=\"1d\", by=\"Channel\")\n", - " .filter(\n", - " pl.col.Propensity.is_not_null()\n", - " & pl.col.Interaction_Outcome_Engagement.is_not_null()\n", - " )\n", - " .group_by([\"OutcomeTime\", \"Channel\"])\n", - " .agg(\n", - " pl.map_groups(\n", - " exprs=[\"Interaction_Outcome_Engagement\", \"Propensity\"],\n", - " function=lambda data: cdh_utils.auc_from_probs(data[0], data[1]),\n", - " return_dtype=pl.Float64,\n", - " ).alias(\"Performance\")\n", - " )\n", - ")\n", - "fig = px.line(\n", - " plot_data.collect().sort([\"OutcomeTime\"]),\n", - " y = \"Performance\",\n", - " x=\"OutcomeTime\",\n", - " color=\"Channel\",\n", - " template=\"pega\",\n", - " title=\"Model Performance over Time\"\n", - ")\n", - "fig" + "ih.plots.model_performance_trend(by=\"Channel\")" ] }, { @@ -307,6 +241,8 @@ "metadata": {}, "outputs": [], "source": [ + "import plotly.express as px\n", + "\n", "outcomes = [\n", " c\n", " for c in ih.data.select(pl.col.Outcome.unique().sort())\n", diff --git a/python/pdstools/ih/Aggregates.py b/python/pdstools/ih/Aggregates.py index c39255ac..17b25a09 100644 --- a/python/pdstools/ih/Aggregates.py +++ b/python/pdstools/ih/Aggregates.py @@ -179,5 +179,6 @@ def summary_outcomes( cdh_utils._apply_query(source, query) .group_by(group_by_clause) .agg(Count=pl.len()) - ) + ).sort(cdh_utils.safe_flatten_list(["Count"]+group_by_clause)) + return summary diff --git a/python/pdstools/ih/Plots.py b/python/pdstools/ih/Plots.py index 22fed280..7db952ec 100644 --- a/python/pdstools/ih/Plots.py +++ b/python/pdstools/ih/Plots.py @@ -102,24 +102,46 @@ def overall_gauges( return fig - def outcomes_tree_map( + def response_count_tree_map( self, by: Optional[List[str]] = None, + title: Optional[str] = None, query: Optional[QUERY] = None, return_df: Optional[bool] = False, ): + if by is None: by = [ f - for f in ["Outcome", "Direction", "Channel", "Issue", "Group", "Name"] + for f in ["Direction", "Channel", "Issue", "Group", "Name"] if f in self.ih.data.collect_schema().names() ] + elif isinstance(by, str): + by = [by] - plot_data = self.ih.aggregates.summary_outcomes(by=by, query=query) - + plot_data = self.ih.aggregates.summary_outcomes( + by=by, + query=query, + ) if return_df: return plot_data - pass + + fig = px.treemap( + plot_data.collect(), + path=[px.Constant("ALL")] + ["Outcome"] + by, + values="Count", + color="Count", + branchvalues="total", + # color_continuous_scale=px.colors.sequential.RdBu_r, + title=title, + height=640, + template="pega", + ) + fig.update_coloraxes(showscale=False) + fig.update_traces(textinfo="label+value+percent parent") + fig.update_layout(margin=dict(t=50, l=25, r=25, b=25)) + + return fig def success_rates_tree_map( self, @@ -169,6 +191,29 @@ def success_rates_tree_map( return fig + def action_distribution( + self, + # TODO change - one is the by, when multiple join together + # other is the facet dimension/condition + by: Optional[str] = "Name", + title: Optional[str] = "Action Distribution", + query: Optional[QUERY] = None, + return_df: Optional[bool] = False, + ): + plot_data = self.ih.aggregates.summary_outcomes(by=by, query=query) + + if return_df: + return plot_data + + fig = px.bar( + plot_data.collect(), + x="Count", + y="Name", + template="pega", + title=title, + ) + return fig + def success_rates_trend_bar( self, condition: Union[str, pl.Expr], @@ -207,7 +252,7 @@ def success_rates_trend_bar( fig.update_yaxes(tickformat=",.3%").update_layout(xaxis_title=None) return fig - def success_rates_trend_line( + def success_rates_trend( self, metric: Optional[str] = "Engagement", every: Union[str, timedelta] = "1d", @@ -236,3 +281,72 @@ def success_rates_trend_line( fig.update_yaxes(tickformat=",.3%").update_layout(xaxis_title=None) return fig + + def response_counts( + self, + every: Union[str, timedelta] = "1d", + by: Optional[str] = None, + title: Optional[str] = "Responses", + query: Optional[QUERY] = None, + return_df: Optional[bool] = False, + ): + plot_data = self.ih.aggregates.ih.aggregates.summary_outcomes( + every=every, by=by, query=query + ).collect() + + if return_df: + return plot_data.lazy() + + fig = px.bar( + plot_data, + x="OutcomeTime", + y="Count", + color="Outcome", + template="pega", + title=title, + facet_row=by, + ) + fig.update_layout(xaxis_title=None) + + return fig + + def model_performance_trend( + self, + metric: Optional[str] = "Engagement", + every: Union[str, timedelta] = "1d", + by: Optional[str] = None, + title: Optional[str] = "Model Performance over Time", + query: Optional[QUERY] = None, + return_df: Optional[bool] = False, + ): + + group_by_clause = cdh_utils.safe_flatten_list([by] + ["OutcomeTime"]) + plot_data = ( + self.ih.aggregates._summary_interactions(every=every, by=by, query=query) + .filter( + pl.col.Propensity.is_not_null() + & pl.col(f"Interaction_Outcome_{metric}").is_not_null() + ) + .group_by(group_by_clause) + .agg( + pl.map_groups( + exprs=[f"Interaction_Outcome_{metric}", "Propensity"], + function=lambda data: cdh_utils.auc_from_probs(data[0], data[1]), + return_dtype=pl.Float64, + ).alias("Performance") + ) + .sort(["OutcomeTime"]) + ) + + if return_df: + return plot_data + + fig = px.line( + plot_data.collect(), + y="Performance", + x="OutcomeTime", + color=by, + template="pega", + title=title, + ) + return fig From 2157a90f6f55cfeb8b38ba0765b1d56c838b5ed4 Mon Sep 17 00:00:00 2001 From: Stijn Kas <78410144+StijnKas@users.noreply.github.com> Date: Mon, 23 Dec 2024 15:51:37 +0100 Subject: [PATCH 21/21] Rename plots namespace to plot for consistency (#305) --- examples/ih/Conversion_Reporting.ipynb | 15 +++++++-------- examples/ih/Example_IH_Analysis.ipynb | 17 ++++++++--------- python/pdstools/ih/IH.py | 4 ++-- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/examples/ih/Conversion_Reporting.ipynb b/examples/ih/Conversion_Reporting.ipynb index 91c84afd..077263fa 100644 --- a/examples/ih/Conversion_Reporting.ipynb +++ b/examples/ih/Conversion_Reporting.ipynb @@ -56,7 +56,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.overall_gauges(\n", + "ih.plot.overall_gauges(\n", " metric=\"Conversion\",\n", " condition=\"ExperimentGroup\",\n", " by=\"Channel\",\n", @@ -79,7 +79,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.success_rates_tree_map(metric=\"Conversion\")\n" + "ih.plot.success_rates_tree_map(metric=\"Conversion\")\n" ] }, { @@ -97,7 +97,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.success_rates_trend_bar(\n", + "ih.plot.success_rates_trend_bar(\n", " metric=\"Conversion\",\n", " condition=\"ExperimentGroup\",\n", " every=\"1w\",\n", @@ -110,7 +110,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.success_rates_trend(metric=\"Conversion\", every=\"1d\")" + "ih.plot.success_rates_trend(metric=\"Conversion\", every=\"1d\")" ] }, { @@ -126,7 +126,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.overall_gauges(\n", + "ih.plot.overall_gauges(\n", " condition=\"ExperimentGroup\",\n", " by=\"Channel\",\n", " reference_values={\"Web\": 0.20, \"Email\": 0.20},\n", @@ -139,7 +139,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.success_rates_trend(\n", + "ih.plot.success_rates_trend(\n", " by=\"Channel\"\n", ")" ] @@ -160,8 +160,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/examples/ih/Example_IH_Analysis.ipynb b/examples/ih/Example_IH_Analysis.ipynb index 5fad5087..30188e6e 100644 --- a/examples/ih/Example_IH_Analysis.ipynb +++ b/examples/ih/Example_IH_Analysis.ipynb @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -88,7 +88,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.response_count_tree_map()\n" + "ih.plot.response_count_tree_map()\n" ] }, { @@ -97,7 +97,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.action_distribution(query=pl.col.Outcome == \"Clicked\")" + "ih.plot.action_distribution(query=pl.col.Outcome == \"Clicked\")" ] }, { @@ -115,7 +115,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.response_counts(every=\"1d\")" + "ih.plot.response_counts(every=\"1d\")" ] }, { @@ -131,7 +131,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.response_counts(\n", + "ih.plot.response_counts(\n", " by=\"Channel\",\n", " query=pl.col.Channel != \"\",\n", ")" @@ -159,7 +159,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.success_rates_trend(\n", + "ih.plot.success_rates_trend(\n", " by=\"Channel\", query=pl.col.Channel.is_not_null() & (pl.col.Channel != \"\")\n", ")" ] @@ -179,7 +179,7 @@ "metadata": {}, "outputs": [], "source": [ - "ih.plots.model_performance_trend(by=\"Channel\")" + "ih.plot.model_performance_trend(by=\"Channel\")" ] }, { @@ -286,8 +286,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" + "pygments_lexer": "ipython3" } }, "nbformat": 4, diff --git a/python/pdstools/ih/IH.py b/python/pdstools/ih/IH.py index c97382be..0ded1472 100644 --- a/python/pdstools/ih/IH.py +++ b/python/pdstools/ih/IH.py @@ -7,7 +7,7 @@ from .Aggregates import Aggregates from .Plots import Plots -from ..utils.cdh_utils import to_prpc_date_time, _polars_capitalize, _apply_query +from ..utils.cdh_utils import _polars_capitalize, _apply_query from ..utils.types import QUERY from ..pega_io.File import read_ds_export @@ -20,7 +20,7 @@ def __init__(self, data: pl.LazyFrame): self.data = _polars_capitalize(data) self.aggregates = Aggregates(ih=self) - self.plots = Plots(ih=self) + self.plot = Plots(ih=self) self.positive_outcome_labels = { "Engagement": ["Accepted", "Accept", "Clicked", "Click"], "Conversion": ["Conversion"],