From a04b69ae1ff882b8db2e107b005f77356d657a51 Mon Sep 17 00:00:00 2001 From: sfc-gh-mvashishtha Date: Thu, 25 Apr 2024 16:09:28 -0700 Subject: [PATCH 1/5] SNOW-1344848: Suppress dynamic pivot warnings Signed-off-by: sfc-gh-mvashishtha --- src/snowflake/snowpark/_internal/utils.py | 7 ++- .../plugin/_internal/ordered_dataframe.py | 45 +++++++++++++++---- tests/integ/modin/frame/test_transpose.py | 12 +++++ .../modin/strings/test_get_dummies_series.py | 12 +++++ .../scala/test_dataframe_aggregate_suite.py | 14 ++++-- 5 files changed, 77 insertions(+), 13 deletions(-) diff --git a/src/snowflake/snowpark/_internal/utils.py b/src/snowflake/snowpark/_internal/utils.py index 01b3cc77d12..e20e7bb9225 100644 --- a/src/snowflake/snowpark/_internal/utils.py +++ b/src/snowflake/snowpark/_internal/utils.py @@ -182,6 +182,9 @@ SUPPORTED_TABLE_TYPES = ["temp", "temporary", "transient"] +PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING = "Parameter values is Optional or DataFrame is in private preview since v1.15.0. Do not use it in production." +PIVOT_DEFAULT_ON_NULL_WARNING = "Parameter default_on_null is not None is in private preview since v1.15.0. Do not use it in production." + class TempObjectType(Enum): TABLE = "TABLE" @@ -899,13 +902,13 @@ def prepare_pivot_arguments( if values is None or isinstance(values, DataFrame): warning( df_name, - "Parameter values is Optional or DataFrame is in private preview since v1.15.0. Do not use it in production.", + PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING, ) if default_on_null is not None: warning( df_name, - "Parameter default_on_null is not None is in private preview since v1.15.0. Do not use it in production.", + PIVOT_DEFAULT_ON_NULL_WARNING, ) if values is not None and not values: diff --git a/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py b/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py index 08fed491a3b..a69b23403eb 100644 --- a/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py +++ b/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py @@ -15,7 +15,11 @@ ColumnOrSqlExpr, LiteralType, ) -from snowflake.snowpark._internal.utils import parse_positional_args_to_list +from snowflake.snowpark._internal.utils import ( + PIVOT_DEFAULT_ON_NULL_WARNING, + PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING, + parse_positional_args_to_list, +) from snowflake.snowpark.column import Column from snowflake.snowpark.dataframe import DataFrame as SnowparkDataFrame from snowflake.snowpark.dataframe_writer import DataFrameWriter @@ -818,16 +822,41 @@ def pivot( See detailed docstring in Snowpark DataFrame's pivot. """ snowpark_dataframe = self.to_projected_snowpark_dataframe() + + # Don't warn the user about our internal usage of private preview + # pivot features. The user should have already been warned that + # Snowpark pandas is in public or private preview. They likely don't + # know or care that we are using Snowpark DataFrame pivot() internally, + # let alone that we are using a private preview features of Snowpark + # Python. + + class NoPivotWarningFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + message = record.getMessage() + return ( + PIVOT_DEFAULT_ON_NULL_WARNING not in message + and PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING not in message + ) + + filter = NoPivotWarningFilter() + + class NoPivotWarningContext: + def __enter__(self): + logging.getLogger("snowflake.snowpark").addFilter(filter) + + def __exit__(self, exc_type, exc_val, exc_tb): + logging.getLogger("snowflake.snowpark").removeFilter(filter) + + with NoPivotWarningContext(): + pivoted_snowpark_dataframe = snowpark_dataframe.pivot( + pivot_col=pivot_col, + values=values, + default_on_null=default_on_null, + ) return OrderedDataFrame( # the pivot result columns for dynamic pivot are data dependent, a schema call is required # to know all the quoted identifiers for the pivot result. - DataFrameReference( - snowpark_dataframe.pivot( - pivot_col=pivot_col, - values=values, - default_on_null=default_on_null, - ).agg(*agg_exprs) - ) + DataFrameReference(pivoted_snowpark_dataframe.agg(*agg_exprs)) ) def unpivot( diff --git a/tests/integ/modin/frame/test_transpose.py b/tests/integ/modin/frame/test_transpose.py index aba9e784fd1..3f922939808 100644 --- a/tests/integ/modin/frame/test_transpose.py +++ b/tests/integ/modin/frame/test_transpose.py @@ -10,6 +10,10 @@ import pytest import snowflake.snowpark.modin.plugin # noqa: F401 +from snowflake.snowpark._internal.utils import ( + PIVOT_DEFAULT_ON_NULL_WARNING, + PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING, +) from snowflake.snowpark.modin.plugin._internal.unpivot_utils import ( UNPIVOT_NULL_REPLACE_VALUE, ) @@ -348,3 +352,11 @@ def test_dataframe_transpose_args_warning_log(caplog, score_test_data): "Transpose ignores args in Snowpark pandas API." in [r.msg for r in caplog.records] ) + + +def test_transpose_does_not_raise_pivot_warning_snow_1344848(caplog): + # Test transpose, which calls snowflake.snowpark.dataframe.pivot() with + # the `values` parameter as None or a Snowpark DataFrame. + pd.DataFrame([1]).T.to_pandas() + assert PIVOT_DEFAULT_ON_NULL_WARNING not in caplog.text + assert PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING not in caplog.text diff --git a/tests/integ/modin/strings/test_get_dummies_series.py b/tests/integ/modin/strings/test_get_dummies_series.py index a6b004f577a..6356bc49ad7 100644 --- a/tests/integ/modin/strings/test_get_dummies_series.py +++ b/tests/integ/modin/strings/test_get_dummies_series.py @@ -6,6 +6,10 @@ import pytest import snowflake.snowpark.modin.plugin # noqa: F401 +from snowflake.snowpark._internal.utils import ( + PIVOT_DEFAULT_ON_NULL_WARNING, + PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING, +) from tests.integ.modin.sql_counter import sql_count_checker from tests.integ.modin.utils import assert_snowpark_pandas_equal_to_pandas @@ -52,3 +56,11 @@ def test_get_dummies_series_negative(data): native_pd.get_dummies(pandas_ser), check_dtype=False, ) + + +def test_get_dummies_does_not_raise_pivot_warning_snow_1344848(caplog): + # Test get_dummies, which uses the `default_on_null` parameter of + # snowflake.snowpark.dataframe.pivot() + pd.get_dummies(pd.Series(["a"])).to_pandas() + assert PIVOT_DEFAULT_ON_NULL_WARNING not in caplog.text + assert PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING not in caplog.text diff --git a/tests/integ/scala/test_dataframe_aggregate_suite.py b/tests/integ/scala/test_dataframe_aggregate_suite.py index 5d786975b3b..a5f6acc3b7e 100644 --- a/tests/integ/scala/test_dataframe_aggregate_suite.py +++ b/tests/integ/scala/test_dataframe_aggregate_suite.py @@ -10,7 +10,11 @@ import pytest from snowflake.snowpark import GroupingSets, Row -from snowflake.snowpark._internal.utils import TempObjectType +from snowflake.snowpark._internal.utils import ( + PIVOT_DEFAULT_ON_NULL_WARNING, + PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING, + TempObjectType, +) from snowflake.snowpark.column import Column from snowflake.snowpark.exceptions import ( SnowparkDataframeException, @@ -97,7 +101,7 @@ def test_group_by_pivot(session): ).agg([sum(col("amount")), avg(col("amount"))]) -def test_group_by_pivot_dynamic_any(session): +def test_group_by_pivot_dynamic_any(session, caplog): Utils.check_answer( TestData.monthly_sales_with_team(session) .group_by("empid") @@ -111,6 +115,8 @@ def test_group_by_pivot_dynamic_any(session): sort=False, ) + assert PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING in caplog.text + Utils.check_answer( TestData.monthly_sales_with_team(session) .group_by(["empid", "team"]) @@ -292,7 +298,7 @@ def test_pivot_dynamic_subquery_with_bad_subquery(session): assert "Pivot subquery must select single column" in str(ex_info.value) -def test_pivot_default_on_none(session): +def test_pivot_default_on_none(session, caplog): class MonthlySales(NamedTuple): empid: int amount: int @@ -326,6 +332,8 @@ class MonthlySales(NamedTuple): sort=False, ) + assert PIVOT_DEFAULT_ON_NULL_WARNING in caplog.text + @pytest.mark.localtest def test_rel_grouped_dataframe_agg(session): From 718df6368cc4128e2ac945ee83d30f28da0dc55c Mon Sep 17 00:00:00 2001 From: sfc-gh-mvashishtha Date: Thu, 25 Apr 2024 16:12:35 -0700 Subject: [PATCH 2/5] Fix typo Signed-off-by: sfc-gh-mvashishtha --- .../snowpark/modin/plugin/_internal/ordered_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py b/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py index a69b23403eb..70fb958287c 100644 --- a/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py +++ b/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py @@ -827,7 +827,7 @@ def pivot( # pivot features. The user should have already been warned that # Snowpark pandas is in public or private preview. They likely don't # know or care that we are using Snowpark DataFrame pivot() internally, - # let alone that we are using a private preview features of Snowpark + # let alone that we are using private preview features of Snowpark # Python. class NoPivotWarningFilter(logging.Filter): From 47c269a405ccd6938286f6b4edd6858c600f7963 Mon Sep 17 00:00:00 2001 From: sfc-gh-mvashishtha Date: Thu, 25 Apr 2024 16:54:08 -0700 Subject: [PATCH 3/5] Clear warning_dict before tests and add sql_count_checker Signed-off-by: sfc-gh-mvashishtha --- tests/conftest.py | 10 +++ tests/integ/modin/frame/test_transpose.py | 1 + .../modin/strings/test_get_dummies_series.py | 1 + tests/integ/test_dataframe.py | 4 +- tests/integ/test_pandas_to_df.py | 3 - tests/integ/test_udf.py | 19 ++--- tests/unit/scala/test_utils_suite.py | 76 ++++++++----------- 7 files changed, 51 insertions(+), 63 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index eeb0173d651..9caecda182d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,8 @@ import pytest +from snowflake.snowpark._internal.utils import warning_dict + logging.getLogger("snowflake.connector").setLevel(logging.ERROR) # TODO: SNOW-1305522: Enable Modin doctests for the below frontend files @@ -91,3 +93,11 @@ def cte_optimization_enabled(pytestconfig): def pytest_sessionstart(session): os.environ["SNOWPARK_LOCAL_TESTING_INTERNAL_TELEMETRY"] = "1" + + +@pytest.fixture(autouse=True) +def clear_warning_dict(): + yield + # clear the warning dict so that warnings from one test don't affect + # warnings from other tests. + warning_dict.clear() diff --git a/tests/integ/modin/frame/test_transpose.py b/tests/integ/modin/frame/test_transpose.py index 3f922939808..7e01cf0c1b7 100644 --- a/tests/integ/modin/frame/test_transpose.py +++ b/tests/integ/modin/frame/test_transpose.py @@ -354,6 +354,7 @@ def test_dataframe_transpose_args_warning_log(caplog, score_test_data): ) +@sql_count_checker(query_count=1, union_count=1) def test_transpose_does_not_raise_pivot_warning_snow_1344848(caplog): # Test transpose, which calls snowflake.snowpark.dataframe.pivot() with # the `values` parameter as None or a Snowpark DataFrame. diff --git a/tests/integ/modin/strings/test_get_dummies_series.py b/tests/integ/modin/strings/test_get_dummies_series.py index 6356bc49ad7..401e30ee338 100644 --- a/tests/integ/modin/strings/test_get_dummies_series.py +++ b/tests/integ/modin/strings/test_get_dummies_series.py @@ -58,6 +58,7 @@ def test_get_dummies_series_negative(data): ) +@sql_count_checker(query_count=1) def test_get_dummies_does_not_raise_pivot_warning_snow_1344848(caplog): # Test get_dummies, which uses the `default_on_null` parameter of # snowflake.snowpark.dataframe.pivot() diff --git a/tests/integ/test_dataframe.py b/tests/integ/test_dataframe.py index 2379f371cda..f0aaad082d6 100644 --- a/tests/integ/test_dataframe.py +++ b/tests/integ/test_dataframe.py @@ -30,7 +30,7 @@ from snowflake.snowpark import Column, Row, Window from snowflake.snowpark._internal.analyzer.analyzer_utils import result_scan_statement from snowflake.snowpark._internal.analyzer.expression import Attribute, Interval, Star -from snowflake.snowpark._internal.utils import TempObjectType, warning_dict +from snowflake.snowpark._internal.utils import TempObjectType from snowflake.snowpark.exceptions import ( SnowparkColumnException, SnowparkCreateDynamicTableException, @@ -2719,8 +2719,6 @@ def test_write_temp_table_no_breaking_change( Utils.assert_table_type(session, table_name, "temp") finally: Utils.drop_table(session, table_name) - # clear the warning dict otherwise it will affect the future tests - warning_dict.clear() @pytest.mark.localtest diff --git a/tests/integ/test_pandas_to_df.py b/tests/integ/test_pandas_to_df.py index 712cf1f1588..b0d6b7da81f 100644 --- a/tests/integ/test_pandas_to_df.py +++ b/tests/integ/test_pandas_to_df.py @@ -23,7 +23,6 @@ TempObjectType, is_in_stored_procedure, random_name_for_temp_object, - warning_dict, ) from snowflake.snowpark.exceptions import SnowparkPandasException from tests.utils import Utils @@ -304,8 +303,6 @@ def test_write_temp_table_no_breaking_change(session, table_type, caplog): Utils.assert_table_type(session, table_name, "temp") finally: Utils.drop_table(session, table_name) - # clear the warning dict otherwise it will affect the future tests - warning_dict.clear() @pytest.mark.localtest diff --git a/tests/integ/test_udf.py b/tests/integ/test_udf.py index a76a620401d..d810a67783d 100644 --- a/tests/integ/test_udf.py +++ b/tests/integ/test_udf.py @@ -45,10 +45,7 @@ from snowflake.connector.version import VERSION as SNOWFLAKE_CONNECTOR_VERSION from snowflake.snowpark import Row, Session -from snowflake.snowpark._internal.utils import ( - unwrap_stage_location_single_quote, - warning_dict, -) +from snowflake.snowpark._internal.utils import unwrap_stage_location_single_quote from snowflake.snowpark.exceptions import ( SnowparkInvalidObjectNameException, SnowparkSQLException, @@ -2177,15 +2174,11 @@ def test_deprecate_call_udf_with_list(session, caplog): return_type=IntegerType(), input_types=[IntegerType(), IntegerType()], ) - try: - with caplog.at_level(logging.WARNING): - add_udf(["a", "b"]) - assert ( - "Passing arguments to a UDF with a list or tuple is deprecated" - in caplog.text - ) - finally: - warning_dict.clear() + with caplog.at_level(logging.WARNING): + add_udf(["a", "b"]) + assert ( + "Passing arguments to a UDF with a list or tuple is deprecated" in caplog.text + ) def test_strict_udf(session): diff --git a/tests/unit/scala/test_utils_suite.py b/tests/unit/scala/test_utils_suite.py index f4ad5944867..e483877025c 100644 --- a/tests/unit/scala/test_utils_suite.py +++ b/tests/unit/scala/test_utils_suite.py @@ -28,7 +28,6 @@ unwrap_stage_location_single_quote, validate_object_name, warning, - warning_dict, zip_file_or_directory_to_stream, ) from tests.utils import IS_WINDOWS, TestFiles @@ -445,43 +444,36 @@ def test_warning(caplog): def f(): return 1 - try: - with caplog.at_level(logging.WARNING): - warning("aaa", "bbb", 2) - warning("aaa", "bbb", 2) - warning("aaa", "bbb", 2) - assert caplog.text.count("bbb") == 2 - with caplog.at_level(logging.WARNING): - warning(f.__qualname__, "ccc", 2) - warning(f.__qualname__, "ccc", 2) - warning(f.__qualname__, "ccc", 2) - assert caplog.text.count("ccc") == 2 - finally: - warning_dict.clear() + with caplog.at_level(logging.WARNING): + warning("aaa", "bbb", 2) + warning("aaa", "bbb", 2) + warning("aaa", "bbb", 2) + assert caplog.text.count("bbb") == 2 + with caplog.at_level(logging.WARNING): + warning(f.__qualname__, "ccc", 2) + warning(f.__qualname__, "ccc", 2) + warning(f.__qualname__, "ccc", 2) + assert caplog.text.count("ccc") == 2 @pytest.mark.parametrize("decorator", [deprecated, experimental]) def test_func_decorator(caplog, decorator): - try: - - @decorator( - version="1.0.0", - extra_warning_text="extra_warning_text", - extra_doc_string="extra_doc_string", - ) - def f(): - return 1 + @decorator( + version="1.0.0", + extra_warning_text="extra_warning_text", + extra_doc_string="extra_doc_string", + ) + def f(): + return 1 - assert "extra_doc_string" in f.__doc__ - with caplog.at_level(logging.WARNING): - f() - f() + assert "extra_doc_string" in f.__doc__ + with caplog.at_level(logging.WARNING): + f() + f() - assert decorator.__name__ in caplog.text - assert caplog.text.count("1.0.0") == 1 - assert caplog.text.count("extra_warning_text") == 1 - finally: - warning_dict.clear() + assert decorator.__name__ in caplog.text + assert caplog.text.count("1.0.0") == 1 + assert caplog.text.count("extra_warning_text") == 1 def test_is_sql_select_statement(): @@ -564,18 +556,14 @@ def foo(): pass caplog.clear() - warning_dict.clear() - try: - with caplog.at_level(logging.WARNING): - foo() - assert extra_doc in foo.__doc__ - assert expected_warning_text in caplog.messages - caplog.clear() - with caplog.at_level(logging.WARNING): - foo() - assert expected_warning_text not in caplog.text - finally: - warning_dict.clear() + with caplog.at_level(logging.WARNING): + foo() + assert extra_doc in foo.__doc__ + assert expected_warning_text in caplog.messages + caplog.clear() + with caplog.at_level(logging.WARNING): + foo() + assert expected_warning_text not in caplog.text @pytest.mark.parametrize("function", [result_set_to_iter, result_set_to_rows]) From f643c84c298f1e20577b9957d37a243857dc879e Mon Sep 17 00:00:00 2001 From: sfc-gh-mvashishtha Date: Mon, 29 Apr 2024 16:46:22 -0700 Subject: [PATCH 4/5] Skip pivot warnings for snowpark pandas users, and improve messages Signed-off-by: sfc-gh-mvashishtha --- src/snowflake/snowpark/_internal/utils.py | 33 ++++++++------ .../snowpark/modin/plugin/__init__.py | 13 ++++++ .../plugin/_internal/ordered_dataframe.py | 45 ++++--------------- .../scala/test_dataframe_aggregate_suite.py | 11 ++++- 4 files changed, 50 insertions(+), 52 deletions(-) diff --git a/src/snowflake/snowpark/_internal/utils.py b/src/snowflake/snowpark/_internal/utils.py index e20e7bb9225..ff6689ab75e 100644 --- a/src/snowflake/snowpark/_internal/utils.py +++ b/src/snowflake/snowpark/_internal/utils.py @@ -182,8 +182,12 @@ SUPPORTED_TABLE_TYPES = ["temp", "temporary", "transient"] -PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING = "Parameter values is Optional or DataFrame is in private preview since v1.15.0. Do not use it in production." -PIVOT_DEFAULT_ON_NULL_WARNING = "Parameter default_on_null is not None is in private preview since v1.15.0. Do not use it in production." +PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING = ( + "Calling pivot() with the `value` parameter set to None or to a Snowpark " + + "DataFrame is in private preview since v1.15.0. Do not use this feature " + + "in production." +) +PIVOT_DEFAULT_ON_NULL_WARNING = "Calling pivot() with the default_on_null is not None is in private preview since v1.15.0. Do not use it in production." class TempObjectType(Enum): @@ -879,6 +883,9 @@ def escape_quotes(unescaped: str) -> str: return unescaped.replace(DOUBLE_QUOTE, DOUBLE_QUOTE + DOUBLE_QUOTE) +should_warn_dynamic_pivot_is_in_private_preview = True + + def prepare_pivot_arguments( df: "snowflake.snowpark.DataFrame", df_name: str, @@ -899,17 +906,17 @@ def prepare_pivot_arguments( """ from snowflake.snowpark.dataframe import DataFrame - if values is None or isinstance(values, DataFrame): - warning( - df_name, - PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING, - ) - - if default_on_null is not None: - warning( - df_name, - PIVOT_DEFAULT_ON_NULL_WARNING, - ) + if should_warn_dynamic_pivot_is_in_private_preview: + if values is None or isinstance(values, DataFrame): + warning( + df_name, + PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING, + ) + if default_on_null is not None: + warning( + df_name, + PIVOT_DEFAULT_ON_NULL_WARNING, + ) if values is not None and not values: raise ValueError("values cannot be empty") diff --git a/src/snowflake/snowpark/modin/plugin/__init__.py b/src/snowflake/snowpark/modin/plugin/__init__.py index bd7cab8b4d9..35042684921 100644 --- a/src/snowflake/snowpark/modin/plugin/__init__.py +++ b/src/snowflake/snowpark/modin/plugin/__init__.py @@ -7,6 +7,8 @@ from packaging import version +import snowflake.snowpark._internal.utils + if sys.version_info.major == 3 and sys.version_info.minor == 8: raise RuntimeError( "Snowpark pandas does not support Python 3.8. Please update to Python 3.9 or later." @@ -60,3 +62,14 @@ from snowflake.snowpark.modin.plugin import docstrings # isort: skip # noqa: E402 DocModule.put(docstrings.__name__) + + +# Don't warn the user about our internal usage of private preview pivot +# features. The user should have already been warned that Snowpark pandas +# is in public or private preview. They likely don't know or care that we are +# using Snowpark DataFrame pivot() internally, let alone that we are using +# private preview features of Snowpark Python. + +snowflake.snowpark._internal.utils.should_warn_dynamic_pivot_is_in_private_preview = ( + False +) diff --git a/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py b/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py index 70fb958287c..08fed491a3b 100644 --- a/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py +++ b/src/snowflake/snowpark/modin/plugin/_internal/ordered_dataframe.py @@ -15,11 +15,7 @@ ColumnOrSqlExpr, LiteralType, ) -from snowflake.snowpark._internal.utils import ( - PIVOT_DEFAULT_ON_NULL_WARNING, - PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING, - parse_positional_args_to_list, -) +from snowflake.snowpark._internal.utils import parse_positional_args_to_list from snowflake.snowpark.column import Column from snowflake.snowpark.dataframe import DataFrame as SnowparkDataFrame from snowflake.snowpark.dataframe_writer import DataFrameWriter @@ -822,41 +818,16 @@ def pivot( See detailed docstring in Snowpark DataFrame's pivot. """ snowpark_dataframe = self.to_projected_snowpark_dataframe() - - # Don't warn the user about our internal usage of private preview - # pivot features. The user should have already been warned that - # Snowpark pandas is in public or private preview. They likely don't - # know or care that we are using Snowpark DataFrame pivot() internally, - # let alone that we are using private preview features of Snowpark - # Python. - - class NoPivotWarningFilter(logging.Filter): - def filter(self, record: logging.LogRecord) -> bool: - message = record.getMessage() - return ( - PIVOT_DEFAULT_ON_NULL_WARNING not in message - and PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING not in message - ) - - filter = NoPivotWarningFilter() - - class NoPivotWarningContext: - def __enter__(self): - logging.getLogger("snowflake.snowpark").addFilter(filter) - - def __exit__(self, exc_type, exc_val, exc_tb): - logging.getLogger("snowflake.snowpark").removeFilter(filter) - - with NoPivotWarningContext(): - pivoted_snowpark_dataframe = snowpark_dataframe.pivot( - pivot_col=pivot_col, - values=values, - default_on_null=default_on_null, - ) return OrderedDataFrame( # the pivot result columns for dynamic pivot are data dependent, a schema call is required # to know all the quoted identifiers for the pivot result. - DataFrameReference(pivoted_snowpark_dataframe.agg(*agg_exprs)) + DataFrameReference( + snowpark_dataframe.pivot( + pivot_col=pivot_col, + values=values, + default_on_null=default_on_null, + ).agg(*agg_exprs) + ) ) def unpivot( diff --git a/tests/integ/scala/test_dataframe_aggregate_suite.py b/tests/integ/scala/test_dataframe_aggregate_suite.py index a5f6acc3b7e..5bdf12a7846 100644 --- a/tests/integ/scala/test_dataframe_aggregate_suite.py +++ b/tests/integ/scala/test_dataframe_aggregate_suite.py @@ -3,6 +3,7 @@ # Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. # +import sys from decimal import Decimal from math import sqrt from typing import NamedTuple @@ -115,7 +116,10 @@ def test_group_by_pivot_dynamic_any(session, caplog): sort=False, ) - assert PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING in caplog.text + if "snowflake.snowpark.modin.plugin" not in sys.modules: + # Snowpark pandas users don't get warnings about dynamic pivot + # features. See SNOW-1344848. + assert PIVOT_VALUES_NONE_OR_DATAFRAME_WARNING in caplog.text Utils.check_answer( TestData.monthly_sales_with_team(session) @@ -332,7 +336,10 @@ class MonthlySales(NamedTuple): sort=False, ) - assert PIVOT_DEFAULT_ON_NULL_WARNING in caplog.text + if "snowflake.snowpark.modin.plugin" not in sys.modules: + # Snowpark pandas users don't get warnings about dynamic pivot + # features. See SNOW-1344848. + assert PIVOT_DEFAULT_ON_NULL_WARNING in caplog.text @pytest.mark.localtest From 36e801bb034d70de861cd74f22083a3d3fbaa9fd Mon Sep 17 00:00:00 2001 From: sfc-gh-mvashishtha Date: Mon, 29 Apr 2024 16:49:29 -0700 Subject: [PATCH 5/5] Fix warning text Signed-off-by: sfc-gh-mvashishtha --- src/snowflake/snowpark/_internal/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/snowflake/snowpark/_internal/utils.py b/src/snowflake/snowpark/_internal/utils.py index ff6689ab75e..daf953888b8 100644 --- a/src/snowflake/snowpark/_internal/utils.py +++ b/src/snowflake/snowpark/_internal/utils.py @@ -187,7 +187,10 @@ + "DataFrame is in private preview since v1.15.0. Do not use this feature " + "in production." ) -PIVOT_DEFAULT_ON_NULL_WARNING = "Calling pivot() with the default_on_null is not None is in private preview since v1.15.0. Do not use it in production." +PIVOT_DEFAULT_ON_NULL_WARNING = ( + "Calling pivot() with a non-None value for `default_on_null` is in " + + "private preview since v1.15.0. Do not use this feature in production." +) class TempObjectType(Enum):