From bea0338eaf9fc45caa5116a7d220fb5e5358bd75 Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 10:26:24 -0700 Subject: [PATCH 1/9] Ensure pip install -e '.[modin-development]' install all necessary dependencies Signed-off-by: Naren Krishna --- .github/workflows/precommit.yml | 2 +- setup.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 4501a6f76c8..b9537855810 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -531,7 +531,7 @@ jobs: - name: Upgrade setuptools and pip run: python -m pip install -U setuptools pip - name: Install Snowpark - run: python -m pip install ".[modin-development, development, pandas]" + run: python -m pip install ".[modin-development]" - name: Install Sphinx run: python -m pip install sphinx - name: Build document diff --git a/setup.py b/setup.py index f4641a62811..8c9e2331470 100644 --- a/setup.py +++ b/setup.py @@ -114,6 +114,14 @@ "pre-commit", ], "modin-development": [ + f"snowflake-connector-python[pandas]{CONNECTOR_DEPENDENCY_VERSION}", + "pytest<8.0.0", # check SNOW-1022240 for more details on the pin here + "pytest-cov", + "coverage", + "sphinx==5.0.2", + "cachetools", # used in UDF doctest + "pytest-timeout", + "pre-commit", "pytest-assume", # Snowpark pandas "decorator", # Snowpark pandas "scipy", # Snowpark pandas 3rd party library testing From 57a1d31ffbf1372be86dc4d28da17e1d807dded2 Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 10:31:11 -0700 Subject: [PATCH 2/9] Add CODEOWNERS Signed-off-by: Naren Krishna --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8c4aff4faa9..7206f76db6d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,3 +2,4 @@ /src/snowflake/snowpark/modin/ @snowflakedb/snowpandas /tests/integ/modin/ @snowflakedb/snowpandas /tests/unit/modin/ @snowflakedb/snowpandas +setup.py @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers From 63d61b2d50752ae35537d7fb5310a7dfa45c1cde Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 10:33:08 -0700 Subject: [PATCH 3/9] add precommit.yml to both team CODEOWNERS Signed-off-by: Naren Krishna --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7206f76db6d..6d06692edf4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,5 @@ /src/snowflake/snowpark/modin/ @snowflakedb/snowpandas /tests/integ/modin/ @snowflakedb/snowpandas /tests/unit/modin/ @snowflakedb/snowpandas +/.github/worfklows/precommit.yml @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers setup.py @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers From d2d1721968618e558194eb35d167b54b04394ed0 Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 10:35:59 -0700 Subject: [PATCH 4/9] add .github to CODEOWNERS for both teams Signed-off-by: Naren Krishna --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6d06692edf4..d011b0b8cc6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,5 +2,5 @@ /src/snowflake/snowpark/modin/ @snowflakedb/snowpandas /tests/integ/modin/ @snowflakedb/snowpandas /tests/unit/modin/ @snowflakedb/snowpandas -/.github/worfklows/precommit.yml @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers +/.github/ @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers setup.py @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers From 8d0f6f5094ba7f717236808440865ffa05c76e58 Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 10:48:37 -0700 Subject: [PATCH 5/9] add scripts/ to CODEOWNERS for both teams Signed-off-by: Naren Krishna --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d011b0b8cc6..647a5a3fd8c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,4 +3,5 @@ /tests/integ/modin/ @snowflakedb/snowpandas /tests/unit/modin/ @snowflakedb/snowpandas /.github/ @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers +/scripts/ @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers setup.py @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers From 3425f4e00672109e46e2ee7abb989aae7bc356e1 Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 13:03:59 -0700 Subject: [PATCH 6/9] update setup.py with development requirements Signed-off-by: Naren Krishna --- setup.py | 30 ++++++++++++++---------------- tox.ini | 5 +---- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index 8c9e2331470..7e2ed193f9a 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,16 @@ PANDAS_REQUIREMENTS = [ f"snowflake-connector-python[pandas]{CONNECTOR_DEPENDENCY_VERSION}", ] +DEVELOPMENT_REQUIREMENTS = [ + "pytest<8.0.0", # check SNOW-1022240 for more details on the pin here + "pytest-cov", + "coverage", + "sphinx==5.0.2", + "cachetools", # used in UDF doctest + "pytest-timeout", + "pytest-xdist", + "pre-commit", +] # read the version VERSION = () @@ -105,28 +115,16 @@ f"snowflake-connector-python[secure-local-storage]{CONNECTOR_DEPENDENCY_VERSION}", ], "development": [ - "pytest<8.0.0", # check SNOW-1022240 for more details on the pin here - "pytest-cov", - "coverage", - "sphinx==5.0.2", - "cachetools", # used in UDF doctest - "pytest-timeout", - "pre-commit", + *DEVELOPMENT_REQUIREMENTS, ], "modin-development": [ - f"snowflake-connector-python[pandas]{CONNECTOR_DEPENDENCY_VERSION}", - "pytest<8.0.0", # check SNOW-1022240 for more details on the pin here - "pytest-cov", - "coverage", - "sphinx==5.0.2", - "cachetools", # used in UDF doctest - "pytest-timeout", - "pre-commit", + *PANDAS_REQUIREMENTS, + *DEVELOPMENT_REQUIREMENTS, + f"modin{MODIN_DEPENDENCY_VERSION}", "pytest-assume", # Snowpark pandas "decorator", # Snowpark pandas "scipy", # Snowpark pandas 3rd party library testing "statsmodels", # Snowpark pandas 3rd party library testing - f"modin{MODIN_DEPENDENCY_VERSION}", ], "localtest": [ "pandas", diff --git a/tox.ini b/tox.ini index 51a38c99efb..f18849e2b2b 100644 --- a/tox.ini +++ b/tox.ini @@ -104,10 +104,7 @@ commands = {env:SNOWFLAKE_PYTEST_DAILY_CMD} -vvv -m "integ or unit" {posargs:} [testenv:modin_extra_without_pandas_extra] allowlist_externals = bash description = SNOW-1345421: test that we can use modin by specifying the modin extra but not the pandas extra. -deps = - .[development] - .[modin-development] - .[modin] +deps = .[modin-development] commands = pytest {env:SNOWFLAKE_PYTEST_VERBOSITY:} {env:SNOWFLAKE_PYTEST_COV_CMD} {posargs:} tests/integ/modin/test_modin_extra_without_pandas_extra.py [testenv:coverage] From ba9fc492fb5231a29faf36c4fd9d454cb2852616 Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 13:06:07 -0700 Subject: [PATCH 7/9] add tox.ini to CODENOWNERS Signed-off-by: Naren Krishna --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 647a5a3fd8c..0f361c3087b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,3 +5,4 @@ /.github/ @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers /scripts/ @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers setup.py @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers +tox.ini @snowflakedb/snowpandas @snowflakedb/snowpark-python-api-reviewers From 031918eb9261b6b358b3ec36bbe1b26165243ec0 Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 13:09:28 -0700 Subject: [PATCH 8/9] add MODIN_REQUIREMENTS Signed-off-by: Naren Krishna --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 7e2ed193f9a..cd77bbe691f 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,10 @@ PANDAS_REQUIREMENTS = [ f"snowflake-connector-python[pandas]{CONNECTOR_DEPENDENCY_VERSION}", ] +MODIN_REQUIREMENTS = [ + *PANDAS_REQUIREMENTS, + f"modin{MODIN_DEPENDENCY_VERSION}", +] DEVELOPMENT_REQUIREMENTS = [ "pytest<8.0.0", # check SNOW-1022240 for more details on the pin here "pytest-cov", @@ -107,10 +111,7 @@ }, extras_require={ "pandas": PANDAS_REQUIREMENTS, - "modin": [ - f"modin{MODIN_DEPENDENCY_VERSION}", - *PANDAS_REQUIREMENTS, - ], + "modin": MODIN_REQUIREMENTS, "secure-local-storage": [ f"snowflake-connector-python[secure-local-storage]{CONNECTOR_DEPENDENCY_VERSION}", ], @@ -118,9 +119,8 @@ *DEVELOPMENT_REQUIREMENTS, ], "modin-development": [ - *PANDAS_REQUIREMENTS, + *MODIN_REQUIREMENTS, *DEVELOPMENT_REQUIREMENTS, - f"modin{MODIN_DEPENDENCY_VERSION}", "pytest-assume", # Snowpark pandas "decorator", # Snowpark pandas "scipy", # Snowpark pandas 3rd party library testing From ad93580c93c65c1bd08cb39902cc4b6744e5b007 Mon Sep 17 00:00:00 2001 From: Naren Krishna Date: Thu, 25 Apr 2024 13:10:59 -0700 Subject: [PATCH 9/9] remove list development Signed-off-by: Naren Krishna --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index cd77bbe691f..c6fca6c4256 100644 --- a/setup.py +++ b/setup.py @@ -115,9 +115,7 @@ "secure-local-storage": [ f"snowflake-connector-python[secure-local-storage]{CONNECTOR_DEPENDENCY_VERSION}", ], - "development": [ - *DEVELOPMENT_REQUIREMENTS, - ], + "development": DEVELOPMENT_REQUIREMENTS, "modin-development": [ *MODIN_REQUIREMENTS, *DEVELOPMENT_REQUIREMENTS,