Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update the spark version to the current version #1055

Merged
merged 4 commits into from
Jun 26, 2024
Merged

Update the spark version to the current version #1055

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
892cc8c
update the spark version to the current version
mikealfare Jun 21, 2024
ad4aaa1
Merge branch 'main' into fix-spacy-dep-tests
mikealfare Jun 24, 2024
021809a
update pin for pydanic to resolve https://github.com/explosion/spaCy/…
mikealfare Jun 24, 2024
9c033f0
exclude koalas dataframes from test
mikealfare Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions tests/functional/adapter/test_python_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,22 @@ class TestPythonModelSpark(BasePythonModelTests):

@pytest.mark.skip_profile("apache_spark", "spark_session", "databricks_sql_endpoint")
class TestPySpark(BasePySparkTests):
@pytest.mark.skip("https://github.com/dbt-labs/dbt-spark/issues/1054")
def test_different_dataframes(self, project):
return super().test_different_dataframes(project)
"""
Test that python models are supported using dataframes from:
- pandas
- pyspark
- pyspark.pandas (formerly dataspark.koalas)

Note:
The CI environment is on Apache Spark >3.1, which includes koalas as pyspark.pandas.
The only Databricks runtime that supports Apache Spark <=3.1 is 9.1 LTS, which is EOL 2024-09-23.
For more information, see:
- https://github.com/databricks/koalas
- https://docs.databricks.com/en/release-notes/runtime/index.html
"""
results = run_dbt(["run", "--exclude", "koalas_df"])
assert len(results) == 3


@pytest.mark.skip_profile("apache_spark", "spark_session", "databricks_sql_endpoint")
Expand All @@ -37,7 +50,7 @@ def model(dbt, spark):
materialized='table',
submission_method='job_cluster',
job_cluster_config={
"spark_version": "7.3.x-scala2.12",
"spark_version": "12.2.x-scala2.12",
"node_type_id": "i3.xlarge",
"num_workers": 0,
"spark_conf": {
Expand All @@ -48,7 +61,7 @@ def model(dbt, spark):
"ResourceClass": "SingleNode"
}
},
packages=['spacy', 'torch', 'pydantic<1.10.3']
packages=['spacy', 'torch', 'pydantic>=1.10.8']
)
data = [[1,2]] * 10
return spark.createDataFrame(data, schema=['test', 'test2'])
Expand All @@ -67,11 +80,23 @@ def model(dbt, spark):

@pytest.mark.skip_profile("apache_spark", "spark_session", "databricks_sql_endpoint")
class TestChangingSchemaSpark:
"""
Confirm that we can setup a spot instance and parse required packages into the Databricks job.

Notes:
- This test generates a spot instance on demand using the settings from `job_cluster_config`
in `models__simple_python_model` above. It takes several minutes to run due to creating the cluster.
The job can be monitored via "Data Engineering > Job Runs" or "Workflows > Job Runs"
in the Databricks UI (instead of via the normal cluster).
- The `spark_version` argument will need to periodically be updated. It will eventually become
unsupported and start experiencing issues.
- See https://github.com/explosion/spaCy/issues/12659 for why we're pinning pydantic
"""

@pytest.fixture(scope="class")
def models(self):
return {"simple_python_model.py": models__simple_python_model}

@pytest.mark.skip("https://github.com/dbt-labs/dbt-spark/issues/1054")
def test_changing_schema_with_log_validation(self, project, logs_dir):
run_dbt(["run"])
write_file(
Expand Down
Loading