From dc5c49244ec1b9ad5d49b41c30f7f410b680e6f6 Mon Sep 17 00:00:00 2001 From: Ilias Xenogiannis Date: Mon, 11 Nov 2024 11:38:11 +0200 Subject: [PATCH] Update spark-defaults.conf --- .../spark_deployment/spark-defaults.conf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/spark_deployment/spark-defaults.conf b/.github/workflows/spark_deployment/spark-defaults.conf index 81c63b3..6534b55 100644 --- a/.github/workflows/spark_deployment/spark-defaults.conf +++ b/.github/workflows/spark_deployment/spark-defaults.conf @@ -7,6 +7,24 @@ spark.sql.catalog.glue.io-impl org.apache.iceberg.aws.s3.S3FileI spark.sql.defaultCatalog glue spark.sql.catalog.glue.database dbt-spark-iceberg +# Table capabilities and operation settings +spark.sql.catalog.glue.table-default.format-version 2 +spark.sql.catalog.glue.table-default.write.update.mode merge-on-read +spark.sql.catalog.glue.table-default.write.delete.mode merge-on-read +spark.sql.catalog.glue.table-default.write.operation.mode batch +spark.sql.catalog.glue.table-default.write.merge.mode merge-on-read +spark.sql.catalog.glue.table-default.write.distribution-mode none +spark.sql.catalog.glue.table-default.write.data.path s3a://dbt-spark-iceberg/github-integration-testing +spark.sql.catalog.glue.table-default.write.metadata.path s3a://dbt-spark-iceberg/github-integration-testing/metadata +spark.sql.catalog.glue.table-default.write.metadata.previous-versions-max 10 +spark.sql.catalog.glue.table-default.write.format.default iceberg +spark.sql.catalog.glue.table-default.engine.hive.enabled true +spark.sql.table.is.transactional true +spark.sql.iceberg.check-nullability false +spark.sql.iceberg.vectorization.enabled true +spark.sql.iceberg.allow-append-in-batch-mode true +spark.sql.iceberg.handle-timestamp-without-timezone true + # Performance Tuning spark.master local[3] spark.driver.memory 10g