Skip to content

Commit

Permalink
nit
Browse files Browse the repository at this point in the history
  • Loading branch information
turboFei committed Dec 21, 2023
1 parent 31d4fc1 commit 4219d28
Showing 1 changed file with 3 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -183,18 +183,14 @@ class ExecuteStatement(
// Rename all col name to avoid duplicate columns
val colName = range(0, result.schema.size).map(x => "col" + x)

val compress = if (SPARK_ENGINE_RUNTIME_VERSION >= "3.2") {
"zstd"
} else {
"zlib"
}
val codec = if (SPARK_ENGINE_RUNTIME_VERSION >= "3.2") "zstd" else "zlib"
// df.write will introduce an extra shuffle for the outermost limit, and hurt performance
if (resultMaxRows > 0) {
result.toDF(colName: _*).limit(resultMaxRows).write
.option("compression", compress).format("orc").save(saveFileName.get)
.option("compression", codec).format("orc").save(saveFileName.get)
} else {
result.toDF(colName: _*).write
.option("compression", compress).format("orc").save(saveFileName.get)
.option("compression", codec).format("orc").save(saveFileName.get)
}
info(s"Save result to $saveFileName")
fetchOrcStatement = Some(new FetchOrcStatement(spark))
Expand Down

0 comments on commit 4219d28

Please sign in to comment.