Skip to content

Commit

Permalink
enable parquet prefetch (lakesoul-io#350)
Browse files Browse the repository at this point in the history
Signed-off-by: chenxu <[email protected]>
Co-authored-by: chenxu <[email protected]>
  • Loading branch information
xuchen-plus and dmetasoul01 authored Oct 12, 2023
1 parent c4e9ba3 commit 69cc485
Show file tree
Hide file tree
Showing 6 changed files with 367 additions and 392 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import org.apache.spark.sql.lakesoul.sources.LakeSoulSQLConf
* Run with following commands with local minio env:
*
* mvn package -Prelease-linux-x86-64 -pl lakesoul-spark -am -DskipTests
* docker run --rm -ti --net host -v /opt/spark/work-dir/data:/opt/spark/work-dir/data -v $PWD/lakesoul-spark/target:/opt/spark/work-dir/jars bitnami/spark:3.3.1 spark-submit --driver-memory 4g --jars /opt/spark/work-dir/jars/lakesoul-spark-2.2.0-spark-3.3-SNAPSHOT.jar --class org.apache.spark.sql.lakesoul.benchmark.io.ParquetScanBenchmark /opt/spark/work-dir/jars/lakesoul-spark-2.2.0-spark-3.3-SNAPSHOT-tests.jar --localtest
* docker run --cpus=1 --rm -ti --net host -v /opt/spark/work-dir/data:/opt/spark/work-dir/data -v $PWD/lakesoul-spark/target:/opt/spark/work-dir/jars bitnami/spark:3.3.1 spark-submit --driver-memory 4g --jars /opt/spark/work-dir/jars/lakesoul-spark-2.4.0-spark-3.3-SNAPSHOT.jar --class org.apache.spark.sql.lakesoul.benchmark.io.ParquetScanBenchmark /opt/spark/work-dir/jars/lakesoul-spark-2.4.0-spark-3.3-SNAPSHOT-tests.jar --localtest
*/
object ParquetScanBenchmark {
def main(args: Array[String]): Unit = {
Expand Down Expand Up @@ -44,7 +44,7 @@ object ParquetScanBenchmark {

var bucketName = "lakesoul-test-bucket"
if (args.length >= 1 && args(0) == "--localtest") {
builder.config("spark.hadoop.fs.s3a.endpoint", "http://minio:9000")
builder.config("spark.hadoop.fs.s3a.endpoint", "http://localhost:9000")
.config("spark.hadoop.fs.s3a.endpoint.region", "us-east-1")
.config("spark.hadoop.fs.s3a.access.key", "minioadmin1")
.config("spark.hadoop.fs.s3a.secret.key", "minioadmin1")
Expand Down
Loading

0 comments on commit 69cc485

Please sign in to comment.