Skip to content

Commit

Permalink
update release version to 2.4.1
Browse files Browse the repository at this point in the history
Signed-off-by: chenxu <[email protected]>
  • Loading branch information
dmetasoul01 committed Oct 12, 2023
1 parent 28ac174 commit 349c510
Show file tree
Hide file tree
Showing 23 changed files with 60 additions and 60 deletions.
4 changes: 2 additions & 2 deletions lakesoul-common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ SPDX-License-Identifier: Apache-2.0
<parent>
<artifactId>lakesoul-parent</artifactId>
<groupId>com.dmetasoul</groupId>
<version>2.4.0</version>
<version>2.4.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>lakesoul-common</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
<build>
<plugins>
<plugin>
Expand Down
8 changes: 4 additions & 4 deletions lakesoul-flink/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ SPDX-License-Identifier: Apache-2.0
<parent>
<artifactId>lakesoul-parent</artifactId>
<groupId>com.dmetasoul</groupId>
<version>2.4.0</version>
<version>2.4.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>lakesoul-flink</artifactId>
<version>2.4.0-flink-1.17</version>
<version>2.4.1-flink-1.17</version>
<properties>
<flink.version>1.17.1</flink.version>
<scala.binary.version>2.12</scala.binary.version>
Expand All @@ -30,7 +30,7 @@ SPDX-License-Identifier: Apache-2.0
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-common</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
Expand All @@ -41,7 +41,7 @@ SPDX-License-Identifier: Apache-2.0
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-io-java</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
Expand Down
8 changes: 4 additions & 4 deletions lakesoul-presto/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
<parent>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-parent</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
</parent>

<artifactId>lakesoul-presto</artifactId>
<version>2.4.0-presto-0.28</version>
<version>2.4.1-presto-0.28</version>

<properties>
<maven.compiler.source>8</maven.compiler.source>
Expand All @@ -33,12 +33,12 @@
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-common</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
</dependency>
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-io-java</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
Expand Down
8 changes: 4 additions & 4 deletions lakesoul-spark/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ SPDX-License-Identifier: Apache-2.0
<parent>
<artifactId>lakesoul-parent</artifactId>
<groupId>com.dmetasoul</groupId>
<version>2.4.0</version>
<version>2.4.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>lakesoul-spark</artifactId>
<version>2.4.0-spark-3.3</version>
<version>2.4.1-spark-3.3</version>

<properties>
<scala.version>2.12.10</scala.version>
Expand All @@ -37,12 +37,12 @@ SPDX-License-Identifier: Apache-2.0
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-common</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
</dependency>
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-io-java</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
Expand Down
4 changes: 2 additions & 2 deletions native-io/lakesoul-io-java/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ SPDX-License-Identifier: Apache-2.0
<parent>
<artifactId>lakesoul-parent</artifactId>
<groupId>com.dmetasoul</groupId>
<version>2.4.0</version>
<version>2.4.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>

<modelVersion>4.0.0</modelVersion>
<artifactId>lakesoul-io-java</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
<packaging>jar</packaging>

<properties>
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0
<modelVersion>4.0.0</modelVersion>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-parent</artifactId>
<version>2.4.0</version>
<version>2.4.1</version>
<inceptionYear>2022</inceptionYear>
<modules>
<module>lakesoul-common</module>
Expand Down
2 changes: 1 addition & 1 deletion website/docs/01-Getting Started/01-setup-local-env.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ After unpacking spark package, you could find LakeSoul distribution jar from htt
wget https://dmetasoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/releases/spark/spark-3.3.2-bin-hadoop-3.tgz
tar xf spark-3.3.2-bin-hadoop-3.tgz
export SPARK_HOME=${PWD}/spark-3.3.2-bin-hadoop3
wget https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-spark-2.4.0-spark-3.3.jar -P $SPARK_HOME/jars
wget https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-spark-2.4.1-spark-3.3.jar -P $SPARK_HOME/jars
```

:::tip
Expand Down
2 changes: 1 addition & 1 deletion website/docs/01-Getting Started/02-docker-compose.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ docker run --net lakesoul-docker-compose-env_default --rm -ti \
-v $(pwd)/lakesoul.properties:/opt/spark/work-dir/lakesoul.properties \
--env lakesoul_home=/opt/spark/work-dir/lakesoul.properties bitnami/spark:3.3.1 \
spark-shell \
--packages com.dmetasoul:lakesoul-spark:2.4.0-spark-3.3 \
--packages com.dmetasoul:lakesoul-spark:2.4.1-spark-3.3 \
--conf spark.sql.extensions=com.dmetasoul.lakesoul.sql.LakeSoulSparkSessionExtension \
--conf spark.sql.catalog.lakesoul=org.apache.spark.sql.lakesoul.catalog.LakeSoulCatalog \
--conf spark.sql.defaultCatalog=lakesoul \
Expand Down
12 changes: 6 additions & 6 deletions website/docs/03-Usage Docs/02-setup-spark.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ To use `spark-shell`, `pyspark` or `spark-sql` shells, you should include LakeSo

#### Use Maven Coordinates via --packages
```bash
spark-shell --packages com.dmetasoul:lakesoul-spark:2.4.0-spark-3.3
spark-shell --packages com.dmetasoul:lakesoul-spark:2.4.1-spark-3.3
```

#### Use Local Packages
You can find the LakeSoul packages from our release page: [Releases](https://github.com/lakesoul-io/LakeSoul/releases).
Download the jar file and pass it to `spark-submit`.
```bash
spark-submit --jars "lakesoul-spark-2.4.0-spark-3.3.jar"
spark-submit --jars "lakesoul-spark-2.4.1-spark-3.3.jar"
```

Or you could directly put the jar into `$SPARK_HOME/jars`
Expand All @@ -34,7 +34,7 @@ Include maven dependencies in your project:
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul</artifactId>
<version>2.4.0-spark-3.3</version>
<version>2.4.1-spark-3.3</version>
</dependency>
```

Expand Down Expand Up @@ -93,7 +93,7 @@ spark.sql.sources.default lakesoul
## Setup Flink Project or Job

### Required Flink Version
Since 2.4.0, Flink version 1.17 is supported.
Since 2.4.1, Flink version 1.17 is supported.

### Setup Metadata Database Connection for Flink

Expand Down Expand Up @@ -144,7 +144,7 @@ taskmanager.memory.task.off-heap.size: 3000m
:::
### Add LakeSoul Jar to Flink's directory
Download LakeSoul Flink Jar from: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-flink-2.4.0-flink-1.17.jar
Download LakeSoul Flink Jar from: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-flink-2.4.1-flink-1.17.jar
And put the jar file under `$FLINK_HOME/lib`. After this, you could start flink session cluster or application as usual.

Expand All @@ -155,6 +155,6 @@ Add the following to your project's pom.xml
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul</artifactId>
<version>2.4.0-flink-1.17</version>
<version>2.4.1-flink-1.17</version>
</dependency>
```
6 changes: 3 additions & 3 deletions website/docs/03-Usage Docs/05-flink-cdc-sync.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ In the Stream API, the main functions of LakeSoul Sink are:

## How to use the command line
### 1. Download LakeSoul Flink Jar
It can be downloaded from the LakeSoul Release page: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-flink-2.4.0-flink-1.17.jar.
It can be downloaded from the LakeSoul Release page: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-flink-2.4.1-flink-1.17.jar.

The currently supported Flink version is 1.17.

Expand Down Expand Up @@ -60,7 +60,7 @@ export LAKESOUL_PG_PASSWORD=root
#### 2.2 Start sync job
```bash
bin/flink run -c org.apache.flink.lakesoul.entry.MysqlCdc \
lakesoul-flink-2.4.0-flink-1.17.jar \
lakesoul-flink-2.4.1-flink-1.17.jar \
--source_db.host localhost \
--source_db.port 3306 \
--source_db.db_name default \
Expand All @@ -79,7 +79,7 @@ Description of required parameters:
| Parameter | Meaning | Value Description |
|----------------|------------------------------------|-------------------------------------------- |
| -c | The task runs the main function entry class | org.apache.flink.lakesoul.entry.MysqlCdc |
| Main package | Task running jar | lakesoul-flink-2.4.0-flink-1.17.jar |
| Main package | Task running jar | lakesoul-flink-2.4.1-flink-1.17.jar |
| --source_db.host | The address of the MySQL database | |
| --source_db.port | MySQL database port | |
| --source_db.user | MySQL database username | |
Expand Down
6 changes: 3 additions & 3 deletions website/docs/03-Usage Docs/06-flink-lakesoul-connector.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0
:::tip
Since 2.3.0

LakeSoul with version 2.3.0 is targeting Flink 1.14 while 2.4.0 is targeting Flink 1.17。
LakeSoul with version 2.3.0 is targeting Flink 1.14 while 2.4.1 is targeting Flink 1.17。
:::

LakeSoul provides Flink Connector which implements the Dynamic Table interface, through which developers can use Flink's DataStream API, Table API or SQL to read and write LakeSoul data, and supports both streaming and batch modes for read and write. Read and Write in Flink streaming both support Flink Changelog Stream semantics.
Expand All @@ -18,14 +18,14 @@ LakeSoul provides Flink Connector which implements the Dynamic Table interface,

To setup Flink environment, please refer to [Setup Spark/Flink Job/Project](../03-Usage%20Docs/02-setup-spark.md)

Introduce LakeSoul dependency: package and compile the lakesoul-flink folder to get lakesoul-flink-2.4.0-flink-1.17.jar.
Introduce LakeSoul dependency: package and compile the lakesoul-flink folder to get lakesoul-flink-2.4.1-flink-1.17.jar.

In order to use Flink to create LakeSoul tables, it is recommended to use Flink SQL Client, which supports direct use of Flink SQL commands to operate LakeSoul tables. In this document, the Flink SQL is to directly enter statements on the Flink SQL Client cli interface; whereas the Table API needs to be used in a Java projects.

Switch to the flink folder and execute the command to start the SQLclient client.
```bash
# Start Flink SQL Client
bin/sql-client.sh embedded -j lakesoul-flink-2.4.0-flink-1.14.jar
bin/sql-client.sh embedded -j lakesoul-flink-2.4.1-flink-1.14.jar
```

## 2. DDL
Expand Down
2 changes: 1 addition & 1 deletion website/docs/03-Usage Docs/09-clean-redundant-data.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ Start the Spark cleanup command locally:
--executor-cores 1 \
--num-executors 20 \
--class com.dmetasoul.lakesoul.spark.clean.CleanExpiredData \
jars/lakesoul-spark-2.4.0-spark-3.3.jar
jars/lakesoul-spark-2.4.1-spark-3.3.jar

```
:::tip
Expand Down
2 changes: 1 addition & 1 deletion website/docs/03-Usage Docs/10-setup-presto.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ Available since version 2.4.
LakeSoul implements Presto Connector and currently supports reading tables. It can read tables without primary keys and tables with primary keys (including [CDC format tables](04-cdc-ingestion-table.mdx)). When reading, Merge on Read will be automatically executed to obtain the latest data.

## Download Jar package
You can download the Presto package from the Github Release page: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-presto-2.4.0-presto-0.28.jar
You can download the Presto package from the Github Release page: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-presto-2.4.1-presto-0.28.jar

## Configure Presto
Create a new lakesoul subdirectory in the plugin subdirectory under the Presto directory. Place the jar downloaded above into this subdirectory.
Expand Down
6 changes: 3 additions & 3 deletions website/docs/03-Usage Docs/13-setup-kyuubi.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ LakeSoul implements Flink/Spark Connector.We could use Spark/Flink SQL queries t
| Kyuubi | 1.8 |
| Spark | 3.3 |
| Flink | 1.17 |
| LakeSoul | 2.4.0 |
| LakeSoul | 2.4.1 |
| Java | 1.8 |

The operating environment is Linux, and Spark, Flink, and Kyuubi have been installed. It is recommended to use Hadoop Yarn to run the Kyuubi Engine. Also, you could start local spark/flink cluster.
Expand All @@ -31,7 +31,7 @@ The operating environment is Linux, and Spark, Flink, and Kyuubi have been insta

### 1. Dependencies

Download LakeSoul Flink Jar from: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-flink-2.4.0-flink-1.17.jar
Download LakeSoul Flink Jar from: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-flink-2.4.1-flink-1.17.jar

And put the jar file under `$FLINK_HOME/lib`.

Expand Down Expand Up @@ -74,7 +74,7 @@ More details about Flink SQL with LakeSoul refer to : [Flink Lakesoul Connector]

### 1. Dependencies

Download LakeSoul Spark Jar from: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-spark-2.4.0-spark-3.3.jar
Download LakeSoul Spark Jar from: https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-spark-2.4.1-spark-3.3.jar

And put the jar file under `$SPARK_HOME/jars`.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@ https://dlcdn.apache.org/spark/spark-3.3.2/spark-3.3.2-bin-without-hadoop.tgz

LakeSoul 发布 jar 包可以从 GitHub Releases 页面下载:https://github.com/lakesoul-io/LakeSoul/releases 。下载后请将 Jar 包放到 Spark 安装目录下的 jars 目录中:
```bash
wget https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-spark-2.4.0-spark-3.3.jar -P $SPARK_HOME/jars
wget https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-spark-2.4.1-spark-3.3.jar -P $SPARK_HOME/jars
```

如果访问 Github 有问题,也可以从如下链接下载:https://dmetasoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/releases/lakesoul/lakesoul-spark-2.4.0-spark-3.3.jar
如果访问 Github 有问题,也可以从如下链接下载:https://dmetasoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/releases/lakesoul/lakesoul-spark-2.4.1-spark-3.3.jar

:::tip
从 2.1.0 版本起,LakeSoul 自身的依赖已经通过 shade 方式打包到一个 jar 包中。之前的版本是多个 jar 包以 tar.gz 压缩包的形式发布。
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ docker run --net lakesoul-docker-compose-env_default --rm -ti \
-v $(pwd)/lakesoul.properties:/opt/spark/work-dir/lakesoul.properties \
--env lakesoul_home=/opt/spark/work-dir/lakesoul.properties bitnami/spark:3.3.1 \
spark-shell \
--packages com.dmetasoul:lakesoul-spark:2.4.0-spark-3.3 \
--packages com.dmetasoul:lakesoul-spark:2.4.1-spark-3.3 \
--conf spark.sql.extensions=com.dmetasoul.lakesoul.sql.LakeSoulSparkSessionExtension \
--conf spark.sql.catalog.lakesoul=org.apache.spark.sql.lakesoul.catalog.LakeSoulCatalog \
--conf spark.sql.defaultCatalog=lakesoul \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ $FLINK_HOME/bin/start-cluster.sh
```bash
./bin/flink run -ys 1 -yjm 1G -ytm 2G \
-c org.apache.flink.lakesoul.entry.MysqlCdc \
lakesoul-flink-2.4.0-flink-1.17.jar \
lakesoul-flink-2.4.1-flink-1.17.jar \
--source_db.host localhost \
--source_db.port 3306 \
--source_db.db_name test_cdc \
Expand All @@ -105,7 +105,7 @@ $FLINK_HOME/bin/start-cluster.sh
--server_time_zone UTC
```

其中 lakesoul-flink 的 jar 包可以从 [Github Release](https://github.com/lakesoul-io/LakeSoul/releases/) 页面下载。如果访问 Github 有问题,也可以通过这个链接下载:https://dmetasoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/releases/lakesoul/lakesoul-flink-2.4.0-flink-1.17.jar
其中 lakesoul-flink 的 jar 包可以从 [Github Release](https://github.com/lakesoul-io/LakeSoul/releases/) 页面下载。如果访问 Github 有问题,也可以通过这个链接下载:https://dmetasoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/releases/lakesoul/lakesoul-flink-2.4.1-flink-1.17.jar

在 http://localhost:8081 Flink 作业页面中,点击 Running Job,进入查看 LakeSoul 作业是否已经处于 `Running` 状态。

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,30 +15,30 @@ LakeSoul 目前支持 Spark 3.3 + Scala 2.12.

#### 使用 `--packages` 传 Maven 仓库和包名
```bash
spark-shell --packages com.dmetasoul:lakesoul-spark:2.4.0-spark-3.3
spark-shell --packages com.dmetasoul:lakesoul-spark:2.4.1-spark-3.3
```

#### 使用打包好的 LakeSoul 包
可以从 [Releases](https://github.com/lakesoul-io/LakeSoul/releases) 页面下载已经打包好的 LakeSoul Jar 包。
下载 jar 并传给 `spark-submit` 命令:
```bash
spark-submit --jars "lakesoul-spark-2.4.0-spark-3.3.jar"
spark-submit --jars "lakesoul-spark-2.4.1-spark-3.3.jar"
```

#### 直接将 Jar 包放在 Spark 环境中
可以将 Jar 包下载后,放在 $SPARK_HOME/jars 中。

Jar 包可以从 Github Release 页面下载:https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-spark-2.4.0-spark-3.3.jar
Jar 包可以从 Github Release 页面下载:https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-spark-2.4.1-spark-3.3.jar

或者从国内地址下载:https://dmetasoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/releases/lakesoul/lakesoul-spark-2.4.0-spark-3.3.jar
或者从国内地址下载:https://dmetasoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/releases/lakesoul/lakesoul-spark-2.4.1-spark-3.3.jar

### 设置 Java/Scala 项目
增加以下 Maven 依赖项:
```xml
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-spark</artifactId>
<version>2.4.0-spark-3.3</version>
<version>2.4.1-spark-3.3</version>
</dependency>
```

Expand Down Expand Up @@ -139,7 +139,7 @@ taskmanager.memory.task.off-heap.size: 3000m


### 添加 LakeSoul Jar 到 Flink 部署的目录
从以下地址下载 LakeSoul Flink Jar:https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.0/lakesoul-flink-2.4.0-flink-1.17.jar
从以下地址下载 LakeSoul Flink Jar:https://github.com/lakesoul-io/LakeSoul/releases/download/v2.4.1/lakesoul-flink-2.4.1-flink-1.17.jar

并将 jar 文件放在 `$FLINK_HOME/lib` 下。在此之后,您可以像往常一样启动 flink 会话集群或应用程序。

Expand All @@ -160,6 +160,6 @@ export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul</artifactId>
<version>2.4.0-flink-1.17</version>
<version>2.4.1-flink-1.17</version>
</dependency>
```
Loading

0 comments on commit 349c510

Please sign in to comment.