Merge pull request #192 from cubefs/docker-playground

chore: clean lz4 and zstd version dependency and fix docker readme.

docker/playground/README.md

@@ -60,7 +60,7 @@ Note: You can install virtulbox to start docker compose, which requires about 10
     ```
     cp canal.deployer-*.tar.gz docker/playground
     cp canal.adapter-*.tar.gz docker/playground
-    docker rm --force playground-compass-demo-1 (if you start the compass demo)
+    docker rm -f compass-demo (if you start the compass demo)
     docker compose --profile compass up -d --build
     ```
 
@@ -71,27 +71,29 @@ Note: You can install virtulbox to start docker compose, which requires about 10
 
 5. With hadoop (Optional)
 
-```
-wget https://archive.apache.org/dist/hadoop/common/hadoop-3.1.1/hadoop-3.1.1.tar.gz
-wget https://dlcdn.apache.org/spark/spark-3.3.4/spark-3.3.4-bin-hadoop3.tgz
-
-docker compose --profile hadoop up -d
-```
+    ```
+    wget https://archive.apache.org/dist/hadoop/common/hadoop-3.1.1/hadoop-3.1.1.tar.gz
+    wget https://dlcdn.apache.org/spark/spark-3.3.4/spark-3.3.4-bin-hadoop3.tgz
+    
+    docker compose --profile hadoop up -d
+    ```
 
-Run spark example
+    Run spark example
+    ```
+    docker exec hadoop /bin/bash -c "/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode cluster --executor-memory 1G  --num-executors 1 /opt/spark/examples/jars/spark-examples_2.12-3.3.4.jar"
+    ```
 
-```
-docker exec hadoop /bin/bash -c "/opt/spark/bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode cluster --executor-memory 1G  --num-executors 1 /opt/spark/examples/jars/spark-examples_2.12-3.3.4.jar"
-```
+6. With Dolphinscheduler (Optional) (dependent components: dolphinscheduler, hadoop, spark, flume)
+    ```
+    wget https://archive.apache.org/dist/dolphinscheduler/3.1.5/apache-dolphinscheduler-3.1.5-bin.tar.gz
+    wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.19/mysql-connector-java-8.0.19.jar
+    wget https://archive.apache.org/dist/flume/1.11.0/apache-flume-1.11.0-bin.tar.gz
+    
+        
+    # start dolphinscheduler services
+    docker compose --profile dolphinscheduler up -d
+    ```
 
-6. With Dolphinscheduler (Optional)
-```
-wget https://archive.apache.org/dist/dolphinscheduler/3.1.5/apache-dolphinscheduler-3.1.5-bin.tar.gz
-wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.19/mysql-connector-java-8.0.19.jar
-wget https://archive.apache.org/dist/flume/1.11.0/apache-flume-1.11.0-bin.tar.gz
+Run dolphinscheduler example, please refer to [dolphinscheduler example document](../../document/manual/ds_example.md)
 
-
-# start dolphinscheduler services
-docker compose --profile dolphinscheduler up -d
-```
 For more details, please refer to [deployment document](../../document/manual/deployment.md)

docker/playground/compose.yml

@@ -187,6 +187,7 @@ services:
     ports:
       - "7075:7075"
     tty: true
+    container_name: compass-demo
     profiles: ["compass-demo"]
 
   compass:
@@ -196,4 +197,5 @@ services:
     ports:
       - "7075:7075"
     tty: true
+    container_name: compass
     profiles: ["compass"]

docker/playground/conf/application-hadoop.yml

@@ -1,8 +1,8 @@
 hadoop:
   namenodes:
-    - nameservices: log-hdfs
-      namenodesAddr: [ "host1", "host2" ]
-      namenodes: [ "namenode1", "namenode2" ]
+    - nameservices: hadoop
+      namenodesAddr: [ "hadoop"]
+      namenodes: [ "hadoop"]
       user: hdfs
       password:
       port: 8020
@@ -21,8 +21,8 @@ hadoop:
 
   yarn:
     - clusterName: "bigdata"
-      resourceManager: [ "localhost:8088" ]
-      jobHistoryServer: "localhost:19888"
+      resourceManager: [ "hadoop:8088" ]
+      jobHistoryServer: "hadoop:19888"
 
   spark:
-    sparkHistoryServer: [ "localhost:18018" ]
+    sparkHistoryServer: [ "hadoop:18018" ]

docker/playground/conf/compass_env.sh

@@ -5,11 +5,11 @@ export SCHEDULER="dolphinscheduler"
 export SPRING_PROFILES_ACTIVE="hadoop,${SCHEDULER}"
 
 # Configuration for Scheduler MySQL, compass will subscribe data from scheduler database via canal
-export SCHEDULER_MYSQL_ADDRESS="localhost:33066"
+export SCHEDULER_MYSQL_ADDRESS="dolphinscheduler:3306"
 export SCHEDULER_MYSQL_DB="dolphinscheduler"
 export SCHEDULER_DATASOURCE_URL="jdbc:mysql://${SCHEDULER_MYSQL_ADDRESS}/${SCHEDULER_MYSQL_DB}?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai"
-export SCHEDULER_DATASOURCE_USERNAME=""
-export SCHEDULER_DATASOURCE_PASSWORD=""
+export SCHEDULER_DATASOURCE_USERNAME="root"
+export SCHEDULER_DATASOURCE_PASSWORD="Root@666"
 
 # Configuration for compass database(mysql or postgresql)
 export DATASOURCE_TYPE="postgresql"
@@ -79,14 +79,14 @@ export CANAL_SERVERMODE="kafka"
 
 # task-canal-adapter
 export CANAL_ADAPTER_KAFKA_BOOTSTRAP_SERVERS=${SPRING_KAFKA_BOOTSTRAPSERVERS}
-# source mysql
-export CANAL_ADAPTER_SOURCE_MYSQL_URL=${SCHEDULER_DATASOURCE_URL}
-export CANAL_ADAPTER_SOURCE_MYSQL_USERNAME=${SCHEDULER_DATASOURCE_USERNAME}
-export CANAL_ADAPTER_SOURCE_MYSQL_PASSWORD=${SCHEDULER_DATASOURCE_PASSWORD}
-# destination mysql
-export CANAL_ADAPTER_DESTINATION_MYSQL_URL=${SPRING_DATASOURCE_URL}
-export CANAL_ADAPTER_DESTINATION_MYSQL_USERNAME=${SPRING_DATASOURCE_USERNAME}
-export CANAL_ADAPTER_DESTINATION_MYSQL_PASSWORD=${SPRING_DATASOURCE_PASSWORD}
+# source datasource
+export CANAL_ADAPTER_SOURCE_DATASOURCE_URL=${SCHEDULER_DATASOURCE_URL}
+export CANAL_ADAPTER_SOURCE_DATASOURCE_USERNAME=${SCHEDULER_DATASOURCE_USERNAME}
+export CANAL_ADAPTER_SOURCE_DATASOURCE_PASSWORD=${SCHEDULER_DATASOURCE_PASSWORD}
+# destination datasource
+export CANAL_ADAPTER_DESTINATION_DATASOURCE_URL=${SPRING_DATASOURCE_URL}
+export CANAL_ADAPTER_DESTINATION_DATASOURCE_USERNAME=${SPRING_DATASOURCE_USERNAME}
+export CANAL_ADAPTER_DESTINATION_DATASOURCE_PASSWORD=${SPRING_DATASOURCE_PASSWORD}
 
 # task-syncer
 # source mysql

docker/playground/conf/dolphinscheduler/ds-entrypoint.sh

@@ -7,6 +7,7 @@ useradd -ms /bin/bash dolphinscheduler && echo 'dolphinscheduler:dolphinschedule
 sed -i '$adolphinscheduler  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
 sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers
 bash ./tools/bin/upgrade-schema.sh
+mysql -uroot -pRoot@666 -e "insert into dolphinscheduler.t_ds_tenant(id,tenant_code,queue_id) values(1,'root',1);update dolphinscheduler.t_ds_user set tenant_id=1 where id=1;"
 bash ./bin/dolphinscheduler-daemon.sh start master-server
 bash ./bin/dolphinscheduler-daemon.sh start worker-server
 bash ./bin/dolphinscheduler-daemon.sh start api-server

docker/playground/script/ds2hdfs.py

@@ -172,7 +172,7 @@ def on_created(self, event):
     working_directory = "/opt/"
     os.chdir(working_directory)
     # Watch the root path of log
-    directory_to_watch = "/opt/ds/worker-server/logs"
+    directory_to_watch = "/opt/dolphinscheduler/worker-server/logs"
     event_handler = LogFileEventHandler(directory_to_watch)
     observer = Observer()
     observer.schedule(event_handler, directory_to_watch, recursive=True)

document/manual/ds_example.md

@@ -0,0 +1,53 @@
+# Run DolphinScheduler example
+
+DolphinScheduler Web UI : http://localhost:12345/dolphinscheduler
+
+Login Username & Password: admin / dolphinscheduler123
+
+1. Create File Resources
+
+![Resources](img/ds-example/resources.png)
+
+pi.py example
+
+```shell
+import sys
+from random import random
+from operator import add
+
+from pyspark.sql import SparkSession
+
+
+if __name__ == "__main__":
+    """
+        Usage: pi [partitions]
+    """
+    spark = SparkSession\
+        .builder\
+        .appName("PythonPi")\
+        .getOrCreate()
+
+    partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2
+    n = 100000 * partitions
+
+    def f(_: int) -> float:
+        x = random() * 2 - 1
+        y = random() * 2 - 1
+        return 1 if x ** 2 + y ** 2 <= 1 else 0
+
+    count = spark.sparkContext.parallelize(range(1, n + 1), partitions).map(f).reduce(add)
+    print("Pi is roughly %f" % (4.0 * count / n))
+    spark.stop()
+    sys.exit(-1)
+```
+
+2. Create project and spark workflow definition
+
+![project](img/ds-example/project.png)
+![workflow](img/ds-example/workflow.png)
+
+Save workflow and change tenant to root
+
+![basic](img/ds-example/workflow-basic.png)
+
+3. Online and start pi workflow

pom.xml

@@ -73,6 +73,8 @@
         <derby.version>10.14.2.0</derby.version>
         <testcontainers.version>1.19.0</testcontainers.version>
         <chatgpt-java.version>1.1.3</chatgpt-java.version>
+        <lz4-java.version>1.8.0</lz4-java.version>
+        <zstd-jni.version>1.5.5-2</zstd-jni.version>
         <flink.ui/>
         <spark.ui/>
 
@@ -380,6 +382,18 @@
                 <version>${chatgpt-java.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.lz4</groupId>
+                <artifactId>lz4-java</artifactId>
+                <version>${lz4-java.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.github.luben</groupId>
+                <artifactId>zstd-jni</artifactId>
+                <version>${zstd-jni.version}</version>
+            </dependency>
+
             <!--Vulnerability version fixes-->
             <dependency>
                 <groupId>commons-beanutils</groupId>

task-parser/pom.xml

@@ -148,13 +148,11 @@
         <dependency>
             <groupId>org.lz4</groupId>
             <artifactId>lz4-java</artifactId>
-            <version>1.8.0</version>
         </dependency>
 
         <dependency>
             <groupId>com.github.luben</groupId>
             <artifactId>zstd-jni</artifactId>
-            <version>1.5.5-2</version>
         </dependency>
 
     </dependencies>