ci(services/sftp): Move setup logic into docker-compose (#3430)

.github/scripts/behavior_test/plan.py

@@ -214,11 +214,6 @@ def generate_binding_python_cases(
 ) -> list[dict[str, str]]:
     cases = unique_cases(cases)
 
-    # REMOVE ME after https://github.com/apache/incubator-opendal/issues/3429 addressed.
-    #
-    # Sftp can't pass on python bindings, remove it for now.
-    cases = [v for v in cases if v["service"] != "sftp"]
-
     if os.getenv("GITHUB_IS_PUSH") == "true":
         return cases
 

.github/services/azblob/azurite_azblob/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup azurite azblob service
       shell: bash
       working-directory: fixtures/azblob
-      run: docker-compose -f docker-compose-azurite.yml up -d
+      run: docker compose -f docker-compose-azurite.yml up -d --wait
     - name: Setup test bucket
       shell: bash
       run: |

.github/services/etcd/etcd-cluster/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup etcd cluster
       shell: bash
       working-directory: fixtures/etcd
-      run: docker-compose -f docker-compose-cluster.yml up -d
+      run: docker compose -f docker-compose-cluster.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/etcd/etcd-tls/action.yml

@@ -30,7 +30,7 @@ runs:
     - name: Setup etcd tls
       shell: bash
       working-directory: fixtures/etcd
-      run: docker-compose -f docker-compose-standalone-tls.yml up -d
+      run: docker compose -f docker-compose-standalone-tls.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/etcd/etcd/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup etcd cluster
       shell: bash
       working-directory: fixtures/etcd
-      run: docker-compose -f docker-compose-standalone.yml up -d
+      run: docker compose -f docker-compose-standalone.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/http/caddy/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup http in caddy
       shell: bash
       working-directory: fixtures/http
-      run: docker-compose -f docker-compose-caddy.yml up -d
+      run: docker compose -f docker-compose-caddy.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/http/nginx/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup http in nginx
       shell: bash
       working-directory: fixtures/http
-      run: docker-compose -f docker-compose-nginx.yml up -d
+      run: docker compose -f docker-compose-nginx.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/memcached/memcached/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup memcached server
       shell: bash
       working-directory: fixtures/memcached
-      run: docker-compose -f docker-compose-memcached.yml up -d
+      run: docker compose -f docker-compose-memcached.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/mongodb/mongodb_with_basic_auth/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup MongoDB Server
       shell: bash
       working-directory: fixtures/mongodb
-      run: docker-compose -f docker-compose-basic-auth.yml up -d
+      run: docker compose -f docker-compose-basic-auth.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/mongodb/mongodb_with_no_auth/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup MongoDB Server
       shell: bash
       working-directory: fixtures/mongodb
-      run: docker-compose -f docker-compose-no-auth.yml up -d
+      run: docker compose -f docker-compose-no-auth.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/mysql/mysql/action.yml

@@ -26,7 +26,7 @@ runs:
       working-directory: fixtures/mysql
       run: |
         apt update && apt install -y mysql-client
-        docker-compose -f docker-compose.yml up -d
+        docker compose -f docker-compose.yml up -d --wait
         while ! mysql -h localhost -P 3306 --protocol=tcp -u root -p'password' -e "SELECT 1"; do
             echo "Waiting for MySQL..."
             sleep 1

.github/services/postgresql/postgresql/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup PostgreSQL Server
       shell: bash
       working-directory: fixtures/postgresql
-      run: docker-compose -f docker-compose.yml up -d
+      run: docker compose -f docker-compose.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/redis/dragonfly/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup Dragonfly Server
       shell: bash
       working-directory: fixtures/redis
-      run: docker-compose -f docker-compose-dragonfly.yml up -d
+      run: docker compose -f docker-compose-dragonfly.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/redis/kvrocks/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup Kvrocks Server
       shell: bash
       working-directory: fixtures/redis
-      run: docker-compose -f docker-compose-kvrocks.yml up -d
+      run: docker compose -f docker-compose-kvrocks.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/redis/redis/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup Redis Server
       shell: bash
       working-directory: fixtures/redis
-      run: docker-compose -f docker-compose-redis.yml up -d
+      run: docker compose -f docker-compose-redis.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/redis/redis_with_cluster/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup Redis Cluster
       shell: bash
       working-directory: fixtures/redis
-      run: docker-compose -f docker-compose-redis-cluster.yml up -d
+      run: docker compose -f docker-compose-redis-cluster.yml up -d --wait
     - name: Setup
       shell: bash
       run: |

.github/services/s3/minio_s3/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Setup MinIO Server
       shell: bash
       working-directory: fixtures/s3
-      run: docker-compose -f docker-compose-minio.yml up -d
+      run: docker compose -f docker-compose-minio.yml up -d --wait
     - name: Setup test bucket
       shell: bash
       env:

.github/services/s3/minio_s3_with_anonymous/action.yml

@@ -25,7 +25,7 @@ runs:
       shell: bash
       working-directory: fixtures/s3
       run: |
-        docker-compose -f docker-compose-minio.yml up -d
+        docker compose -f docker-compose-minio.yml up -d --wait
     - name: Setup test bucket
       shell: bash
       env:

.github/services/sftp/sftp/action.yml

@@ -21,23 +21,20 @@ description: 'Behavior test for SFTP'
 runs:
   using: "composite"
   steps:
-    - name: Setup sftp
+    - name: Setup Sftp
       shell: bash
+      working-directory: fixtures/sftp
       run: |
-        mkdir -p target/ssh
-        ssh-keygen -t rsa -b 4096 -f target/ssh/id_rsa -q -N "" < /dev/null
-        docker run \
-            -v `pwd`/target/ssh/id_rsa.pub:/home/foo/.ssh/keys/id_rsa.pub:ro \
-            --ulimit nofile=65536:65536 \
-            -p 2222:22 -d atmoz/sftp \
-            foo::::upload
+        docker compose -f docker-compose-sftp.yml up -d --wait
     - name: Setup
       shell: bash
       run: |
+        chmod 600 ${{ github.workspace }}/fixtures/sftp/test_ssh_key
+
         cat << EOF >> $GITHUB_ENV
         OPENDAL_SFTP_ENDPOINT=ssh://127.0.0.1:2222
         OPENDAL_SFTP_ROOT=/upload/sftp_test/
         OPENDAL_SFTP_USER=foo
-        OPENDAL_SFTP_KEY=${{ github.workspace }}/target/ssh/id_rsa
+        OPENDAL_SFTP_KEY=${{ github.workspace }}/fixtures/sftp/test_ssh_key
         OPENDAL_SFTP_KNOWN_HOSTS_STRATEGY=accept
         EOF

.github/services/sftp/sftp_with_default_root/action.yml

@@ -21,27 +21,20 @@ description: 'Behavior test for SFTP with default root'
 runs:
   using: "composite"
   steps:
-    - name: Setup sftp
+    - name: Setup Sftp
       shell: bash
+      working-directory: fixtures/sftp
       run: |
-        mkdir -p target/ssh
-        ssh-keygen -t rsa -b 4096 -f target/ssh/id_rsa -q -N "" < /dev/null
-        tmpScript="$(mktemp)"
-        echo "sed -i -e 's#ForceCommand internal-sftp#ForceCommand internal-sftp -d /upload#' /etc/ssh/sshd_config" > "$tmpScript"
-        chmod +x "$tmpScript"
-        docker run \
-            -v "$tmpScript:/etc/sftp.d/change_root_dir" \
-            -v `pwd`/target/ssh/id_rsa.pub:/home/foo/.ssh/keys/id_rsa.pub:ro \
-            --ulimit nofile=65536:65536 \
-            -p 2222:22 -d atmoz/sftp \
-            foo::::upload
+        docker compose -f docker-compose-sftp-with-default-root.yml up -d --wait
     - name: Setup
       shell: bash
       run: |
+        chmod 600 ${{ github.workspace }}/fixtures/sftp/test_ssh_key
+
         cat << EOF >> $GITHUB_ENV
         OPENDAL_SFTP_ENDPOINT=ssh://127.0.0.1:2222
         OPENDAL_SFTP_USER=foo
-        OPENDAL_SFTP_KEY=${{ github.workspace }}/target/ssh/id_rsa
+        OPENDAL_SFTP_KEY=${{ github.workspace }}/fixtures/sftp/test_ssh_key
         OPENDAL_SFTP_KNOWN_HOSTS_STRATEGY=accept
         OPENDAL_DISABLE_RANDOM_ROOT=true
         EOF

.typos.toml

@@ -23,3 +23,10 @@
 "hellow" = "hellow"
 # Showed up in examples.
 "thw" = "thw"
+
+[files]
+extend-exclude = [
+  # Generated SSH Keys.
+  "fixtures/sftp/test_ssh_key",
+  "fixtures/sftp/test_ssh_key.pub",
+]

bindings/python/Cargo.toml

@@ -84,7 +84,6 @@ services-all = [
 # Default services provided by opendal.
 services-azblob = ["opendal/services-azblob"]
 services-azdls = ["opendal/services-azdls"]
-services-azfile = ["opendal/services-azfile"]
 services-cos = ["opendal/services-cos"]
 services-fs = ["opendal/services-fs"]
 services-gcs = ["opendal/services-gcs"]
@@ -99,6 +98,7 @@ services-webdav = ["opendal/services-webdav"]
 services-webhdfs = ["opendal/services-webhdfs"]
 
 # Optional services provided by opendal.
+services-azfile = ["opendal/services-azfile"]
 services-cacache = ["opendal/services-cacache"]
 services-dashmap = ["opendal/services-dashmap"]
 services-dropbox = ["opendal/services-dropbox"]

fixtures/sftp/change_root_dir.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+sed -i -e 's#ForceCommand internal-sftp#ForceCommand internal-sftp -d /upload#' /etc/ssh/sshd_config

fixtures/sftp/docker-compose-sftp-with-default-root.yml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: '3.8'
+
+services:
+  sftp-with-default-root:
+    image: atmoz/sftp:debian
+    ports:
+      - "2222:22"
+    volumes:
+      - ./change_root_dir.sh:/etc/sftp.d/change_root_dir
+      - ./test_ssh_key.pub:/home/foo/.ssh/keys/id_rsa.pub:ro
+      - ./test_ssh_key:/home/foo/.ssh/keys/id_rsa:ro
+      - ./health-check.sh:/health-check.sh
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    command: foo::::upload
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "bash",
+          "-c",
+          "/health-check.sh"
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 5

fixtures/sftp/docker-compose-sftp.yml

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: '3.8'
+
+services:
+  sftp:
+    image: atmoz/sftp:debian
+    ports:
+      - "2222:22"
+    volumes:
+      - ./test_ssh_key.pub:/home/foo/.ssh/keys/id_rsa.pub:ro
+      - ./test_ssh_key:/home/foo/.ssh/keys/id_rsa:ro
+      - ./health-check.sh:/health-check.sh
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    command: foo::::upload
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "bash",
+          "-c",
+          "/health-check.sh"
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 5