Adding sftp adapter (#38)

.github/workflows/test.yml

@@ -13,17 +13,21 @@ jobs:
       matrix:
         ruby-version: ["2.6", "2.7", "3.0"]
     services:
-      ftp_server:
+      ftp:
         image: delfer/alpine-ftp-server
         ports:
           - 7021:21
           - 21000-21010:21000-21010
         env:
-          USERS: "ftpuser|ftppass"
-      ssh_server:
+          USERS: "user|pass"
+      ssh:
         image: sickp/alpine-sshd:7.5-r2
         ports:
           - 7022:22
+      sftp:
+        image: watersalesman/sftp-alpine
+        ports:
+          - 7023:22
     steps:
       - uses: actions/checkout@v2
       - uses: ruby/setup-ruby@v1

.version

@@ -1 +1 @@
-0.8.4
+0.9.0

Gemfile

@@ -5,6 +5,10 @@ gem 'bfs-ftp', path: './ftp'
 gem 'bfs-gs', path: './gs'
 gem 'bfs-s3', path: './s3'
 gem 'bfs-scp', path: './scp'
+gem 'bfs-sftp', path: './sftp'
+
+gem 'bcrypt_pbkdf'
+gem 'ed25519'
 gem 'rake'
 gem 'rspec'
 gem 'rubocop-bsm'

Gemfile.lock

@@ -1,63 +1,72 @@
 PATH
   remote: core
   specs:
-    bfs (0.8.4)
+    bfs (0.9.0)
 
 PATH
   remote: ftp
   specs:
-    bfs-ftp (0.8.4)
-      bfs (= 0.8.4)
+    bfs-ftp (0.9.0)
+      bfs (= 0.9.0)
       net-ftp-list
 
 PATH
   remote: gs
   specs:
-    bfs-gs (0.8.4)
-      bfs (= 0.8.4)
+    bfs-gs (0.9.0)
+      bfs (= 0.9.0)
       google-cloud-storage (~> 1.18)
 
 PATH
   remote: s3
   specs:
-    bfs-s3 (0.8.4)
+    bfs-s3 (0.9.0)
       aws-sdk-s3 (~> 1.38)
-      bfs (= 0.8.4)
+      bfs (= 0.9.0)
 
 PATH
   remote: scp
   specs:
-    bfs-scp (0.8.4)
-      bfs (= 0.8.4)
+    bfs-scp (0.9.0)
+      bfs (= 0.9.0)
       net-scp
 
+PATH
+  remote: sftp
+  specs:
+    bfs-sftp (0.9.0)
+      bfs (= 0.9.0)
+      net-sftp
+
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
     aws-eventstream (1.1.1)
-    aws-partitions (1.463.0)
-    aws-sdk-core (3.114.0)
+    aws-partitions (1.472.0)
+    aws-sdk-core (3.115.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.43.0)
+    aws-sdk-kms (1.44.0)
       aws-sdk-core (~> 3, >= 3.112.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.95.1)
+    aws-sdk-s3 (1.96.1)
       aws-sdk-core (~> 3, >= 3.112.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.3)
       aws-eventstream (~> 1, >= 1.0.2)
+    bcrypt_pbkdf (1.1.0)
     declarative (0.0.20)
     diff-lcs (1.4.4)
     digest-crc (0.6.3)
       rake (>= 12.0.0, < 14.0.0)
-    faraday (1.4.2)
+    ed25519 (1.2.4)
+    faraday (1.4.3)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -70,33 +79,32 @@ GEM
     faraday-excon (1.1.0)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.1.0)
-    google-apis-core (0.3.0)
+    google-apis-core (0.4.0)
       addressable (~> 2.5, >= 2.5.1)
-      googleauth (~> 0.14)
-      httpclient (>= 2.8.1, < 3.0)
+      googleauth (>= 0.16.2, < 2.a)
+      httpclient (>= 2.8.1, < 3.a)
       mini_mime (~> 1.0)
       representable (~> 3.0)
-      retriable (>= 2.0, < 4.0)
+      retriable (>= 2.0, < 4.a)
       rexml
-      signet (~> 0.14)
       webrick
-    google-apis-iamcredentials_v1 (0.4.0)
-      google-apis-core (~> 0.1)
-    google-apis-storage_v1 (0.4.0)
-      google-apis-core (~> 0.1)
+    google-apis-iamcredentials_v1 (0.5.0)
+      google-apis-core (>= 0.3, < 2.a)
+    google-apis-storage_v1 (0.5.0)
+      google-apis-core (>= 0.3, < 2.a)
     google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
     google-cloud-env (1.5.0)
       faraday (>= 0.17.3, < 2.0)
     google-cloud-errors (1.1.0)
-    google-cloud-storage (1.31.1)
+    google-cloud-storage (1.32.0)
       addressable (~> 2.5)
       digest-crc (~> 0.4)
       google-apis-iamcredentials_v1 (~> 0.1)
       google-apis-storage_v1 (~> 0.1)
-      google-cloud-core (~> 1.2)
-      googleauth (~> 0.9)
+      google-cloud-core (~> 1.6)
+      googleauth (>= 0.16.2, < 2.a)
       mini_mime (~> 1.0)
     googleauth (0.16.2)
       faraday (>= 0.17.3, < 2.0)
@@ -115,6 +123,8 @@ GEM
     net-ftp-list (3.3.0)
     net-scp (3.0.0)
       net-ssh (>= 2.6.5, < 7.0.0)
+    net-sftp (3.0.0)
+      net-ssh (>= 5.0.0, < 7.0.0)
     net-ssh (6.1.0)
     os (1.1.1)
     parallel (1.20.1)
@@ -143,16 +153,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
-    rubocop (1.15.0)
+    rubocop (1.18.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.5.0, < 2.0)
+      rubocop-ast (>= 1.7.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.6.0)
+    rubocop-ast (1.7.0)
       parser (>= 3.0.1.1)
     rubocop-bsm (0.6.0)
       rubocop (~> 1.0)
@@ -162,9 +172,9 @@ GEM
     rubocop-performance (1.11.3)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    rubocop-rake (0.5.1)
-      rubocop
-    rubocop-rspec (2.3.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.4.0)
       rubocop (~> 1.0)
       rubocop-ast (>= 1.1.0)
     ruby-progressbar (1.11.0)
@@ -183,15 +193,18 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  bcrypt_pbkdf
   bfs!
   bfs-ftp!
   bfs-gs!
   bfs-s3!
   bfs-scp!
+  bfs-sftp!
+  ed25519
   rake
   rspec
   rubocop-bsm
   webrick
 
 BUNDLED WITH
-   2.2.17
+   2.2.21

docker-compose.yml

@@ -1,14 +1,17 @@
 version: "3"
 services:
-  vsftpd:
+  ftp:
     image: delfer/alpine-ftp-server
     ports:
       - "7021:21"
       - "21000-21010:21000-21010"
     environment:
-      USERS: "ftpuser|ftppass"
-
-  sshd:
+      USERS: "user|pass"
+  ssh:
     image: sickp/alpine-sshd:7.5-r2
     ports:
       - "7022:22"
+  sftp:
+    image: watersalesman/sftp-alpine
+    ports:
+      - "7023:22"

ftp/lib/bfs/bucket/ftp.rb

@@ -125,15 +125,15 @@ def mkdir_p(path)
   end
 end
 
-BFS.register('ftp', 'sftp') do |url, opts, block|
+BFS.register('ftp') do |url, opts, block|
   prefix = BFS.norm_path(opts[:prefix] || url.path)
   opts[:prefix] = prefix unless prefix.empty?
 
   extra = {
     username: url.user ? CGI.unescape(url.user) : nil,
     password: url.password ? CGI.unescape(url.password) : nil,
     port: url.port,
-    ssl: opts.key?(:ssl) || url.scheme == 'sftp',
+    ssl: opts.key?(:ssl),
   }
   BFS::Bucket::FTP.open url.host, **opts, **extra, &block
 end

ftp/spec/bfs/bucket/ftp_spec.rb

@@ -4,7 +4,7 @@
   subject { described_class.new hostname, **conn_opts }
 
   let(:hostname) { '127.0.0.1' }
-  let(:conn_opts) { { port: 7021, username: 'ftpuser', password: 'ftppass', prefix: SecureRandom.uuid } }
+  let(:conn_opts) { { port: 7021, username: 'user', password: 'pass', prefix: SecureRandom.uuid } }
 
   after { subject.close }
 
@@ -13,14 +13,14 @@
                   metadata: false
 
   it 'resolves from URL' do
-    bucket = BFS.resolve('ftp://ftpuser:ftppass@127.0.0.1:7021')
+    bucket = BFS.resolve('ftp://user:pass@127.0.0.1:7021')
     expect(bucket).to be_instance_of(described_class)
-    expect(bucket.instance_variable_get(:@client).pwd).to eq('/ftp/ftpuser')
+    expect(bucket.instance_variable_get(:@client).pwd).to eq('/ftp/user')
     bucket.close
 
-    bucket = BFS.resolve('ftp://ftpuser:ftppass@127.0.0.1:7021/a/b/')
+    bucket = BFS.resolve('ftp://user:pass@127.0.0.1:7021/a/b/')
     expect(bucket).to be_instance_of(described_class)
-    expect(bucket.instance_variable_get(:@client).pwd).to eq('/ftp/ftpuser/a/b')
+    expect(bucket.instance_variable_get(:@client).pwd).to eq('/ftp/user/a/b')
     bucket.close
   end
 end

scp/lib/bfs/bucket/scp.rb

@@ -1,6 +1,5 @@
 require 'bfs'
 require 'net/scp'
-require 'net/ssh'
 require 'shellwords'
 
 module BFS
@@ -32,7 +31,7 @@ def initialize(host, prefix: nil, **opts)
         super(**opts)
 
         @prefix = prefix
-        @client = Net::SCP.start(host, nil, **opts.slice(*Net::SSH::VALID_OPTIONS))
+        @client = Net::SCP.start(host, nil, **opts.slice(*Net::SSH::VALID_OPTIONS), non_interactive: true)
 
         if @prefix # rubocop:disable Style/GuardClause
           @prefix = "#{norm_path(@prefix)}/"

sftp/bfs-sftp.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name        = 'bfs-sftp'
+  s.version     = File.read(File.expand_path('../.version', __dir__)).strip
+  s.platform    = Gem::Platform::RUBY
+
+  s.licenses    = ['Apache-2.0']
+  s.summary     = 'SFTP adapter for bfs'
+  s.description = 'https://github.com/bsm/bfs.rb'
+
+  s.authors     = ['Dimitrij Denissenko']
+  s.email       = '[email protected]'
+  s.homepage    = 'https://github.com/bsm/bfs.rb'
+
+  s.executables   = []
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.require_paths = ['lib']
+  s.required_ruby_version = '>= 2.6.0'
+
+  s.add_dependency 'bfs', s.version
+  s.add_dependency 'net-sftp'
+end