Add support for Delete Multiple Objects

.gitignore

@@ -6,3 +6,5 @@ test_root
 
 # Don't check in RVM/rbenv files
 .ruby-version
+
+fakes3.sublime-workspace

Gemfile.lock

@@ -16,17 +16,14 @@ GEM
       json (~> 1.4)
       nokogiri (>= 1.4.4)
     builder (3.2.2)
-    byebug (4.0.1)
-      columnize (= 0.9.0)
-      rb-readline (= 0.5.2)
-    columnize (0.9.0)
     json (1.8.1)
     mime-types (1.25)
     mini_portile (0.6.1)
     nokogiri (1.6.4.1)
       mini_portile (~> 0.6.0)
+    nokogiri (1.6.4.1-x64-mingw32)
+      mini_portile (~> 0.6.0)
     rake (10.1.0)
-    rb-readline (0.5.2)
     rest-client (1.6.7)
       mime-types (>= 1.16)
     right_aws (3.1.0)
@@ -37,13 +34,16 @@ GEM
 
 PLATFORMS
   ruby
+  x64-mingw32
 
 DEPENDENCIES
   aws-s3
   aws-sdk-v1
   bundler (>= 1.0.0)
-  byebug
   fakes3!
   rake
   rest-client
   right_aws
+
+BUNDLED WITH
+   1.10.6

Rakefile

@@ -6,7 +6,7 @@ Bundler::GemHelper.install_tasks
 
 Rake::TestTask.new(:test) do |t|
   t.libs << "."
-  t.test_files =
+  t.test_files = 
     FileList['test/*_test.rb'].exclude('test/s3_commands_test.rb')
 end
 

lib/fakes3/file_store.rb

@@ -5,6 +5,8 @@
 require 'fakes3/rate_limitable_file'
 require 'digest/md5'
 require 'yaml'
+require 'rexml/document'
+include REXML
 
 module FakeS3
   class FileStore
@@ -24,6 +26,20 @@ def initialize(root)
         bucket_obj = Bucket.new(bucket_name,Time.now,[])
         @buckets << bucket_obj
         @bucket_hash[bucket_name] = bucket_obj
+
+        #pre-load objects into bucket, so ListObjects calls work.
+        Dir[File.join(bucket,'/**/.fakes3_metadataFFF')].each do |fullpath|
+          key = fullpath.sub('/.fakes3_metadataFFF', '').sub(bucket + '/', '')
+          object = get_object(bucket_name, key, 'norequest')
+          bucket_obj.add(object)
+          object.io.close
+        end
+      end
+
+      puts "=================================================="
+      puts "Buckets initialized with contents:"
+      buckets.each do |b|
+        puts "#{b.name} - #{b.objects.count} items"
       end
     end
 
@@ -228,7 +244,7 @@ def combine_object_parts(bucket, upload_id, object_name, parts, request)
         File.open(content_path, 'rb') { |f| chunk = f.read }
         etag = Digest::MD5.hexdigest(chunk)
 
-        raise new Error "invalid file chunk" unless part[:etag] == etag
+      raise new Error "invalid file chunk" unless part[:etag] == etag
         complete_file << chunk
         part_paths    << part_path
       end
@@ -256,6 +272,19 @@ def delete_object(bucket,object_name,request)
       end
     end
 
+    def delete_objects(bucket,request)
+      begin
+        xmldoc = Document.new(request.body)
+        xmldoc.elements.each("Delete/Object/Key") do |key|
+          delete_object(bucket, key.text, request)
+        end
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
     # TODO: abstract getting meta data from request.
     def create_metadata(content,request)
       metadata = {}

lib/fakes3/server.rb

@@ -22,8 +22,9 @@ class Request
     GET_ACL = "GET_ACL"
     SET_ACL = "SET_ACL"
     MOVE = "MOVE"
-    DELETE_OBJECT = "DELETE_OBJECT"
     DELETE_BUCKET = "DELETE_BUCKET"
+    DELETE_OBJECT = "DELETE_OBJECT"
+    DELETE_OBJECTS = "DELETE_OBJECTS"
 
     attr_accessor :bucket,:object,:type,:src_bucket,
                   :src_object,:method,:webrick_request,
@@ -73,12 +74,7 @@ def do_GET(request, response)
         if bucket_obj
           response.status = 200
           response['Content-Type'] = "application/xml"
-          query = {
-            :marker => s_req.query["marker"] ? s_req.query["marker"].to_s : nil,
-            :prefix => s_req.query["prefix"] ? s_req.query["prefix"].to_s : nil,
-            :max_keys => s_req.query["max_keys"] ? s_req.query["max_keys"].to_s : nil,
-            :delimiter => s_req.query["delimiter"] ? s_req.query["delimiter"].to_s : nil
-          }
+          query = get_options(s_req)
           bq = bucket_obj.query_for_range(query)
           response.body = XmlAdapter.bucket_query(bq)
         else
@@ -229,6 +225,10 @@ def do_multipartPUT(request, response)
     end
 
     def do_POST(request,response)
+      if request.query_string =~ /delete/i
+        return do_DELETE(request, response)
+      end
+
       s_req = normalize_request(request)
       key   = request.query['key']
       query = CGI::parse(request.request_uri.query || "")
@@ -301,17 +301,25 @@ def do_POST(request,response)
 
     def do_DELETE(request,response)
       s_req = normalize_request(request)
-
+	  response.status = 204
+      response.body = ""
+
       case s_req.type
       when Request::DELETE_OBJECT
         bucket_obj = @store.get_bucket(s_req.bucket)
         @store.delete_object(bucket_obj,s_req.object,s_req.webrick_request)
       when Request::DELETE_BUCKET
         @store.delete_bucket(s_req.bucket)
+      when Request::DELETE_OBJECTS
+        bucket_obj = @store.get_bucket(s_req.bucket)
+        @store.delete_objects(bucket_obj,s_req.webrick_request)
+		response.status = 200
+		response.body = <<-eos.strip
+              <?xml version="1.0" encoding="UTF-8"?>
+              <DeleteResponse>
+              </DeleteResponse>
+            eos
       end
-
-      response.status = 204
-      response.body = ""
     end
 
     def do_OPTIONS(request, response)
@@ -441,6 +449,10 @@ def normalize_post(webrick_req,s_req)
       else
         s_req.object = path[1..-1]
       end
+
+      if webrick_req.query_string =~ /delete/i
+        s_req.type = Request::DELETE_OBJECTS
+      end
     end
 
     # This method takes a webrick request and generates a normalized FakeS3 request
@@ -477,6 +489,11 @@ def normalize_request(webrick_req)
       return s_req
     end
 
+	def gral_strip(string, chars)
+      chars = Regexp.escape(chars)
+      string.gsub(/\A[#{chars}]+|[#{chars}]+\z/, "")
+    end
+
     def parse_complete_multipart_upload request
       parts_xml   = ""
       request.body { |chunk| parts_xml << chunk }
@@ -487,7 +504,7 @@ def parse_complete_multipart_upload request
       parts_xml.collect do |xml|
         {
           number: xml[/\<PartNumber\>(\d+)\<\/PartNumber\>/, 1].to_i,
-          etag:   xml[/\<ETag\>\"(.+)\"\<\/ETag\>/, 1]
+          etag:   gral_strip(xml[/\<ETag\>(.+)\<\/ETag\>/, 1], "\"")
         }
       end
     end
@@ -501,6 +518,15 @@ def dump_request(request)
       end
       puts "----------End Dump -------------"
     end
+
+    def get_options(s_req)
+      return {
+            :marker => s_req.query["marker"] ? s_req.query["marker"].to_s : nil,
+            :prefix => s_req.query["prefix"] ? s_req.query["prefix"].to_s : nil,
+            :max_keys => s_req.query["max_keys"] ? s_req.query["max_keys"].to_s : nil,
+            :delimiter => s_req.query["delimiter"] ? s_req.query["delimiter"].to_s : nil
+          }
+    end
   end
 
 

lib/fakes3/sorted_object_list.rb

@@ -1,11 +1,12 @@
 require 'set'
 module FakeS3
   class S3MatchSet
-    attr_accessor :matches,:is_truncated,:common_prefixes
+    attr_accessor :matches,:is_truncated,:common_prefixes,:next_marker
     def initialize
       @matches = []
       @is_truncated = false
       @common_prefixes = []
+	  @next_marker = ""
     end
   end
 
@@ -102,7 +103,7 @@ def list(options)
                   ms.common_prefixes << base_prefix + chunks[0] + delimiter
                   last_chunk = chunks[0]
                 else
-                  is_truncated = true
+                  ms.is_truncated = true
                   break
                 end
               end
@@ -117,7 +118,8 @@ def list(options)
           if count <= max_keys
             ms.matches << s3_object
           else
-            is_truncated = true
+            ms.is_truncated = true
+			ms.next_marker = s3_object.name
             break
           end
         end

lib/fakes3/xml_adapter.rb

@@ -218,5 +218,20 @@ def self.complete_multipart_result(object)
       }
       output
     end
+
+	# <DeleteObjectsResult>
+    #   <LastModified>2009-10-28T22:32:00</LastModified>
+    #   <ETag>"9b2cf535f27731c974343645a3985328"</ETag>
+    # </CopyObjectResult>
+    def self.delete_objects_result(object)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.DeleteResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |result|
+
+      }
+      output
+    end
+
   end
 end

test/aws_sdk_commands_test.rb

@@ -10,6 +10,21 @@ def setup
                       :use_ssl => false)
   end
 
+  def test_list_objects
+    #assemble
+    bucket = @s3.buckets["test_list_objects"]
+    object = bucket.objects["key1"]
+    object.write("asdf")
+    object.copy_to("prefix1/sub1/key2")
+    object.copy_to("prefix1/sub2/key3")
+
+    #act & assert
+    assert_equal 3, bucket.objects.count
+    assert_equal 2, bucket.objects.with_prefix('prefix1/').count
+    assert_equal 1, bucket.objects.with_prefix('prefix1/sub2/').count
+    assert (not bucket.objects.with_prefix('prefix1/').collect(&:key).include? 'key1')
+  end
+
   def test_copy_to
     bucket = @s3.buckets["test_copy_to"]
     object = bucket.objects["key1"]
@@ -28,4 +43,35 @@ def test_multipart_upload
     assert object.exists?
     assert_equal "thisisaverybigfile", object.read
   end
+
+  def test_delete_multiple
+    #assemble
+    bucket = @s3.buckets["test_delete_multiple"]
+    object = bucket.objects["key1"]
+    object.write("asdf")
+    object.copy_to("key2")
+    assert_equal 2, bucket.objects.count
+
+    #act
+    bucket.objects.delete_all
+
+    #assert
+    assert_equal 0, bucket.objects.count
+  end
+
+  def test_delete_multiple_with_prefix
+    #assemble
+    bucket = @s3.buckets["test_delete_multiple"]
+    object = bucket.objects["key1"]
+    object.write("asdf")
+    object.copy_to("prefix1/key2")
+    object.copy_to("prefix1/key3")
+    assert_equal 3, bucket.objects.count
+
+    #act
+    bucket.objects.with_prefix('prefix1/').delete_all
+
+    #assert
+    assert_equal 1, bucket.objects.count
+  end
 end