DSM-148: restore task

lib/tasks/aws.rake

@@ -6,25 +6,15 @@ namespace :atc do
     # S3_CLIENT#list_objects_v2 method returns up to 1000 results per call, and returns a token that can be
     # used in subsequent calls to get the next page of results.  This method wraps that paging functionality.
     def auto_paginating_list_object_v2(list_objects_v2_opts)
-      next_continuation_token = nil
-
       counter = 0
-      loop do
-        counter += 1
-        result_object = S3_CLIENT.list_objects_v2(list_objects_v2_opts.merge({
-          continuation_token: next_continuation_token
-        }))
-
-        S3_CLIENT.list_objects_v2(list_objects_v2_opts).contents.each do |object|
-          yield object
+      S3_CLIENT.list_objects_v2(list_objects_v2_opts).each do |response|
+        response.contents.each do |object|
+          yield object, counter
+          counter += 1
         end
-
-        next_continuation_token = result_object.next_continuation_token
-        break if next_continuation_token.nil?
       end
     end
 
-
     desc  'For the given bucket_name and key_prefix, iterates over objects and generates a list of their file extensions and counts'
     task list_file_extensions: :environment do
       bucket_name = ENV['bucket_name']
@@ -47,6 +37,23 @@ namespace :atc do
       end
     end
 
+    desc  'For the given bucket_name and key_prefix, iterates over objects and prints a list'
+    task list_objects: :environment do
+      bucket_name = ENV['bucket_name']
+      key_prefix = ENV['key_prefix']
+
+      count = 0
+      auto_paginating_list_object_v2({
+        bucket: bucket_name,
+        prefix: key_prefix
+      }) do |object, i|
+        puts "#{i} #{object.key}"
+        count += 1
+      end
+
+      puts "Total count: #{count}"
+    end
+
     desc  'For the given bucket_name and key_prefix, iterates over objects in Intelligent Tiering and restores them '\
           ' if they have already transitioned to the Archive Access tier.'
     task restore_archived_objects: :environment do
@@ -74,9 +81,10 @@ namespace :atc do
       auto_paginating_list_object_v2({
         bucket: bucket_name,
         prefix: key_prefix
-      }) do |object|
+      }) do |object, i|
         object_key = object.key
         storage_class = object.storage_class
+        object_number = i + 1 # i is 0-indexed
 
         if storage_class == 'INTELLIGENT_TIERING'
           if key_suffix_filter.present? && !object_key.end_with?(key_suffix_filter)
@@ -93,20 +101,26 @@ namespace :atc do
               restore_request: {}
             }) unless dry_run
             number_of_intelligent_tiering_object_resoration_requests_submitted += 1
+
+            puts "#{object_number}: Restore: #{object_key}"
           rescue Aws::S3::Errors::ServiceError => e
             if e.message.include?("Restore is not allowed for the object's current storage class")
               # If we got here, that means that this object was already restored and doesn't need to be restored again
               # because it is available.  We'll silently ignore this error.
               number_of_intelligent_tiering_objects_already_available += 1
+              puts "#{object_number}: Skip: #{object_key} (object does not need to be restored)"
             elsif e.message.include?("Object restore is already in progress")
               # If we got here, that means that this object's restoration is already in progress and we do not need to
               # initiate another restoration request.  We'll silently ignore this error.
               number_of_intelligent_tiering_objects_with_restoration_in_progress += 1
+              puts "#{object_number}: Skip: #{object_key} (restoration is in progress)"
             else
               errors_encountered << "An unexpected error occured while attempting to restore #{object_key}: #{e.message}"
+              puts "#{object_number}: Skip: #{object_key} (unexpected error, see details at end of restoration process)"
             end
           end
         else
+          puts "#{object_number}: Skip: #{object_key} (storage class is not Intelligent Tiering)"
           number_of_non_intelligent_tiering_objects_skipped += 1
         end
       end
@@ -123,8 +137,9 @@ namespace :atc do
       end
       puts "Number of objects skipped based on key_suffix_filter: #{number_of_objects_skipped_based_on_key_suffix_filter}"
       puts "Number of non intelligent tiering objects skipped: #{number_of_non_intelligent_tiering_objects_skipped}"
-      puts  "\nReminder: After restoration has been initiated, it will take 3-5 hours until the files are available for download.  "\
-            "The current time is #{Time.current}, so the files should be available after #{Time.current + 5.hours}."
+      puts  "\nReminder: After restoration has been initiated, it will take 3-5 hours until the files are available for download if they were in the Archive Access tier.  "\
+            "For objects in the Deep Archive Access tier, it could take up to 12 hours until the files are available for download.  "\
+            "The current time is #{Time.current}, and 12 hours from now would be #{Time.current + 12.hours}."
       puts  "--------------------"
       puts "Errors: " + (errors_encountered.empty? ? 'None' : "\n#{errors_encountered.join("\n")}")
     end