TRICLOPS-16: aliased cache paths

.github/workflows/ci.yml

@@ -32,6 +32,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+      - name: Update apt
+        run: sudo apt-get update
       - name: Install vips
         run: sudo apt install -y libvips-tools
       - name: Check vips version

.gitignore

@@ -57,3 +57,5 @@ node_modules
 # https://vitejs.dev/guide/env-and-mode.html#env-files
 *.local
 
+# Local rake file
+local.rake

Gemfile

@@ -73,7 +73,7 @@ group :development, :test do
 end
 
 group :development do
-  gem 'capistrano', '~> 3.18.0', require: false
+  gem 'capistrano', '~> 3.19.0', require: false
   gem 'capistrano-cul', require: false
   gem 'capistrano-passenger', '~> 0.1', require: false
   gem 'capistrano-rails', '~> 1.4', require: false

Gemfile.lock

@@ -85,7 +85,7 @@ GEM
       tzinfo (~> 2.0)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
-    airbrussh (1.5.0)
+    airbrussh (1.5.3)
       sshkit (>= 1.6.1, != 1.7.0)
     ast (2.4.2)
     base64 (0.1.1)
@@ -99,7 +99,7 @@ GEM
       msgpack (~> 1.2)
     builder (3.2.4)
     byebug (11.1.3)
-    capistrano (3.18.0)
+    capistrano (3.19.2)
       airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)
@@ -119,7 +119,7 @@ GEM
     capistrano-rails (1.6.3)
       capistrano (~> 3.1)
       capistrano-bundler (>= 1.1, < 3)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.5)
     connection_pool (2.4.1)
     crass (1.0.6)
     date (3.3.4)
@@ -178,7 +178,7 @@ GEM
     globalid (1.2.1)
       activesupport (>= 6.1)
     hashie (5.0.0)
-    i18n (1.14.1)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     io-console (0.7.2)
     io-wait (0.2.0)
@@ -197,6 +197,7 @@ GEM
     listen (3.8.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
+    logger (1.7.0)
     loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -217,7 +218,7 @@ GEM
     multi_json (1.15.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
-    mutex_m (0.2.0)
+    mutex_m (0.3.0)
     mysql2 (0.5.5)
     net-imap (0.4.10)
       date
@@ -226,11 +227,13 @@ GEM
       net-protocol
     net-protocol (0.2.2)
       timeout
-    net-scp (4.0.0)
+    net-scp (4.1.0)
       net-ssh (>= 2.6.5, < 8.0.0)
+    net-sftp (4.0.0)
+      net-ssh (>= 5.0.0, < 8.0.0)
     net-smtp (0.4.0.1)
       net-protocol
-    net-ssh (7.2.1)
+    net-ssh (7.3.0)
     nio4r (2.7.0)
     nokogiri (1.16.2)
       mini_portile2 (~> 2.8.2)
@@ -243,6 +246,7 @@ GEM
       devise (>= 4.9)
       omniauth (>= 2.0)
     orm_adapter (0.5.0)
+    ostruct (0.6.3)
     parallel (1.24.0)
     parser (3.3.0.2)
       ast (~> 2.4.1)
@@ -296,7 +300,7 @@ GEM
       thor (~> 1.0, >= 1.2.2)
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
-    rake (13.1.0)
+    rake (13.3.0)
     rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
@@ -414,10 +418,13 @@ GEM
       sprockets (>= 3.0.0)
     sqlite3 (1.7.0)
       mini_portile2 (~> 2.8.0)
-    sshkit (1.21.7)
-      mutex_m
+    sshkit (1.24.0)
+      base64
+      logger
       net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
+      ostruct
     stringio (3.1.0)
     thor (1.3.0)
     tilt (2.3.0)
@@ -454,7 +461,7 @@ DEPENDENCIES
   best_type (~> 0.0.10)
   bootsnap (>= 1.4.2)
   byebug
-  capistrano (~> 3.18.0)
+  capistrano (~> 3.19.0)
   capistrano-cul
   capistrano-passenger (~> 0.1)
   capistrano-rails (~> 1.4)

app/controllers/concerns/triclops/iiif/images_controller/raster_opt_fallback_logic.rb

@@ -0,0 +1,60 @@
+module Triclops
+  module Iiif
+    module ImagesController
+      module RasterOptFallbackLogic
+        extend ActiveSupport::Concern
+
+        # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/BlockNesting
+        def raster_opts_for_ready_resource_with_fallback(resource, base_type, original_raster_opts, normalized_raster_opts)
+          raster_opts_to_try = normalized_raster_opts.dup
+          cache_hit = resource.raster_exists?(base_type, raster_opts_to_try)
+
+          unless cache_hit
+            Rails.logger.error(
+              "[#{resource.identifier}] "\
+              "Cache MISS: (original_raster_opts: #{original_raster_opts}) "\
+              "(raster_opts_to_try: #{raster_opts_to_try.inspect})"
+            )
+
+            # -- BEGIN temporary fallback code --
+            # Try a backup raster path, using the original size opt
+            raster_opts_to_try = normalized_raster_opts.merge(size: original_raster_opts[:size])
+            cache_hit = resource.raster_exists?(base_type, raster_opts_to_try)
+
+            unless cache_hit
+              Rails.logger.error(
+                "[#{resource.identifier}] "\
+                "Second try: Cache #{cache_hit ? 'HIT' : 'MISS'} for raster_opts_to_try: #{raster_opts_to_try}"
+              )
+
+              # If nothing was found at the backup raster path AND this a request for a 'full' region,
+              # try converting the size to a "!long_side,long_side" size value and see if that version exists in the cache.
+              if normalized_raster_opts[:region] == 'full'
+                # We expect normalized_raster_opts to have a size value of the format: "width,height" in most cases,
+                # but if it doesn't then we should skip the rest of this block.
+                size_opt = normalized_raster_opts[:size]
+                matches = /(\d+),(\d+)/.match(size_opt)
+                if matches
+                  width = matches[1].to_i
+                  height = matches[2].to_i
+                  long_side = width > height ? width : height
+
+                  raster_opts_to_try = normalized_raster_opts.merge(size: "!#{long_side},#{long_side}")
+                  cache_hit = resource.raster_exists?(base_type, raster_opts_to_try)
+                  Rails.logger.error(
+                    "[#{resource.identifier}] "\
+                    "Third try: Cache #{cache_hit ? 'HIT' : 'MISS'} for raster_opts_to_try: #{raster_opts_to_try}"
+                  )
+                end
+              end
+            end
+            # -- END temporary fallback code --
+          end
+
+          [raster_opts_to_try, cache_hit]
+        end
+        # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/BlockNesting
+      end
+    end
+  end
+end

app/controllers/iiif/images_controller.rb

@@ -2,6 +2,7 @@ class Iiif::ImagesController < ApplicationController
   include ActionController::Live
   include Triclops::Iiif::ImagesController::Schemas
   include Triclops::Iiif::ImagesController::Sizing
+  include Triclops::Iiif::ImagesController::RasterOptFallbackLogic
 
   # skip_before_action :verify_authenticity_token, only: [:raster_preflight_check]
   before_action :add_cors_header!, only: [
@@ -41,9 +42,8 @@ def raster
     original_raster_opts = params_validation_result.to_h
     original_raster_opts.delete(:identifier) # :identifier isn't part of our "raster opts"
     base_type = params[:base_type] # :base_type isn't part of our "raster opts"
-    normalized_raster_opts = Triclops::Iiif::RasterOptNormalizer.normalize_raster_opts(@resource, original_raster_opts)
 
-    handle_ready_resource_or_redirect(@resource, base_type, original_raster_opts, normalized_raster_opts)
+    handle_ready_resource_or_redirect(@resource, base_type, original_raster_opts)
   end
 
   def test_viewer
@@ -73,7 +73,7 @@ def validate_image_request_token(token, base_type, resource_identifier, client_i
     Triclops::Utils::TokenUtils.token_is_valid?(token, base_type, resource_identifier, client_ip)
   end
 
-  def handle_ready_resource_or_redirect(resource, base_type, original_raster_opts, normalized_raster_opts)
+  def handle_ready_resource_or_redirect(resource, base_type, original_raster_opts)
     # Whenever a valid resource is requested, cache the Resource identifier in
     # our ResourceAccessStatCache. This cache will be periodically flushed to the
     # Resource database (by a separate process) so that many access time updates
@@ -87,6 +87,7 @@ def handle_ready_resource_or_redirect(resource, base_type, original_raster_opts,
       TRICLOPS[:raster_cache][:access_stats_enabled]
 
     if resource.ready?
+      normalized_raster_opts = Triclops::Iiif::RasterOptNormalizer.normalize_raster_opts(resource, original_raster_opts)
       handle_ready_resource(base_type, original_raster_opts, normalized_raster_opts)
     else
       Rails.logger.debug(
@@ -96,29 +97,25 @@ def handle_ready_resource_or_redirect(resource, base_type, original_raster_opts,
     end
   end
 
-  # rubocop:disable Metrics/MethodLength
   def handle_ready_resource(base_type, original_raster_opts, normalized_raster_opts)
-    cache_hit = @resource.raster_exists?(base_type, normalized_raster_opts)
-    unless cache_hit
-      Rails.logger.error(
-        "[#{@resource.identifier}] "\
-        "Cache MISS: (original_raster_opts: #{original_raster_opts}) "\
-        "(normalized_raster_opts: #{normalized_raster_opts.inspect})"
-      )
-    end
+    raster_opts_to_try, cache_hit = raster_opts_for_ready_resource_with_fallback(
+      @resource, base_type, original_raster_opts, normalized_raster_opts
+    )
+
     if cache_hit || TRICLOPS[:raster_cache][:on_miss] == Triclops::Iiif::Constants::CacheMissMode::GENERATE_AND_CACHE || @resource.source_uri_is_placeholder?
-      @resource.yield_cached_raster(base_type, normalized_raster_opts) do |raster_file|
-        send_raster_file(raster_file, normalized_raster_opts, @resource.updated_at, delivery_method: :send_file)
+      @resource.yield_cached_raster(base_type, raster_opts_to_try) do |raster_file|
+        send_raster_file(raster_file, raster_opts_to_try, @resource.updated_at, delivery_method: :send_file)
       end
     elsif TRICLOPS[:raster_cache][:on_miss] == Triclops::Iiif::Constants::CacheMissMode::GENERATE_AND_DO_NOT_CACHE
-      @resource.yield_uncached_raster(base_type, normalized_raster_opts) do |raster_file|
-        send_raster_file(raster_file, normalized_raster_opts, @resource.updated_at, delivery_method: :send_data)
+      @resource.yield_uncached_raster(base_type, raster_opts_to_try) do |raster_file|
+        send_raster_file(raster_file, raster_opts_to_try, @resource.updated_at, delivery_method: :send_data)
       end
     else # TRICLOPS[:raster_cache][:on_miss] == Triclops::Iiif::Constants::CacheMissMode::ERROR
+      # If we got here, that means we have a cache miss and we will treat this as an error (because we do not expect
+      # clients to request this resource with the given raster opts).  We'll render a 404 response to the user.
       render plain: 'not found', status: :not_found
     end
   end
-  # rubocop:enable Metrics/MethodLength
 
   def error_response(errors)
     { result: false, errors: errors }
@@ -183,13 +180,15 @@ def assign_headers_for_sent_file!(resp, raster_file, modification_time)
     resp['ETag'] = format('"%x"', modification_time)
   end
 
+  def compliance_level_url
+    if TRICLOPS[:raster_cache][:on_miss] == Triclops::Iiif::Constants::CacheMissMode::ERROR
+      'http://iiif.io/api/image/2/level0.json'
+    else
+      'http://iiif.io/api/image/2/level1.json'
+    end
+  end
+
   def assign_compliance_level_header!(resp)
-    compliance_level_url =
-      if TRICLOPS[:raster_cache][:on_miss] == Triclops::Iiif::Constants::CacheMissMode::ERROR
-        'http://iiif.io/api/image/2/level0.json'
-      else
-        'http://iiif.io/api/image/2/level1.json'
-      end
     resp.set_header('Link', compliance_level_url)
   end
 
@@ -203,7 +202,8 @@ def info_json_for_resource(resource, base_type)
       Triclops::Iiif::Constants::ALLOWED_FORMATS.keys,
       Triclops::Iiif::Constants::ALLOWED_QUALITIES,
       Triclops::Iiif::Constants::TILE_SIZE,
-      Imogen::Iiif::Tiles.scale_factors_for(width, height, Triclops::Iiif::Constants::TILE_SIZE)
+      Imogen::Iiif::Tiles.scale_factors_for(width, height, Triclops::Iiif::Constants::TILE_SIZE),
+      compliance_level_url
     )
   end