LIBAVALON-373. Configure local development environment to use Minio S3 storage.

config/environments/development.rb

@@ -34,7 +34,7 @@
   end
 
   # Store uploaded files on the local file system (see config/storage.yml for options).
-  config.active_storage.service = :local
+  config.active_storage.service = :generic_s3
 
   # Don't care if the mailer can't send.
   config.action_mailer.raise_delivery_errors = false

config/settings.yml

@@ -6,10 +6,8 @@ domain:
   protocol: http
 # End UMD Customization
 dropbox:
-  # UMD Customization
-  path: /masterfiles/dropbox
-  upload_uri: /masterfiles/dropbox
-  # End UMD Customization
+  path: /srv/avalon/dropbox
+  upload_uri: sftp://avalon.example.edu
   # google_drive:
   #   client_id: id
   #   client_secret: secret

docker-compose.yml

@@ -8,10 +8,7 @@ volumes:
   fedora:
   solr:
   npms:
-  # UMD Customization
-  # data:
-  work:
-  # End UMD Customization
+  data:
 
 networks:
   internal:
@@ -69,19 +66,20 @@ services:
 
   hls:
     # UMD Customization
-    image: docker.lib.umd.edu/nginx:avalon-7.6-umd-0
+    image: docker.lib.umd.edu/nginx:avalon-7.6-umd-LIBAVALON-373
     platform: linux/amd64
     # End UMD Customization
     environment:
       - AVALON_DOMAIN=http://avalon:3000
-      - AVALON_STREAMING_BASE_URL=http://localhost:8880
+      - AVALON_STREAMING_BUCKET_URL=http://minio:9000/derivatives/
     # UMD Customization
     volumes:
       - streaming:/data
     # End UMD Customization
     ports:
       - '8880:80'
     networks:
+      internal:
       external:
 
   redis: &redis
@@ -105,30 +103,46 @@ services:
       - solr
       - redis
       - hls
+      - minio
     environment:
       # UMD Customization
       - APP_NAME=avalon
       - SETTINGS__DOMAIN__PROTOCOL=http
       - SETTINGS__DOMAIN_HOST=av-local
       - SETTINGS__DOMAIN_PORT=3000
       - BUNDLE_FLAGS=--with development postgres --without production test
-      - ENCODE_WORK_DIR=/streamfiles
+      - ENCODE_WORK_DIR=/tmp
       - CONTROLLED_VOCABULARY=config/controlled_vocabulary.yml
       - DATABASE_URL=postgres://postgres:password@db/avalon
       - DIGITAL_COLLECTIONS_URL=https://digital.lib.umd.edu/
       - DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL=true
       - FEDORA_NAMESPACE=avalon
       - FEDORA_URL=http://fedoraAdmin:fedoraAdmin@fedora:8080/fedora/rest
+      - DIGITAL_COLLECTIONS_URL=https://digital.lib.umd.edu/rest
       - RAILS_ENV=development
       - RAILS_ADDITIONAL_HOSTS=avalon
       - SETTINGS__REDIS__HOST=redis
       - SETTINGS__REDIS__PORT=6379
       - SECRET_KEY_BASE=abcd
       - SOLR_URL=http://solr:8983/solr/avalon
-      - SETTINGS__STREAMING__CONTENT_PATH=/streamfiles
+      # - SETTINGS__STREAMING__CONTENT_PATH=/streamfiles
+      - AWS_REGION=us-east-1
+      - SETTINGS__ACTIVE_STORAGE__BUCKET=supplementalfiles
+      - SETTINGS__ACTIVE_STORAGE__SERVICE=generic_s3
       - SETTINGS__FFMPEG__PATH=/usr/bin/ffmpeg
+      - SETTINGS__MINIO__ENDPOINT=http://minio:9000
+      - SETTINGS__MINIO__PUBLIC_HOST=http://localhost:9000
+      - SETTINGS__MINIO__ACCESS=minio
+      - SETTINGS__MINIO__SECRET=minio123
+      - SETTINGS__ENCODING__MASTERFILE_BUCKET=masterfiles
+      - SETTINGS__ENCODING__DERIVATIVE_BUCKET=derivatives
+      - SETTINGS__DROPBOX__PATH=s3://masterfiles/dropbox/
+      - SETTINGS__DROPBOX__UPLOAD_URI=s3://masterfiles/dropbox/
+      - SETTINGS__MASTER_FILE_MANAGEMENT__PATH=s3://masterfiles/archive/
+      - SETTINGS__MASTER_FILE_MANAGEMENT__STRATEGY=move
+      - SETTINGS__STREAMING__CONTENT_PATH=/
       - SETTINGS__STREAMING__STREAM_DEFAULT_QUALITY=medium
-      - SETTINGS__STREAMING__HTTP_BASE=http://localhost:8880/avalon
+      - SETTINGS__STREAMING__HTTP_BASE=http://localhost:8880/s3-avalon
       - SETTINGS__STREAMING__SERVER=nginx
       - SETTINGS__STREAMING__STREAM_TOKEN_TTL=20
       - SYSTEM_GROUPS=administrator,group_manager,manager
@@ -179,41 +193,44 @@ services:
     <<: *avalon
     command: dumb-init -- bash -c "bundle install && bundle exec sidekiq -C config/sidekiq.yml"
     ports: []
-# UMD Customization
-  # minio:
-  #   image: minio/minio:RELEASE.2019-10-12T01-39-57Z
-  #   command: minio server /data
-  #   environment:
-  #     MINIO_ACCESS_KEY: minio
-  #     MINIO_SECRET_KEY: minio123
-  #   volumes:
-  #     - data:/data
-  #   ports:
-  #     - 9000:9000
-  #   networks:
-  #     internal:
-  #     external:
-  #   healthcheck:
-  #     test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live']
-  #     interval: 30s
-  #     timeout: 20s
-  #     retries: 3
 
-  # createbuckets:
-  #   image: minio/mc
-  #   depends_on:
-  #     - minio
-  #   entrypoint: >
-  #     /bin/sh -c "
-  #     /usr/bin/mc config host add myminio http://minio:9000 minio minio123;
-  #     /usr/bin/mc mb -p myminio/fcrepo myminio/masterfiles myminio/derivatives myminio/supplementalfiles;
-  #     /usr/bin/mc policy set download myminio/derivatives;
-  #     /usr/bin/mc policy set download myminio/supplementalfiles;
-  #     exit 0;
-  #     "
-  #   networks:
-  #     internal:
-# End UMD Customization
+  # UMD Customization
+  # Ported in changes from Avalon 7.8
+  minio:
+    image: minio/minio:RELEASE.2024-01-29T03-56-32Z
+    command: minio server /data --console-address ":9090"
+    environment:
+      MINIO_ACCESS_KEY: minio
+      MINIO_SECRET_KEY: minio123
+    volumes:
+      - data:/data
+    ports:
+      - 9000:9000
+      - 9090:9090
+    networks:
+      internal:
+      external:
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set myminio http://minio:9000 minio minio123;
+      /usr/bin/mc mb -p myminio/fcrepo myminio/masterfiles myminio/derivatives myminio/supplementalfiles myminio/preserves;
+      /usr/bin/mc anonymous set download myminio/derivatives;
+      /usr/bin/mc anonymous set download myminio/supplementalfiles;
+      exit 0;
+      "
+    networks:
+      internal:
+  # End UMD Customization
 
   cypress:
     network_mode: host

umd_docs/DockerDevelopmentEnvironment.md

@@ -17,6 +17,12 @@ of Avalon using Docker.
     127.0.0.1 av-local
     ```
 
+3) Install the Minio Client
+
+    ```zsh
+    brew install minio-mc
+    ```
+
 ### Setup Instructions
 
 1) Checkout the application and switch to the directory:
@@ -93,6 +99,16 @@ of Avalon using Docker.
 
    Avalon should be available at: [http://av-local:3000](http://av-local:3000)
 
+   The MinIO browser should be accessible at <http://localhost:9090/browser>
+
+9) In a separate terminal, run the createbuckets container to create the
+   necessary buckets in MinIO
+
+   ```zsh
+   mc alias set minio http://localhost:9000 minio minio123;
+   docker-compose up createbuckets
+   ```
+
 ### Loading Sample Data
 
 Sample data for Avalon is available in the
@@ -117,49 +133,44 @@ in the folder for a description of each dataset.
 
    then left-click the "Create Collection" button in the dialog. Once created,
    the "Manage Content" page with the collection will be displayed. Also, a
-   "masterfiles/dropbox/Test_Collection/" subdirectory will be created in the
-   Avalon project directory.
+   "dropbox/Test_Collection/" folder will be created in the masterfiles bucket.
 
 3) In a terminal, add a `sample-data@example.com` admin user (which is the
-   email address of the submitter in the sample datasets) by executing a Bash
-   shell in the "avalon-avalon-1" Docker container:
-
-   ```zsh
-   docker exec -it avalon-avalon-1 /bin/bash
-   ```
-
-   and running the following command:
-
-   ```zsh
-   rails avalon:user:create \
-     avalon_username=sample-data@example.com avalon_password=PASSWORD \
-     avalon_groups=administrator
-   ```
-
-   then exit the Docker container:
+   email address of the submitter in the sample datasets) by running the
+   following command:
 
    ```zsh
-   exit
+   docker exec -it avalon-avalon-1 /bin/bash -c \
+      'rails avalon:user:create \
+        avalon_username=sample-data@example.com avalon_password=PASSWORD \
+        avalon_groups=administrator'
    ```
 
 4) Download the "Sample_Audio_and_Video" folder (as a Zip file) from
-   <https://umd.app.box.com/folder/156055839002> and place it in the
-   "masterfiles/dropbox/Test_Collection/" folder. In a terminal,
-   switch to the "masterfiles/dropbox/Test_Collection/" subdirectory:
+   <https://umd.app.box.com/folder/156055839002> and place it in a
+   temporary folder. In a terminal, switch to the directory containing
+   the download and extract the file.
 
    ```zsh
-   cd masterfiles/dropbox/Test_Collection/
+   cd <FOLDER_CONTAINING_THE_DOWNLOAD>
+   unzip Sample_Audio_and_Video.zip
    ```
 
-   and extract the file:
+   Copy the files to the masterfiles bucket dropbox folder. In the following
+   the "UPLOAD_DIR" environment variable contains the name of the extracted
+   folder from the Zip file (to simplify uploading different sample datasets).
 
-   ```zsh
-   unzip Sample_Audio_and_Video.zip
+   ```sh
+   export UPLOAD_DIR=Sample_Audio_and_Video
+   mc mirror --exclude "*.xlsx" $UPLOAD_DIR minio/masterfiles/dropbox/Test_Collection/$UPLOAD_DIR/
+   mc cp -r $UPLOAD_DIR/*.xlsx minio/masterfiles/dropbox/Test_Collection/$UPLOAD_DIR/
    ```
 
-   and extract the file.
+   **Note:** Uploading the assets and the "xlsx" manifest file separately is a
+   best practice, because it ensures that the Avalon "worker" process won't
+   start the import before the asset files are available.
 
-5) The "avalon-worker" container scans the "masterfiles/dropbox" directory
+5) The "avalon-worker" container scans the "masterfiles/dropbox" bucket
    once a minute, and ingests any new items found. Depending on the size
    of the dataset, and the need to transcode the content, the ingest may
    take several minutes.