From 4666769f7cc3f7709a3a71e3f3da50d0b3435142 Mon Sep 17 00:00:00 2001
From: Ewoud Kohl van Wijngaarden <ewoud@kohlvanwijngaarden.nl>
Date: Mon, 31 Mar 2025 13:34:46 +0200
Subject: [PATCH] Support s3 storage

Pulp supports multiple storage backends via the django-storages package.
This exposes both the file storage (default) and s3.

No specific s3 options are exposed, but rather the whole options hash.
This makes it easily extensible, both for specific s3 tuning and for
future expansion.

It depends on the django-storages Python package in install. This is too
specific now, but changing that would require a packaging change.
---
 manifests/config.pp           | 14 ++++++++++++++
 manifests/init.pp             |  8 ++++++++
 manifests/install.pp          |  7 +++++++
 spec/classes/pulpcore_spec.rb | 17 +++++++++++++++++
 4 files changed, 46 insertions(+)

diff --git a/manifests/config.pp b/manifests/config.pp
index b499d927..0cf6ba6b 100644
--- a/manifests/config.pp
+++ b/manifests/config.pp
@@ -41,6 +41,20 @@
     order   => '02',
   }
 
+  if $pulpcore::storage_backend == 's3' {
+    $storages = {
+      'default' => {
+        'BACKEND' => 'storages.backends.s3.S3Storage',
+        'OPTIONS' => $pulpcore::storage_options,
+      },
+    }
+    concat::fragment { 'storage':
+      target  => 'pulpcore settings',
+      content => "STORAGES = ${stdlib::to_python($storages)}\n",
+      order   => '03',
+    }
+  }
+
   file { $pulpcore::user_home:
     ensure => directory,
     owner  => $pulpcore::user,
diff --git a/manifests/init.pp b/manifests/init.pp
index f80f2527..4d6bfae1 100644
--- a/manifests/init.pp
+++ b/manifests/init.pp
@@ -203,6 +203,12 @@
 #   What percentage of available-workers will pulpcore use for import tasks at a time.
 #   By default, pulpcore will use all available workers.
 #
+# @param storage_backend
+#   Which storage backend to use
+#
+# @param storage_options
+#   Options to pass to the storage backend
+#
 # @example Default configuration
 #   include pulpcore
 #
@@ -268,6 +274,8 @@
   Optional[Boolean] $analytics = undef,
   Optional[Boolean] $hide_guarded_distributions = undef,
   Optional[Integer[1,100]] $import_workers_percent = undef,
+  Enum['file', 's3'] $storage_backend = 'file',
+  Hash[String[1], Any] $storage_options = {},
 ) {
   $settings_file = "${config_dir}/settings.py"
   $certs_dir = "${config_dir}/certs"
diff --git a/manifests/install.pp b/manifests/install.pp
index 3201d8bb..2b087ff7 100644
--- a/manifests/install.pp
+++ b/manifests/install.pp
@@ -12,6 +12,13 @@
     }
   }
 
+  if $pulpcore::storage_backend != 'file' {
+    # TODO: better virtual for this
+    package { 'python3.11-django-storages':
+      ensure => present,
+    }
+  }
+
   user { $pulpcore::user:
     ensure     => present,
     system     => true,
diff --git a/spec/classes/pulpcore_spec.rb b/spec/classes/pulpcore_spec.rb
index 85aa3966..b8781d53 100644
--- a/spec/classes/pulpcore_spec.rb
+++ b/spec/classes/pulpcore_spec.rb
@@ -12,6 +12,7 @@
           is_expected.to contain_class('pulpcore::install')
           is_expected.to contain_package('pulpcore')
           is_expected.to contain_package('pulpcore-selinux')
+          is_expected.not_to contain_package('python3.11-django-storages')
           is_expected.to contain_user('pulp').with_gid('pulp').with_home('/var/lib/pulp')
           is_expected.to contain_group('pulp')
         end
@@ -49,6 +50,7 @@
                 },
             }
           LOGGING
+          is_expected.not_to contain_concat__fragment('storage')
           is_expected.to contain_file('/etc/pulp')
           is_expected.to contain_file('/etc/pulp/certs/database_fields.symmetric.key')
           is_expected.to contain_file('/var/lib/pulp')
@@ -597,6 +599,21 @@
         end
       end
 
+      context 'with s3 storage' do
+        let :params do
+          { storage_backend: 's3' }
+        end
+
+        it { is_expected.to compile.with_all_deps }
+        it { is_expected.to contain_package('python3.11-django-storages') }
+
+        it 'configures pulpcore with setting STORAGES' do
+          is_expected.to contain_concat__fragment('storage').with_content(<<~STORAGE)
+            STORAGES = {"default": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": {}}}
+          STORAGE
+        end
+      end
+
       context 'with parameter worker_ttl = 60' do
         let :params do
           { worker_ttl: 60 }