diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..fe92ae8 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,219 @@ +name: Build / Test / Push + +on: + push: + branches: + - '**' + workflow_call: + workflow_dispatch: + +env: + BUILD_SUFFIX: -build-${{ github.run_id }}_${{ github.run_attempt }} + DOCKER_METADATA_SET_OUTPUT_ENV: 'true' + +jobs: + build: + runs-on: ${{ matrix.runner }} + outputs: + build-image-arm: ${{ steps.gen-output.outputs.image-arm64 }} + build-image-x64: ${{ steps.gen-output.outputs.image-x64 }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: build-meta + name: Produce the build image tag + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha,suffix=${{ env.BUILD_SUFFIX }} + + # Build cache is shared among all builds of the same architecture + - id: cache-meta + name: Fetch build cache metadata + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=raw,value=buildcache-${{ runner.arch }} + + - id: get-registry + name: Get the sanitized registry name + run: | + echo "registry=$(echo '${{ steps.build-meta.outputs.tags }}' | cut -f1 -d:)" | tee -a "$GITHUB_OUTPUT" + + - id: set_build_url + name: Set BUILD_URL + run: | + echo "build_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | tee -a "$GITHUB_OUTPUT" + + - id: build + name: Build/push the arch-specific image + uses: docker/build-push-action@v6 + with: + platforms: ${{ matrix.platform }} + build-args: | + BUILD_TIMESTAMP=${{ github.event.repository.updated_at }} + BUILD_URL=${{ steps.set_build_url.outputs.build_url }} + GIT_REF_NAME=${{ github.ref_name }} + GIT_SHA=${{ github.sha }} + GIT_REPOSITORY_URL=${{ github.repositoryUrl }} + cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} + cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max + labels: ${{ steps.build-meta.outputs.labels }} + provenance: mode=max + sbom: true + tags: ${{ steps.get-registry.outputs.registry }} + outputs: type=image,push-by-digest=true,push=true + + - id: gen-output + name: Write arch-specific image digest to outputs + run: | + echo "image-${RUNNER_ARCH,,}=${{ steps.get-registry.outputs.registry }}@${{ steps.build.outputs.digest }}" | tee -a "$GITHUB_OUTPUT" + + merge: + runs-on: ubuntu-latest + needs: + - build + env: + DOCKER_APP_IMAGE_ARM64: ${{ needs.build.outputs.build-image-arm }} + DOCKER_APP_IMAGE_X64: ${{ needs.build.outputs.build-image-x64 }} + outputs: + build-image: ${{ steps.meta.outputs.tags }} + build-image-arm: ${{ needs.build.outputs.build-image-arm }} + build-image-x64: ${{ needs.build.outputs.build-image-x64 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha,suffix=-build-${{ github.run_id }}_${{ github.run_attempt }} + + - name: Push the multi-platform image + run: | + docker buildx imagetools create \ + --tag "$DOCKER_METADATA_OUTPUT_TAGS" \ + "$DOCKER_APP_IMAGE_ARM64" "$DOCKER_APP_IMAGE_X64" + + test: + runs-on: ubuntu-24.04 + needs: + - merge + env: + COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml + DOCKER_APP_IMAGE: ${{ needs.merge.outputs.build-image }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Compose + uses: docker/setup-compose-action@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup the stack + run: | + docker compose build --quiet + docker compose pull --quiet + docker compose up --wait + docker compose exec -u root app chown -R alma:alma artifacts + + - name: Run RSpec + if: ${{ always() }} + run: | + docker compose exec app rspec --format progress --format html --out artifacts/rspec.html + + - name: Run Rubocop + if: ${{ always() }} + run: | + docker compose exec app rubocop --format progress --format html --out artifacts/rubocop.html + + - name: Copy out artifacts + if: ${{ always() }} + run: | + docker compose cp app:/opt/app/artifacts ./ + docker compose logs > artifacts/docker-compose-services.log + docker compose config > artifacts/docker-compose.merged.yml + + - name: Upload the test report + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: SFTP_HANDLER Build Report (${{ github.run_id }}_${{ github.run_attempt }}) + path: artifacts/* + if-no-files-found: error + + push: + runs-on: ubuntu-24.04 + needs: + - merge + - test + env: + DOCKER_APP_IMAGE: ${{ needs.merge.outputs.build-image }} + DOCKER_APP_IMAGE_ARM64: ${{ needs.merge.outputs.build-image-arm }} + DOCKER_APP_IMAGE_X64: ${{ needs.merge.outputs.build-image-x64 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Produce permanent image tags + id: branch-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + + - name: Retag and push the image + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("--tag " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $DOCKER_APP_IMAGE_ARM64 $DOCKER_APP_IMAGE_X64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b9a9c62 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,61 @@ +name: Push Release Tags + +on: + push: + tags: + - '**' + workflow_call: + workflow_dispatch: + +env: + DOCKER_METADATA_SET_OUTPUT_ENV: 'true' + +jobs: + retag: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Determine the sha-based image tag to retag + id: get-base-image + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha + + - name: Verify that the image was previously built + env: + BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }} + run: | + docker manifest inspect "$BASE_IMAGE" + + - name: Produce release tags + id: tag-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + flavor: latest=false + tags: | + type=ref,event=tag + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + + - name: Retag the pulled image + env: + BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }} + run: | + docker buildx imagetools create \ + $(jq -cr '.tags | map("--tag " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + "$(echo "$BASE_IMAGE" | cut -f1 -d:)" diff --git a/Gemfile.lock b/Gemfile.lock index 7da6a7c..51ad66d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -62,6 +62,7 @@ GEM hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS + aarch64-linux x86_64-darwin-19 x86_64-linux diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..ad1fb80 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 The Regents of the University of California + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..5360464 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,13 @@ +services: + app: + build: !reset + image: ${DOCKER_APP_IMAGE} + environment: !override + LBNL_FILENAME: "test" + entrypoint: ["tail", "-f", "/dev/null"] + volumes: !override + - artifacts:/opt/app/artifacts + +volumes: + artifacts: + diff --git a/docker-compose.yml b/docker-compose.yml index 4d1ad65..5bfddbd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,10 +5,10 @@ services: target: development environment: LIT_GOBI_PASSWORD: "${LIT_GOBI_PASSWORD}" + LIT_GOBI_USERNAME: "${LIT_GOBI_USERNAME}" LIT_LBNL_KEY_DATA: "${LIT_LBNL_KEY_DATA}" - image: containers.lib.berkeley.edu/lap/sftp_handler/${USER:-development}:latest - init: yes + LIT_LBNL_USERNAME: "${LIT_LBNL_USERNAME}" + LBNL_FILENAME: "${LBNL_FILENAME}" + init: true volumes: - ./:/opt/app - -version: "3.8" diff --git a/env.sample b/env.sample new file mode 100644 index 0000000..a619def --- /dev/null +++ b/env.sample @@ -0,0 +1,5 @@ +LIT_GOBI_PASSWORD='password' +LIT_GOBI_USERNAME='username' +LIT_LBNL_KEY_DATA='ssh key' +LIT_LBNL_USERNAME='username' +LBNL_FILENAME='the base name for the lbnl file' diff --git a/lib/berkeley_library/sftp_handler/downloader/gobi.rb b/lib/berkeley_library/sftp_handler/downloader/gobi.rb index 11e4093..b5a9934 100644 --- a/lib/berkeley_library/sftp_handler/downloader/gobi.rb +++ b/lib/berkeley_library/sftp_handler/downloader/gobi.rb @@ -43,7 +43,7 @@ def default_host end def default_username - ENV['LIT_GOBI_USERNAME]' + ENV['LIT_GOBI_USERNAME'] end def ssh_options diff --git a/lib/berkeley_library/sftp_handler/downloader/lbnl.rb b/lib/berkeley_library/sftp_handler/downloader/lbnl.rb index a61ee43..1c9e1ae 100644 --- a/lib/berkeley_library/sftp_handler/downloader/lbnl.rb +++ b/lib/berkeley_library/sftp_handler/downloader/lbnl.rb @@ -5,7 +5,7 @@ module BerkeleyLibrary module SftpHandler module Downloader - # Downloads the latest lbnl_people_yyyymmdd.zip patron file. + # Downloads the latest ${LBNL_FILENAME}_yyyymmdd.zip patron file. # # LBNL seems to populate this every Monday, so by default the class looks for the file # that would've been added on the most recent Monday. If today is Monday, it uses today's @@ -35,7 +35,7 @@ def default_host end def default_username - ENV['LIT_LBNL_USERNAME'] + ENV['LIT_LBNL_USERNAME'] end def default_filename diff --git a/spec/downloader/lbnl_spec.rb b/spec/downloader/lbnl_spec.rb index d95c50e..2db32d3 100644 --- a/spec/downloader/lbnl_spec.rb +++ b/spec/downloader/lbnl_spec.rb @@ -9,8 +9,8 @@ describe '#download!' do let(:now) { Time.new(2022, 5, 25, 0, 0, 0) } - let(:last_weeks_filename) { 'lbnl_people_20220516.zip' } - let(:this_weeks_filename) { 'lbnl_people_20220523.zip' } + let(:last_weeks_filename) { 'test_20220516.zip' } + let(:this_weeks_filename) { 'test_20220523.zip' } let(:todays_remote_path) { Pathname.new(this_weeks_filename) } let(:sftp_session) { instance_double(Net::SFTP::Session) } @@ -44,7 +44,7 @@ end it 'downloads a specific file' do - filename = 'lbnl_people_20220516.zip' + filename = 'test_20220516.zip' remote_path = Pathname.new(filename) local_path = Pathname.new("/opt/app/data/#{filename}") @@ -76,7 +76,7 @@ context 'on Sunday May 29, 2022' do its(:default_filename) do Timecop.freeze(Time.new(2022, 5, 29, 23, 59, 59)) do - is_expected.to eq 'lbnl_people_20220523.zip' + is_expected.to eq 'test_20220523.zip' end end end @@ -84,7 +84,7 @@ context 'on Monday May 30, 2022' do its(:default_filename) do Timecop.freeze(Time.new(2022, 5, 30, 23, 59, 59)) do - is_expected.to eq 'lbnl_people_20220530.zip' + is_expected.to eq 'test_20220530.zip' end end end @@ -92,7 +92,7 @@ context 'on Tuesday May 31, 2022' do its(:default_filename) do Timecop.freeze(Time.new(2022, 5, 31, 0, 0, 0)) do - is_expected.to eq 'lbnl_people_20220530.zip' + is_expected.to eq 'test_20220530.zip' end end end @@ -100,7 +100,7 @@ context 'on Wednesday May 25, 2022' do its(:default_filename) do Timecop.freeze(Time.new(2022, 5, 25, 23, 59, 59)) do - is_expected.to eq 'lbnl_people_20220523.zip' + is_expected.to eq 'test_20220523.zip' end end end