diff --git a/.asf.yaml b/.asf.yaml index 9a377230..90222824 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -53,6 +53,7 @@ github: # contexts are the names of checks that must pass. contexts: - build_and_unit_test + - code_quality required_pull_request_reviews: dismiss_stale_reviews: false required_approving_review_count: 1 diff --git a/.github/workflows/build_and_unit_test.yml b/.github/workflows/build_and_unit_test.yml index 28f69ecc..f05daa4c 100644 --- a/.github/workflows/build_and_unit_test.yml +++ b/.github/workflows/build_and_unit_test.yml @@ -1,4 +1,4 @@ -name: build_and_unit_test +name: Build and Unit Test on: push: @@ -8,37 +8,220 @@ on: types: [ opened, synchronize, reopened, edited, ready_for_review ] workflow_dispatch: +permissions: + contents: read + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GO_VERSION: '1.21' + GOPATH: ${{ github.workspace }}/.go + GOBIN: ${{ github.workspace }}/.go/bin + jobs: build_and_unit_test: runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 with: fetch-depth: 0 - path: go/src/github.com/apache/cloudberry-backup - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: - go-version: 1.17 + go-version: ${{ env.GO_VERSION }} + cache: true - - name: Set Environment + - name: Setup GOPATH/PATH run: | - echo "GOPATH=/home/runner/work/gpbackup/gpbackup/go" >> $GITHUB_ENV - echo "/home/runner/work/gpbackup/gpbackup/go/bin" >> $GITHUB_PATH + echo "GOPATH=${GOPATH}" >> "$GITHUB_ENV" + echo "${GOBIN}" >> "$GITHUB_PATH" + echo "GOFLAGS=-trimpath" >> "$GITHUB_ENV" + + - name: Install dependencies + run: make depend + + - name: Build binaries + run: make build + + - name: Run unit tests with coverage - - name: Dependencies run: | - cd ${GOPATH}/src/github.com/apache/cloudberry-backup - make depend + make unit + make coverage - - name: Build + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: /tmp/coverage.out + flags: unittests + name: codecov-ubuntu-latest-go${{ env.GO_VERSION }} + fail_ci_if_error: false + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: binaries-ubuntu-latest-go${{ env.GO_VERSION }}-${{ github.sha }} + path: | + ${{ env.GOBIN }}/gpbackup + ${{ env.GOBIN }}/gprestore + ${{ env.GOBIN }}/gpbackup_helper + retention-days: 7 + + # Integration test with Apache Cloudberry + integration_test: + needs: build_and_unit_test + runs-on: ubuntu-22.04 + if: github.event_name == 'push' || (github.event_name == 'pull_request' && !github.event.pull_request.draft) + timeout-minutes: 120 + + container: + image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + options: >- + --privileged + --user root + --hostname cdw + --shm-size=2gb + --ulimit core=-1 + + steps: + - name: Cloudberry Environment Initialization run: | - cd ${GOPATH}/src/github.com/apache/cloudberry-backup - make build + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p build-logs/details + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 build-logs + + - name: Checkout Apache Cloudberry source + uses: actions/checkout@v4 + with: + repository: apache/cloudberry + ref: main + fetch-depth: 1 + persist-credentials: false + path: cloudberry - - name: Unit Test + - name: Checkout backup utility code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + path: cloudberry-backup + + - name: Set up Go in container + run: | + # Install Go 1.21 in the container (retain original approach) + cd /tmp + wget -q https://go.dev/dl/go${{ env.GO_VERSION }}.linux-amd64.tar.gz + tar -C /usr/local -xzf go${{ env.GO_VERSION }}.linux-amd64.tar.gz + echo 'export PATH=$PATH:/usr/local/go/bin' >> /etc/profile + echo 'export GOPATH=/home/gpadmin/go' >> /etc/profile + echo 'export PATH=$PATH:/home/gpadmin/go/bin' >> /etc/profile + echo "/home/gpadmin/go/bin" >> "$GITHUB_PATH" + echo "GOPATH=/home/gpadmin/go" >> "$GITHUB_ENV" + + - name: Configure Apache Cloudberry + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + BUILD_DESTINATION: /usr/local/cloudberry-db + ENABLE_DEBUG: false + run: | + set -eo pipefail + chown -R gpadmin:gpadmin ${SRC_DIR} + + su - gpadmin -c " + cd ${SRC_DIR} + export SRC_DIR=${SRC_DIR} + export BUILD_DESTINATION=${BUILD_DESTINATION} + export ENABLE_DEBUG=${ENABLE_DEBUG} + chmod +x devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + ./devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + " + + - name: Build Apache Cloudberry + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + su - gpadmin -c " + cd ${SRC_DIR} + export SRC_DIR=${SRC_DIR} + chmod +x devops/build/automation/cloudberry/scripts/build-cloudberry.sh + ./devops/build/automation/cloudberry/scripts/build-cloudberry.sh + " + + - name: Create Cloudberry demo cluster + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + NUM_PRIMARY_MIRROR_PAIRS: 1 + run: | + set -eo pipefail + su - gpadmin -c " + cd ${SRC_DIR} + export SRC_DIR=${SRC_DIR} + export NUM_PRIMARY_MIRROR_PAIRS=${NUM_PRIMARY_MIRROR_PAIRS} + chmod +x devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + ./devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + " + + - name: Verify Cloudberry cluster run: | - cd ${GOPATH}/src/github.com/apache/cloudberry-backup - make unit_all_gpdb_versions + su - gpadmin -c " + source /usr/local/cloudberry-db/cloudberry-env.sh + source ${GITHUB_WORKSPACE}/cloudberry/gpAux/gpdemo/gpdemo-env.sh + + gpstate -s + psql -d postgres -c 'SELECT version();' + psql -d postgres -c 'SELECT * FROM gp_segment_configuration;' + " + + - name: Build and test backup utility with Cloudberry + env: + GOPATH: /home/gpadmin/go + run: | + su - gpadmin -c " + export PATH=/usr/local/go/bin:\$PATH + export GOPATH=${GOPATH} + export PATH=\$PATH:\$GOPATH/bin + + source /usr/local/cloudberry-db/cloudberry-env.sh + source ${GITHUB_WORKSPACE}/cloudberry/gpAux/gpdemo/gpdemo-env.sh + + cd ${GITHUB_WORKSPACE}/cloudberry-backup + + mkdir -p \$GOPATH/src/github.com/apache + ln -sf ${GITHUB_WORKSPACE}/cloudberry-backup \$GOPATH/src/github.com/apache/cloudberry-backup + + cd \$GOPATH/src/github.com/apache/cloudberry-backup + + make depend + make build + + # Run unit tests (no DB connection required) + make unit + + echo 'Running integration tests with Cloudberry cluster...' + make integration || echo 'Integration tests completed with some failures (soft fail in CI)' + " + + - name: Upload integration test logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-logs + path: | + build-logs/ + cloudberry/build-logs/ + retention-days: 7 \ No newline at end of file diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml new file mode 100644 index 00000000..7127a485 --- /dev/null +++ b/.github/workflows/code_quality.yml @@ -0,0 +1,74 @@ +name: Code Quality Check + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + types: [ opened, synchronize, reopened, edited, ready_for_review ] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GO_VERSION: '1.21' + GOPATH: ${{ github.workspace }}/.go + GOBIN: ${{ github.workspace }}/.go/bin + +jobs: + code_quality: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Setup GOPATH/PATH + run: | + echo "GOPATH=${GOPATH}" >> "$GITHUB_ENV" + echo "${GOBIN}" >> "$GITHUB_PATH" + + - name: Install dependencies + run: make depend + + - name: Formatting check (read-only) + run: | + set -eo pipefail + command -v goimports >/dev/null || go install golang.org/x/tools/cmd/goimports@latest + files=$(git ls-files '*.go') + if [ -z "$files" ]; then + echo "No Go files found. Skipping formatting check." + exit 0 + fi + CHANGED=$(goimports -l $files) + if [ -n "$CHANGED" ]; then + echo "$CHANGED" + echo "### Formatting issues" >> "$GITHUB_STEP_SUMMARY" + echo "$CHANGED" | sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" + echo "Please run \`make format\` on your local machine" >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + + - name: GolangCI-Lint (no PR annotations) + run: | + set -eo pipefail + command -v golangci-lint >/dev/null || go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.56.2 + OUTPUT=$(golangci-lint run --tests=false --out-format=colored-line-number || true) + if [ -n "$OUTPUT" ]; then + echo "$OUTPUT" + echo "### GolangCI-Lint issues" >> "$GITHUB_STEP_SUMMARY" + echo "$OUTPUT" | sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi \ No newline at end of file