From b45c5aedf924d934553f8099950e9ff00fb0b141 Mon Sep 17 00:00:00 2001 From: chenhang Date: Thu, 23 Mar 2023 18:11:01 +0800 Subject: [PATCH 1/6] upgrade bk to 4.16.0 for test --- .../server/src/assemble/LICENSE.bin.txt | 50 +++++++++---------- pom.xml | 10 +++- .../prometheus/metrics/LongAdderCounter.java | 9 +++- .../auth/SameThreadOrderedSafeExecutor.java | 5 +- pulsar-sql/presto-distribution/LICENSE | 24 ++++----- .../presto/PulsarConnectorMetricsTracker.java | 6 +-- .../bookkeeper/client/TestStatsProvider.java | 8 ++- 7 files changed, 66 insertions(+), 46 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 6b3455127b423..bdba5f0eee553 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -347,31 +347,31 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-proto-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-server-4.15.4.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.15.4.jar - - org.apache.bookkeeper-circe-checksum-4.15.4.jar - - org.apache.bookkeeper-cpu-affinity-4.15.4.jar - - org.apache.bookkeeper-statelib-4.15.4.jar - - org.apache.bookkeeper-stream-storage-api-4.15.4.jar - - org.apache.bookkeeper-stream-storage-common-4.15.4.jar - - org.apache.bookkeeper-stream-storage-java-client-4.15.4.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.15.4.jar - - org.apache.bookkeeper-stream-storage-proto-4.15.4.jar - - org.apache.bookkeeper-stream-storage-server-4.15.4.jar - - org.apache.bookkeeper-stream-storage-service-api-4.15.4.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.15.4.jar - - org.apache.bookkeeper.http-http-server-4.15.4.jar - - org.apache.bookkeeper.http-vertx-http-server-4.15.4.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.4.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.4.jar - - org.apache.distributedlog-distributedlog-common-4.15.4.jar - - org.apache.distributedlog-distributedlog-core-4.15.4-tests.jar - - org.apache.distributedlog-distributedlog-core-4.15.4.jar - - org.apache.distributedlog-distributedlog-protocol-4.15.4.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.4.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.0.jar + - org.apache.bookkeeper-circe-checksum-4.16.0.jar + - org.apache.bookkeeper-cpu-affinity-4.16.0.jar + - org.apache.bookkeeper-statelib-4.16.0.jar + - org.apache.bookkeeper-stream-storage-api-4.16.0.jar + - org.apache.bookkeeper-stream-storage-common-4.16.0.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.0.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.0.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.0.jar + - org.apache.bookkeeper-stream-storage-server-4.16.0.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.0.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.0.jar + - org.apache.bookkeeper.http-http-server-4.16.0.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.0.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.0.jar + - org.apache.distributedlog-distributedlog-common-4.16.0.jar + - org.apache.distributedlog-distributedlog-core-4.16.0-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.0.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.0.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.0.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/pom.xml b/pom.xml index 3f33069b8c252..f2bbb649f5beb 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.15.4 + 4.16.0 3.8.1 1.5.0 1.10.0 @@ -2455,5 +2455,13 @@ flexible messaging model and an intuitive client API. false + + bk-staging + bk-staging + https://repository.apache.org/content/repositories/orgapachebookkeeper-1081/ + + false + + diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java index ff9d9302456d1..8ade2bc883f9a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.stats.Counter; @@ -50,10 +51,16 @@ public void dec() { } @Override - public void add(long delta) { + public void addCount(long delta) { counter.add(delta); } + @Override + public void addLatency(long eventLatency, TimeUnit unit) { + long valueMillis = unit.toMillis(eventLatency); + counter.add(valueMillis); + } + @Override public Long get() { return counter.sum(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java index 5f6a32a61bd25..258188a31f5d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/SameThreadOrderedSafeExecutor.java @@ -21,7 +21,6 @@ import io.netty.util.concurrent.DefaultThreadFactory; import org.apache.bookkeeper.common.util.OrderedExecutor; -import org.apache.bookkeeper.common.util.SafeRunnable; import org.apache.bookkeeper.stats.NullStatsLogger; public class SameThreadOrderedSafeExecutor extends OrderedExecutor { @@ -46,12 +45,12 @@ public void execute(Runnable r) { } @Override - public void executeOrdered(int orderingKey, SafeRunnable r) { + public void executeOrdered(int orderingKey, Runnable r) { r.run(); } @Override - public void executeOrdered(long orderingKey, SafeRunnable r) { + public void executeOrdered(long orderingKey, Runnable r) { r.run(); } diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index c523fae7606e1..4ec81853149b1 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -424,18 +424,18 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.15.4.jar - - bookkeeper-common-allocator-4.15.4.jar - - bookkeeper-proto-4.15.4.jar - - bookkeeper-server-4.15.4.jar - - bookkeeper-stats-api-4.15.4.jar - - bookkeeper-tools-framework-4.15.4.jar - - circe-checksum-4.15.4.jar - - codahale-metrics-provider-4.15.4.jar - - cpu-affinity-4.15.4.jar - - http-server-4.15.4.jar - - prometheus-metrics-provider-4.15.4.jar - - codahale-metrics-provider-4.15.4.jar + - bookkeeper-common-4.16.0.jar + - bookkeeper-common-allocator-4.16.0.jar + - bookkeeper-proto-4.16.0.jar + - bookkeeper-server-4.16.0.jar + - bookkeeper-stats-api-4.16.0.jar + - bookkeeper-tools-framework-4.16.0.jar + - circe-checksum-4.16.0.jar + - codahale-metrics-provider-4.16.0.jar + - cpu-affinity-4.16.0.jar + - http-server-4.16.0.jar + - prometheus-metrics-provider-4.16.0.jar + - codahale-metrics-provider-4.16.0.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java index b586063cc3c9a..12ee2da463c40 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java @@ -185,7 +185,7 @@ public void end_ENTRY_QUEUE_DEQUEUE_WAIT_TIME() { public void register_BYTES_READ(long bytes) { if (statsLogger != null) { bytesReadSum += bytes; - statsLoggerBytesRead.add(bytes); + statsLoggerBytesRead.addCount(bytes); } } @@ -220,7 +220,7 @@ public void end_MESSAGE_QUEUE_ENQUEUE_WAIT_TIME() { public void incr_NUM_MESSAGES_DESERIALIZED_PER_ENTRY() { if (statsLogger != null) { numMessagedDerserializedPerBatch++; - statsLoggerNumMessagesDeserialized.add(1); + statsLoggerNumMessagesDeserialized.addCount(1); } } @@ -295,7 +295,7 @@ public void end_RECORD_DESERIALIZE_TIME() { public void incr_NUM_RECORD_DESERIALIZED() { if (statsLogger != null) { - statsLoggerNumRecordDeserialized.add(1); + statsLoggerNumRecordDeserialized.addCount(1); } } diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java index f015306cb048c..4d08a7f80df5b 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java @@ -59,10 +59,16 @@ public void dec() { } @Override - public void add(long delta) { + public void addCount(long delta) { updateMax(val.addAndGet(delta)); } + @Override + public void addLatency(long eventLatency, TimeUnit unit) { + long valueMillis = unit.toMillis(eventLatency); + updateMax(val.addAndGet(valueMillis)); + } + @Override public Long get() { return val.get(); From 848b84f0f09681c6cd295a2369d7d95508578b44 Mon Sep 17 00:00:00 2001 From: chenhang Date: Thu, 23 Mar 2023 19:56:53 +0800 Subject: [PATCH 2/6] fix license --- distribution/server/src/assemble/LICENSE.bin.txt | 4 +++- distribution/shell/src/assemble/LICENSE.bin.txt | 6 +++--- pulsar-sql/presto-distribution/LICENSE | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index bdba5f0eee553..9205e072f7efe 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -343,7 +343,6 @@ The Apache Software License, Version 2.0 - org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar - org.apache.logging.log4j-log4j-web-2.18.0.jar * Java Native Access JNA - - net.java.dev.jna-jna-5.12.1.jar - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper @@ -372,6 +371,9 @@ The Apache Software License, Version 2.0 - org.apache.distributedlog-distributedlog-core-4.16.0.jar - org.apache.distributedlog-distributedlog-protocol-4.16.0.jar - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.0.jar + - org.apache.bookkeeper-native-io-4.16.0.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 90896790b1fba..0ac08caa1c01d 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -390,9 +390,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.15.4.jar - - cpu-affinity-4.15.4.jar - - circe-checksum-4.15.4.jar + - bookkeeper-common-allocator-4.16.0.jar + - cpu-affinity-4.16.0.jar + - circe-checksum-4.16.0.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 4ec81853149b1..0c73a9908182e 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -436,6 +436,9 @@ The Apache Software License, Version 2.0 - http-server-4.16.0.jar - prometheus-metrics-provider-4.16.0.jar - codahale-metrics-provider-4.16.0.jar + - bookkeeper-slogger-api-4.16.0.jar + - bookkeeper-slogger-slf4j-4.16.0.jar + - native-io-4.16.0.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar @@ -476,6 +479,8 @@ The Apache Software License, Version 2.0 - amqp-client-5.5.3.jar * Stream Lib - stream-2.9.5.jar + * High Performance Primitive Collections for Java + - hppc-0.9.1.jar Protocol Buffers License From 5018116a99422de8204a9832acb2802e07d96c20 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sun, 26 Mar 2023 13:27:27 +0800 Subject: [PATCH 3/6] use 4.16.0 rc1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f2bbb649f5beb..5367b211c6ef9 100644 --- a/pom.xml +++ b/pom.xml @@ -2458,7 +2458,7 @@ flexible messaging model and an intuitive client API. bk-staging bk-staging - https://repository.apache.org/content/repositories/orgapachebookkeeper-1081/ + https://repository.apache.org/content/repositories/orgapachebookkeeper-1082/ false From 079377e2702ecb38240eebbe6f3df0306dda685d Mon Sep 17 00:00:00 2001 From: chenhang Date: Sun, 26 Mar 2023 17:52:52 +0800 Subject: [PATCH 4/6] fix failed test --- .../broker/stats/PrometheusMetricsTest.java | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 13e67762ace6f..e6573ce88971d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1204,24 +1204,7 @@ public void testManagedLedgerBookieClientStats() throws Exception { System.out.println(e.getKey() + ": " + e.getValue()) ); - List cm = (List) metrics.get(keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_completed_tasks")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( - keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_queue")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( - keyNameBySubstrings(metrics, - "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_total_tasks")); - assertEquals(cm.size(), 1); - assertEquals(cm.get(0).tags.get("cluster"), "test"); - - cm = (List) metrics.get( + List cm = (List) metrics.get( keyNameBySubstrings(metrics, "pulsar_managedLedger_client", "bookkeeper_ml_scheduler_threads")); assertEquals(cm.size(), 1); From 09e2aed3488232b5dc623316f9baabb83075a049 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sun, 26 Mar 2023 20:35:03 +0800 Subject: [PATCH 5/6] debug ci --- .github/workflows/pulsar-ci.yaml | 1219 +----------------------------- 1 file changed, 3 insertions(+), 1216 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index f7dbc755264d6..de1576600bafb 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -43,133 +43,6 @@ env: ARTIFACT_RETENTION_DAYS: 3 jobs: - preconditions: - name: Preconditions - runs-on: ubuntu-20.04 - if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar') - outputs: - docs_only: ${{ steps.check_changes.outputs.docs_only }} - changed_tests: ${{ steps.changes.outputs.tests_files }} - need_owasp: ${{ steps.changes.outputs.need_owasp }} - collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: | - if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then - echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT - else - echo docs_only=false >> $GITHUB_OUTPUT - fi - - - name: Check if coverage should be collected - id: check_coverage - run: | - echo "collect_coverage=${{ - (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' - && (github.base_ref == 'master' || github.ref_name == 'master')) - || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true') - }}" >> $GITHUB_OUTPUT - - - name: Check if the PR has been approved for testing - if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} - env: - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - GITHUB_TOKEN: ${{ github.token }} - run: | - build/pulsar_ci_tool.sh check_ready_to_test - - build-and-license-check: - needs: preconditions - name: Build and License check - env: - JOB_NAME: Build and License check - runs-on: ubuntu-20.04 - timeout-minutes: 60 - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache local Maven repository - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: Check source code license headers - run: mvn -B -T 8 -ntp initialize apache-rat:check license:check - - - name: Check source code style - run: mvn -B -T 8 -ntp initialize checkstyle:check - - - name: Build core-modules - run: | - mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true - - - name: Check binary licenses - run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Save maven build results to Github artifact cache so that the results can be reused - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \ - tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \ - -I zstd -cf - .m2/repository/org/apache/pulsar - cd $GITHUB_WORKSPACE - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \ - tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - unit-tests: name: CI - Unit - ${{ matrix.name }} env: @@ -190,20 +63,6 @@ jobs: group: BROKER_GROUP_1 - name: Brokers - Broker Group 2 group: BROKER_GROUP_2 - - name: Brokers - Broker Group 3 - group: BROKER_GROUP_3 - - name: Brokers - Client Api - group: BROKER_CLIENT_API - - name: Brokers - Client Impl - group: BROKER_CLIENT_IMPL - - name: Proxy - group: PROXY - - name: Pulsar IO - group: PULSAR_IO - timeout: 75 - - name: Pulsar Client - group: CLIENT - steps: - name: checkout uses: actions/checkout@v3 @@ -305,1078 +164,6 @@ jobs: with: action: wait - - unit-tests-upload-coverage: - name: CI - Unit - Upload Coverage - runs-on: ubuntu-20.04 - timeout-minutes: 30 - needs: ['preconditions', 'unit-tests'] - if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache Maven dependencies - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.jdk || '17' }} - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - - - name: Restore coverage files from build artifacts and create Jacoco reports - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report - cd $GITHUB_WORKSPACE/target - zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true - - - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 - with: - name: Jacoco-coverage-report-unittests - path: target/jacoco_test_coverage_report_unittests.zip - retention-days: 3 - - - name: Upload to Codecov - uses: ./.github/actions/upload-coverage - with: - flags: unittests - - - name: Delete coverage files from build artifacts - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - pulsar-java-test-image: - name: Build Pulsar java-test-image docker image - runs-on: ubuntu-20.04 - timeout-minutes: 60 - needs: ['preconditions', 'build-and-license-check'] - if: ${{ needs.preconditions.outputs.docs_only != 'true'}} - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache Maven dependencies - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - - name: Pick ubuntu mirror for the docker image build - run: | - # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror - - - name: Build java-test-image docker image - run: | - # build docker image - mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ - -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \ - -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true - - - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - integration-tests: - name: CI - Integration - ${{ matrix.name }} - runs-on: ubuntu-20.04 - timeout-minutes: ${{ matrix.timeout || 60 }} - needs: ['preconditions', 'pulsar-java-test-image'] - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - env: - JOB_NAME: CI - Integration - ${{ matrix.name }} - PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest - strategy: - fail-fast: false - matrix: - include: - - name: Backwards Compatibility - group: BACKWARDS_COMPAT - no_coverage: true - - - name: Cli - group: CLI - - - name: Messaging - group: MESSAGING - - - name: Shade on Java 8 - group: SHADE_RUN - runtime_jdk: 8 - setup: ./build/run_integration_group.sh SHADE_BUILD - no_coverage: true - - - name: Shade on Java 11 - group: SHADE_RUN - runtime_jdk: 11 - setup: ./build/run_integration_group.sh SHADE_BUILD - no_coverage: true - - - name: Shade on Java 17 - group: SHADE_RUN - setup: ./build/run_integration_group.sh SHADE_BUILD - no_coverage: true - - - name: Standalone - group: STANDALONE - - - name: Transaction - group: TRANSACTION - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache Maven dependencies - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - - - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image - - - name: Run setup commands - if: ${{ matrix.setup }} - run: | - ${{ matrix.setup }} - - - name: Set up runtime JDK ${{ matrix.runtime_jdk }} - uses: actions/setup-java@v3 - if: ${{ matrix.runtime_jdk }} - with: - distribution: 'temurin' - java-version: ${{ matrix.runtime_jdk }} - - - name: Run integration test group '${{ matrix.group }}' - run: | - if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then - coverage_args="--coverage" - fi - ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args - - - name: Upload coverage to build artifacts - if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} - run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }} - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories - if: ${{ always() }} - uses: ./.github/actions/copy-test-reports - - - name: Publish Test Report - uses: apache/pulsar-test-infra/action-junit-report@master - if: ${{ always() }} - with: - report_paths: 'test-reports/TEST-*.xml' - annotate_only: 'true' - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: ${{ !success() }} - with: - name: Integration-${{ matrix.group }}-surefire-reports - path: surefire-reports - retention-days: 7 - - - name: Upload container logs - uses: actions/upload-artifact@v3 - if: ${{ !success() }} - continue-on-error: true - with: - name: Integration-${{ matrix.group }}-container-logs - path: tests/integration/target/container-logs - retention-days: 7 - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - integration-tests-upload-coverage: - name: CI - Integration - Upload Coverage - runs-on: ubuntu-20.04 - timeout-minutes: 30 - needs: ['preconditions', 'integration-tests'] - if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} - env: - PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache Maven dependencies - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - - - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image - - - name: Restore coverage files from build artifacts and create Jacoco reports - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report - cd $GITHUB_WORKSPACE/target - zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true - - - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 - with: - name: Jacoco-coverage-report-inttests - path: target/jacoco_test_coverage_report_inttests.zip - retention-days: 3 - - - name: Upload to Codecov - uses: ./.github/actions/upload-coverage - with: - flags: inttests - - - name: Delete coverage files from build artifacts - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - delete-integration-test-docker-image-artifact: - name: "Delete integration test docker image artifact" - runs-on: ubuntu-20.04 - timeout-minutes: 10 - needs: [ - 'preconditions', - 'integration-tests', - 'integration-tests-upload-coverage' - ] - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Delete docker image from GitHub Actions Artifacts - run: | - gh-actions-artifact-client.js delete pulsar-java-test-image.zst - - pulsar-test-latest-version-image: - name: Build Pulsar docker image - runs-on: ubuntu-20.04 - timeout-minutes: 60 - needs: ['preconditions', 'build-and-license-check'] - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Clean Disk - uses: ./.github/actions/clean-disk - - - name: Cache local Maven repository - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - - name: Pick ubuntu mirror for the docker image build - run: | - # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror - - - name: Build latest-version-image docker image - run: | - # build docker image - # include building of Pulsar SQL, Connectors, Offloaders and server distros - mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ - -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \ - -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true - - # check full build artifacts licenses - - name: Check binary licenses - run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz - - - name: Clean up disk space - run: | - # release disk space since saving docker image consumes local disk space - # - echo "::group::Available diskspace before cleaning" - time df -BM / /mnt - echo "::endgroup::" - echo "::group::Clean build directory" - # docker build changes some files to root ownership, fix this before deleting files - sudo chown -R $USER:$GROUP . - # clean build directories - time git clean -fdx - echo "::endgroup::" - echo "::group::Available diskspace after cleaning build directory" - time df -BM / /mnt - echo "::endgroup::" - echo "::group::Delete maven repository" - # delete maven repository - time rm -rf ~/.m2/repository - echo "::endgroup::" - echo "::group::Available diskspace after cleaning maven repository" - time df -BM / /mnt - echo "::endgroup::" - - - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - system-tests: - name: CI - System - ${{ matrix.name }} - runs-on: ubuntu-20.04 - timeout-minutes: 60 - needs: ['preconditions', 'pulsar-test-latest-version-image'] - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - env: - JOB_NAME: CI - System - ${{ matrix.name }} - PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest - strategy: - fail-fast: false - matrix: - include: - - name: Tiered FileSystem - group: TIERED_FILESYSTEM - - - name: Tiered JCloud - group: TIERED_JCLOUD - - - name: Function - group: FUNCTION - - - name: Schema - group: SCHEMA - - - name: Pulsar Connectors - Thread - group: PULSAR_CONNECTORS_THREAD - - - name: Pulsar Connectors - Process - group: PULSAR_CONNECTORS_PROCESS - - - name: Pulsar IO - group: PULSAR_IO - - - name: Sql - group: SQL - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache local Maven repository - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - - - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image - - - name: Run setup commands - if: ${{ matrix.setup }} - run: | - ${{ matrix.setup }} - - - name: Run system test group '${{ matrix.group }}' - run: | - if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then - coverage_args="--coverage" - fi - ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args - - - name: Upload coverage to build artifacts - if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} - run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }} - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories - if: ${{ always() }} - uses: ./.github/actions/copy-test-reports - - - name: Publish Test Report - uses: apache/pulsar-test-infra/action-junit-report@master - if: ${{ always() }} - with: - report_paths: 'test-reports/TEST-*.xml' - annotate_only: 'true' - - - name: Upload container logs - uses: actions/upload-artifact@v3 - if: ${{ !success() }} - continue-on-error: true - with: - name: System-${{ matrix.group }}-container-logs - path: tests/integration/target/container-logs - retention-days: 7 - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: ${{ !success() }} - with: - name: System-${{ matrix.name }}-surefire-reports - path: surefire-reports - retention-days: 7 - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - system-tests-upload-coverage: - name: CI - System - Upload Coverage - runs-on: ubuntu-20.04 - timeout-minutes: 30 - needs: ['preconditions', 'system-tests'] - if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} - env: - PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache local Maven repository - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image - - - name: Restore coverage files from build artifacts and create Jacoco reports - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report - cd $GITHUB_WORKSPACE/target - zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true - - - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 - with: - name: Jacoco-coverage-report-systests - path: target/jacoco_test_coverage_report_systests.zip - retention-days: 3 - - - name: Upload to Codecov - uses: ./.github/actions/upload-coverage - with: - flags: systests - - - name: Delete coverage files from build artifacts - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - flaky-system-tests: - name: CI Flaky - System - ${{ matrix.name }} - runs-on: ubuntu-20.04 - timeout-minutes: 60 - needs: [ 'preconditions', 'pulsar-test-latest-version-image' ] - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - env: - JOB_NAME: CI Flaky - System - ${{ matrix.name }} - PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest - strategy: - fail-fast: false - matrix: - include: - - name: Plugin - group: PLUGIN - - - name: Pulsar IO - Oracle - group: PULSAR_IO_ORA - - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache local Maven repository - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache - run: | - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image - - - name: Run setup commands - if: ${{ matrix.setup }} - run: | - ${{ matrix.setup }} - - - name: Run system test group '${{ matrix.group }}' - run: | - ./build/run_integration_group.sh ${{ matrix.group }} - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories - if: ${{ always() }} - uses: ./.github/actions/copy-test-reports - - - name: Publish Test Report - uses: apache/pulsar-test-infra/action-junit-report@master - if: ${{ always() }} - with: - report_paths: 'test-reports/TEST-*.xml' - annotate_only: 'true' - - - name: Upload container logs - uses: actions/upload-artifact@v3 - if: ${{ !success() }} - continue-on-error: true - with: - name: System-${{ matrix.group }}-container-logs - path: tests/integration/target/container-logs - retention-days: 7 - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: ${{ !success() }} - with: - name: System-${{ matrix.name }}-surefire-reports - path: surefire-reports - retention-days: 7 - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - delete-system-test-docker-image-artifact: - name: "Delete system test docker image artifact" - runs-on: ubuntu-20.04 - timeout-minutes: 10 - needs: [ - 'preconditions', - 'system-tests', - 'system-tests-upload-coverage', - 'flaky-system-tests' - ] - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Delete docker image from GitHub Actions Artifacts - run: | - gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst - - macos-build: - name: Build Pulsar on MacOS - runs-on: macos-11 - timeout-minutes: 120 - needs: ['preconditions', 'integration-tests'] - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Cache Maven dependencies - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-all- - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - - name: build package - run: mvn -B clean package -DskipTests -T 1C -ntp - - owasp-dep-check: - name: OWASP dependency check - runs-on: ubuntu-20.04 - timeout-minutes: 120 - needs: [ 'preconditions', 'integration-tests' ] - if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - uses: ./.github/actions/ssh-access - continue-on-error: true - with: - limit-access-to-actor: true - - - name: Cache Maven dependencies - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.jdk || '17' }} - - - name: Clean Disk - uses: ./.github/actions/clean-disk - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. - - name: run "clean verify" to trigger dependency check - run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' - - - name: Upload report - uses: actions/upload-artifact@v3 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: dependency report - path: target/dependency-check-report.html - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} - continue-on-error: true - with: - action: wait - - # This job is required for pulls to be merged. - # It depends on all other jobs in this workflow. - # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing. - pulsar-ci-checks-completed: - name: "Pulsar CI checks completed" - if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar')) }} - runs-on: ubuntu-20.04 - timeout-minutes: 10 - needs: [ - 'preconditions', - 'unit-tests', - 'integration-tests', - 'system-tests', - 'flaky-system-tests', - 'macos-build', - 'unit-tests-upload-coverage', - 'integration-tests-upload-coverage', - 'system-tests-upload-coverage', - 'owasp-dep-check' - ] - steps: - - name: Check that all required jobs were completed successfully - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - run: | - if [[ ! ( \ - "${{ needs.unit-tests.result }}" == "success" \ - && "${{ needs.integration-tests.result }}" == "success" \ - && "${{ needs.system-tests.result }}" == "success" \ - && "${{ needs.macos-build.result }}" == "success" \ - ) ]]; then - echo "Required jobs haven't been completed successfully." - exit 1 - fi - - - name: checkout - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - uses: actions/checkout@v3 - - - name: Tune Runner VM - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - uses: ./.github/actions/tune-runner-vm - - - name: Install gh-actions-artifact-client.js - if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Delete maven repository binaries from GitHub Actions Artifacts - if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} - run: | - gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true - gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 From 752d709e25e2be5c45762891fbfc60fa776d69bb Mon Sep 17 00:00:00 2001 From: chenhang Date: Sun, 26 Mar 2023 20:41:45 +0800 Subject: [PATCH 6/6] debug ci --- .github/workflows/pulsar-ci.yaml | 1204 ++++++++++++++++++++++++++++++ 1 file changed, 1204 insertions(+) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index de1576600bafb..b9227ccf45915 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -43,6 +43,133 @@ env: ARTIFACT_RETENTION_DAYS: 3 jobs: + preconditions: + name: Preconditions + runs-on: ubuntu-20.04 + if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar') + outputs: + docs_only: ${{ steps.check_changes.outputs.docs_only }} + changed_tests: ${{ steps.changes.outputs.tests_files }} + need_owasp: ${{ steps.changes.outputs.need_owasp }} + collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Detect changed files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: .github/changes-filter.yaml + list-files: csv + + - name: Check changed files + id: check_changes + run: | + if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT + else + echo docs_only=false >> $GITHUB_OUTPUT + fi + + - name: Check if coverage should be collected + id: check_coverage + run: | + echo "collect_coverage=${{ + (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch' + && (github.base_ref == 'master' || github.ref_name == 'master')) + || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true') + }}" >> $GITHUB_OUTPUT + + - name: Check if the PR has been approved for testing + if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} + env: + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_TOKEN: ${{ github.token }} + run: | + build/pulsar_ci_tool.sh check_ready_to_test + + build-and-license-check: + needs: preconditions + name: Build and License check + env: + JOB_NAME: Build and License check + runs-on: ubuntu-20.04 + timeout-minutes: 60 + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Check source code license headers + run: mvn -B -T 8 -ntp initialize apache-rat:check license:check + + - name: Check source code style + run: mvn -B -T 8 -ntp initialize checkstyle:check + + - name: Build core-modules + run: | + mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true + + - name: Check binary licenses + run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Save maven build results to Github artifact cache so that the results can be reused + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \ + tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \ + -I zstd -cf - .m2/repository/org/apache/pulsar + cd $GITHUB_WORKSPACE + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \ + tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + unit-tests: name: CI - Unit - ${{ matrix.name }} env: @@ -63,6 +190,7 @@ jobs: group: BROKER_GROUP_1 - name: Brokers - Broker Group 2 group: BROKER_GROUP_2 + steps: - name: checkout uses: actions/checkout@v3 @@ -167,3 +295,1079 @@ jobs: - name: Setup tmate session if: failure() uses: mxschmitt/action-tmate@v3 + + + unit-tests-upload-coverage: + name: CI - Unit - Upload Coverage + runs-on: ubuntu-20.04 + timeout-minutes: 30 + needs: ['preconditions', 'unit-tests'] + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK ${{ matrix.jdk || '17' }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk || '17' }} + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v3 + with: + name: Jacoco-coverage-report-unittests + path: target/jacoco_test_coverage_report_unittests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: unittests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + pulsar-java-test-image: + name: Build Pulsar java-test-image docker image + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: ['preconditions', 'build-and-license-check'] + if: ${{ needs.preconditions.outputs.docs_only != 'true'}} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Pick ubuntu mirror for the docker image build + run: | + # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror + + - name: Build java-test-image docker image + run: | + # build docker image + mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ + -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \ + -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true + + - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + integration-tests: + name: CI - Integration - ${{ matrix.name }} + runs-on: ubuntu-20.04 + timeout-minutes: ${{ matrix.timeout || 60 }} + needs: ['preconditions', 'pulsar-java-test-image'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + JOB_NAME: CI - Integration - ${{ matrix.name }} + PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest + strategy: + fail-fast: false + matrix: + include: + - name: Backwards Compatibility + group: BACKWARDS_COMPAT + no_coverage: true + + - name: Cli + group: CLI + + - name: Messaging + group: MESSAGING + + - name: Shade on Java 8 + group: SHADE_RUN + runtime_jdk: 8 + setup: ./build/run_integration_group.sh SHADE_BUILD + no_coverage: true + + - name: Shade on Java 11 + group: SHADE_RUN + runtime_jdk: 11 + setup: ./build/run_integration_group.sh SHADE_BUILD + no_coverage: true + + - name: Shade on Java 17 + group: SHADE_RUN + setup: ./build/run_integration_group.sh SHADE_BUILD + no_coverage: true + + - name: Standalone + group: STANDALONE + + - name: Transaction + group: TRANSACTION + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image + + - name: Run setup commands + if: ${{ matrix.setup }} + run: | + ${{ matrix.setup }} + + - name: Set up runtime JDK ${{ matrix.runtime_jdk }} + uses: actions/setup-java@v3 + if: ${{ matrix.runtime_jdk }} + with: + distribution: 'temurin' + java-version: ${{ matrix.runtime_jdk }} + + - name: Run integration test group '${{ matrix.group }}' + run: | + if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then + coverage_args="--coverage" + fi + ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args + + - name: Upload coverage to build artifacts + if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }} + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload Surefire reports + uses: actions/upload-artifact@v3 + if: ${{ !success() }} + with: + name: Integration-${{ matrix.group }}-surefire-reports + path: surefire-reports + retention-days: 7 + + - name: Upload container logs + uses: actions/upload-artifact@v3 + if: ${{ !success() }} + continue-on-error: true + with: + name: Integration-${{ matrix.group }}-container-logs + path: tests/integration/target/container-logs + retention-days: 7 + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + integration-tests-upload-coverage: + name: CI - Integration - Upload Coverage + runs-on: ubuntu-20.04 + timeout-minutes: 30 + needs: ['preconditions', 'integration-tests'] + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + env: + PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v3 + with: + name: Jacoco-coverage-report-inttests + path: target/jacoco_test_coverage_report_inttests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: inttests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + delete-integration-test-docker-image-artifact: + name: "Delete integration test docker image artifact" + runs-on: ubuntu-20.04 + timeout-minutes: 10 + needs: [ + 'preconditions', + 'integration-tests', + 'integration-tests-upload-coverage' + ] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Delete docker image from GitHub Actions Artifacts + run: | + gh-actions-artifact-client.js delete pulsar-java-test-image.zst + + pulsar-test-latest-version-image: + name: Build Pulsar docker image + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: ['preconditions', 'build-and-license-check'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Clean Disk + uses: ./.github/actions/clean-disk + + - name: Cache local Maven repository + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Pick ubuntu mirror for the docker image build + run: | + # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror + + - name: Build latest-version-image docker image + run: | + # build docker image + # include building of Pulsar SQL, Connectors, Offloaders and server distros + mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ + -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \ + -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true + + # check full build artifacts licenses + - name: Check binary licenses + run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz + + - name: Clean up disk space + run: | + # release disk space since saving docker image consumes local disk space + # + echo "::group::Available diskspace before cleaning" + time df -BM / /mnt + echo "::endgroup::" + echo "::group::Clean build directory" + # docker build changes some files to root ownership, fix this before deleting files + sudo chown -R $USER:$GROUP . + # clean build directories + time git clean -fdx + echo "::endgroup::" + echo "::group::Available diskspace after cleaning build directory" + time df -BM / /mnt + echo "::endgroup::" + echo "::group::Delete maven repository" + # delete maven repository + time rm -rf ~/.m2/repository + echo "::endgroup::" + echo "::group::Available diskspace after cleaning maven repository" + time df -BM / /mnt + echo "::endgroup::" + + - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + system-tests: + name: CI - System - ${{ matrix.name }} + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: ['preconditions', 'pulsar-test-latest-version-image'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + JOB_NAME: CI - System - ${{ matrix.name }} + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + strategy: + fail-fast: false + matrix: + include: + - name: Tiered FileSystem + group: TIERED_FILESYSTEM + + - name: Tiered JCloud + group: TIERED_JCLOUD + + - name: Function + group: FUNCTION + + - name: Schema + group: SCHEMA + + - name: Pulsar Connectors - Thread + group: PULSAR_CONNECTORS_THREAD + + - name: Pulsar Connectors - Process + group: PULSAR_CONNECTORS_PROCESS + + - name: Pulsar IO + group: PULSAR_IO + + - name: Sql + group: SQL + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Run setup commands + if: ${{ matrix.setup }} + run: | + ${{ matrix.setup }} + + - name: Run system test group '${{ matrix.group }}' + run: | + if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then + coverage_args="--coverage" + fi + ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args + + - name: Upload coverage to build artifacts + if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }} + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }} + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload container logs + uses: actions/upload-artifact@v3 + if: ${{ !success() }} + continue-on-error: true + with: + name: System-${{ matrix.group }}-container-logs + path: tests/integration/target/container-logs + retention-days: 7 + + - name: Upload Surefire reports + uses: actions/upload-artifact@v3 + if: ${{ !success() }} + with: + name: System-${{ matrix.name }}-surefire-reports + path: surefire-reports + retention-days: 7 + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + system-tests-upload-coverage: + name: CI - System - Upload Coverage + runs-on: ubuntu-20.04 + timeout-minutes: 30 + needs: ['preconditions', 'system-tests'] + if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} + env: + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v3 + with: + name: Jacoco-coverage-report-systests + path: target/jacoco_test_coverage_report_systests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: systests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + flaky-system-tests: + name: CI Flaky - System - ${{ matrix.name }} + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: [ 'preconditions', 'pulsar-test-latest-version-image' ] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + JOB_NAME: CI Flaky - System - ${{ matrix.name }} + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + strategy: + fail-fast: false + matrix: + include: + - name: Plugin + group: PLUGIN + + - name: Pulsar IO - Oracle + group: PULSAR_IO_ORA + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Run setup commands + if: ${{ matrix.setup }} + run: | + ${{ matrix.setup }} + + - name: Run system test group '${{ matrix.group }}' + run: | + ./build/run_integration_group.sh ${{ matrix.group }} + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'test-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload container logs + uses: actions/upload-artifact@v3 + if: ${{ !success() }} + continue-on-error: true + with: + name: System-${{ matrix.group }}-container-logs + path: tests/integration/target/container-logs + retention-days: 7 + + - name: Upload Surefire reports + uses: actions/upload-artifact@v3 + if: ${{ !success() }} + with: + name: System-${{ matrix.name }}-surefire-reports + path: surefire-reports + retention-days: 7 + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + delete-system-test-docker-image-artifact: + name: "Delete system test docker image artifact" + runs-on: ubuntu-20.04 + timeout-minutes: 10 + needs: [ + 'preconditions', + 'system-tests', + 'system-tests-upload-coverage', + 'flaky-system-tests' + ] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Delete docker image from GitHub Actions Artifacts + run: | + gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst + + macos-build: + name: Build Pulsar on MacOS + runs-on: macos-11 + timeout-minutes: 120 + needs: ['preconditions', 'integration-tests'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-all- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: build package + run: mvn -B clean package -DskipTests -T 1C -ntp + + owasp-dep-check: + name: OWASP dependency check + runs-on: ubuntu-20.04 + timeout-minutes: 120 + needs: [ 'preconditions', 'integration-tests' ] + if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Configure Gradle Enterprise + uses: ./.github/actions/gradle-enterprise + with: + token: ${{ secrets.GE_ACCESS_TOKEN }} + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + - name: Set up JDK ${{ matrix.jdk || '17' }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk || '17' }} + + - name: Clean Disk + uses: ./.github/actions/clean-disk + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. + - name: run "clean verify" to trigger dependency check + run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' + + - name: Upload report + uses: actions/upload-artifact@v3 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: dependency report + path: target/dependency-check-report.html + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + + # This job is required for pulls to be merged. + # It depends on all other jobs in this workflow. + # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing. + pulsar-ci-checks-completed: + name: "Pulsar CI checks completed" + if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/pulsar')) }} + runs-on: ubuntu-20.04 + timeout-minutes: 10 + needs: [ + 'preconditions', + 'unit-tests', + 'integration-tests', + 'system-tests', + 'flaky-system-tests', + 'macos-build', + 'unit-tests-upload-coverage', + 'integration-tests-upload-coverage', + 'system-tests-upload-coverage', + 'owasp-dep-check' + ] + steps: + - name: Check that all required jobs were completed successfully + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + run: | + if [[ ! ( \ + "${{ needs.unit-tests.result }}" == "success" \ + && "${{ needs.integration-tests.result }}" == "success" \ + && "${{ needs.system-tests.result }}" == "success" \ + && "${{ needs.macos-build.result }}" == "success" \ + ) ]]; then + echo "Required jobs haven't been completed successfully." + exit 1 + fi + + - name: checkout + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + uses: actions/checkout@v3 + + - name: Tune Runner VM + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + uses: ./.github/actions/tune-runner-vm + + - name: Install gh-actions-artifact-client.js + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Delete maven repository binaries from GitHub Actions Artifacts + if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} + run: | + gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true + gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true