misc: Updated MEM_POOL_CAP_EXCEEDED error message (#15616)

.github/workflows/gluten.yml

@@ -0,0 +1,225 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Gluten Build
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/gluten.yml
+env:
+  MVN_CMD: mvn -ntp
+
+jobs:
+
+  gluten-cpp-build:
+    name: gluten cpp build
+    # prevent errors when forks ff their main branch
+    if: ${{ github.repository == 'IBM/velox' }}
+    runs-on: ubuntu-22.04
+    env:
+      CCACHE_DIR: "${{ github.workspace }}/.ccache"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get Ccache
+        uses: actions/cache/restore@v4
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-ibm-velox
+      - name: Setup Gluten
+        run: |
+          git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
+          sed -i 's/ibm/ibm-xxx/gI' ep/build-velox/src/get-velox.sh
+          BRANCH=$(echo ${GITHUB_REF#refs/heads/})
+          # sed -i 's/VELOX_BRANCH=2025.*/VELOX_BRANCH=${BRANCH}/g' ep/build-velox/src/get-velox.sh
+      - name: Build Gluten native libraries
+        run: |
+          docker pull apache/gluten:vcpkg-centos-7
+          docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c "
+            git config --global --add safe.directory /work
+            set -e
+            df -a
+            cd /work
+            git log -n 3
+            cd /work/gluten
+            git log -n 3
+            export CCACHE_DIR=/work/.ccache
+            export CCACHE_SLOPPINESS=file_macro,locale,time_macros
+            mkdir -p /work/.ccache
+            ccache -M 1G
+            ccache -sz
+            source /opt/rh/devtoolset-11/enable
+            source /opt/rh/rh-git227/enable
+            export NUM_THREADS=4
+            ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \
+                           --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON --enable_enhanced_features=ON --velox_home=/work               
+            pushd /work
+            git log -n 3
+            popd
+            ccache -s
+            mkdir -p /work/.m2/repository/org/apache/arrow/
+            cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
+          "
+      - name: "Save ccache"
+        if: always()
+        uses: actions/cache/save@v4
+        id: ccache
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-ibm-velox
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: velox-native-lib-centos-7-${{github.sha}}
+          path: ./gluten/cpp/build/releases/
+          if-no-files-found: error
+      - uses: actions/upload-artifact@v4
+        with:
+          name: arrow-jars-centos-7-${{github.sha}}
+          path: .m2/repository/org/apache/arrow/
+          if-no-files-found: error
+
+  spark-test-spark32:
+    needs: gluten-cpp-build
+    runs-on: ubuntu-22.04
+    container: apache/gluten:centos-8-jdk8
+    steps:
+      - name: Setup Gluten
+        run: |
+          git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: velox-native-lib-centos-7-${{github.sha}}
+          path: ./gluten/cpp/build/releases
+      - name: Download Arrow Jars
+        uses: actions/download-artifact@v4
+        with:
+          name: arrow-jars-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
+      - name: Build package for Spark 3.2
+        run: |
+          cd $GITHUB_WORKSPACE/gluten
+          export SPARK_SCALA_VERSION=2.12
+          yum install -y java-17-openjdk-devel
+          export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+          export PATH=$JAVA_HOME/bin:$PATH
+          java -version
+          $MVN_CMD clean package -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -Phudi -DskipTests
+
+  spark-test-spark34:
+    needs: gluten-cpp-build
+    runs-on: ubuntu-22.04
+    container: apache/gluten:centos-8-jdk8
+    steps:
+      - name: Setup Gluten
+        run: |
+          git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: velox-native-lib-centos-7-${{github.sha}}
+          path: ./gluten/cpp/build/releases
+      - name: Download Arrow Jars
+        uses: actions/download-artifact@v4
+        with:
+          name: arrow-jars-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
+      - name: Prepare spark.test.home for Spark 3.4.4 (other tests)
+        run: |
+          dnf module -y install python39 && \
+          alternatives --set python3 /usr/bin/python3.9 && \
+          pip3 install setuptools==77.0.3 && \
+          pip3 install pyspark==3.4.4 cython && \
+          pip3 install pandas==2.2.3 pyarrow==20.0.0
+      - name: Build and Run unit test for Spark 3.4.4 (other tests)
+        run: |
+          cd $GITHUB_WORKSPACE/gluten
+          export SPARK_SCALA_VERSION=2.12
+          yum install -y java-17-openjdk-devel
+          export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+          export PATH=$JAVA_HOME/bin:$PATH
+          java -version
+          export SPARK_HOME=/opt/shims/spark34/spark_home/
+          ls -l $SPARK_HOME
+          $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Pspark-ut \
+          -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
+          -DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}"
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}-report
+          path: '**/surefire-reports/TEST-*.xml'
+      - name: Upload unit tests log files
+        if: ${{ !success() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}-test-log
+          path: |
+            **/target/*.log
+            **/gluten-ut/**/hs_err_*.log
+            **/gluten-ut/**/core.*
+      - name: Upload golden files
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}-golden-files
+          path: /tmp/tpch-approved-plan/**
+
+  spark-test-spark34-slow:
+    needs: gluten-cpp-build
+    runs-on: ubuntu-22.04
+    container: apache/gluten:centos-8-jdk8
+    steps:
+      - name: Setup Gluten
+        run: |
+          git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: velox-native-lib-centos-7-${{github.sha}}
+          path: ./gluten/cpp/build/releases
+      - name: Download Arrow Jars
+        uses: actions/download-artifact@v4
+        with:
+          name: arrow-jars-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
+      - name: Build and Run unit test for Spark 3.4.4 (slow tests)
+        run: |
+          cd $GITHUB_WORKSPACE/gluten
+          yum install -y java-17-openjdk-devel
+          export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+          export PATH=$JAVA_HOME/bin:$PATH
+          java -version
+          export SPARK_HOME=/opt/shims/spark34/spark_home/
+          ls -l $SPARK_HOME
+          $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg  -Pdelta -Pspark-ut -Phudi \
+          -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
+          -DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}"
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}-report
+          path: '**/surefire-reports/TEST-*.xml'
+      - name: Upload unit tests log files
+        if: ${{ !success() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}-test-log
+          path: |
+            **/target/*.log
+            **/gluten-ut/**/hs_err_*.log
+            **/gluten-ut/**/core.*

.github/workflows/linux-build.yml

@@ -15,21 +15,6 @@
 name: Linux Build using GCC
 
 on:
-  push:
-    branches:
-      - main
-    paths:
-      - velox/**
-      - '!velox/docs/**'
-      - CMakeLists.txt
-      - CMake/**
-      - scripts/setup-ubuntu.sh
-      - scripts/setup-common.sh
-      - scripts/setup-versions.sh
-      - scripts/setup-helper-functions.sh
-      - .github/workflows/linux-build.yml
-      - .github/workflows/linux-build-base.yml
-
   pull_request:
     paths:
       - velox/**

velox/common/base/AsyncSource.h

@@ -121,7 +121,7 @@ class AsyncSource {
         return nullptr;
       }
       if (making_) {
-        promise_ = std::make_unique<ContinuePromise>();
+        promise_ = std::make_unique<ContinuePromise>("AsyncSource::move");
         wait = promise_->getSemiFuture();
       } else {
         if (!make_) {
@@ -178,7 +178,7 @@ class AsyncSource {
     {
       std::lock_guard<std::mutex> l(mutex_);
       if (making_) {
-        promise_ = std::make_unique<ContinuePromise>();
+        promise_ = std::make_unique<ContinuePromise>("AsyncSource::close");
         wait = promise_->getSemiFuture();
       } else if (make_) {
         make_ = nullptr;

velox/common/caching/AsyncDataCache.h

@@ -445,7 +445,7 @@ class CoalescedLoad {
     return state_;
   }
 
-  void cancel() {
+  virtual void cancel() {
     setEndState(State::kCancelled);
   }
 

velox/common/future/VeloxPromise.h

@@ -28,12 +28,15 @@ class VeloxPromise : public folly::Promise<T> {
   VeloxPromise() : folly::Promise<T>() {}
 
   explicit VeloxPromise(const std::string& context)
-      : folly::Promise<T>(), context_(context) {}
+      : folly::Promise<T>(), context_(context) {
+    if (context.empty()) {
+      LOG(WARNING)
+          << "PROMISE: VeloxPromise must be constructed with a context.";
+    }
+  }
 
-  VeloxPromise(
-      folly::futures::detail::EmptyConstruct,
-      const std::string& context) noexcept
-      : folly::Promise<T>(folly::Promise<T>::makeEmpty()), context_(context) {}
+  explicit VeloxPromise(folly::futures::detail::EmptyConstruct) noexcept
+      : folly::Promise<T>(folly::Promise<T>::makeEmpty()) {}
 
   ~VeloxPromise() {
     if (!this->isFulfilled()) {
@@ -52,8 +55,8 @@ class VeloxPromise : public folly::Promise<T> {
     return *this;
   }
 
-  static VeloxPromise makeEmpty(const std::string& context = "") noexcept {
-    return VeloxPromise<T>(folly::futures::detail::EmptyConstruct{}, context);
+  static VeloxPromise makeEmpty() noexcept {
+    return VeloxPromise<T>(folly::futures::detail::EmptyConstruct{});
   }
 
  private:
@@ -72,7 +75,7 @@ using ContinueFuture = folly::SemiFuture<folly::Unit>;
 /// exception throwing and stack unwinding thus performance issue.  See
 /// https://github.com/prestodb/presto/issues/26094 for details.
 static inline std::pair<ContinuePromise, ContinueFuture>
-makeVeloxContinuePromiseContract(const std::string& promiseContext = "") {
+makeVeloxContinuePromiseContract(const std::string& promiseContext) {
   auto p = ContinuePromise(promiseContext);
   auto f = p.getSemiFuture();
   return std::make_pair(std::move(p), std::move(f));

velox/common/memory/SharedArbitrator.cpp

@@ -897,13 +897,16 @@ void SharedArbitrator::growCapacity(ArbitrationOperation& op) {
   RETURN_IF_TRUE(maybeGrowFromSelf(op));
 
   if (!ensureCapacity(op)) {
+    const auto maxCapacity = op.participant()->maxCapacity();
     MEM_POOL_CAP_EXCEEDED(
         fmt::format(
-            "Can't grow {} capacity with {}. This will exceed its max capacity "
+            "Can't grow {} capacity with {}. This will exceed its {} "
             "{}, current capacity {}.",
             op.participant()->name(),
             succinctBytes(op.requestBytes()),
-            succinctBytes(op.participant()->maxCapacity()),
+            capacity_ < maxCapacity ? "arbitrator capacity"
+                                    : "memory pool capacity",
+            succinctBytes(std::min(capacity_, maxCapacity)),
             succinctBytes(op.participant()->capacity())),
         op.participant()->pool());
   }
@@ -1401,7 +1404,6 @@ uint64_t SharedArbitrator::reclaim(
   if (participant->aborted()) {
     removeGlobalArbitrationWaiter(participant->id());
   }
-  freeCapacity(reclaimedBytes);
 
   updateMemoryReclaimStats(
       reclaimedBytes, reclaimTimeNs, localArbitration, stats);
@@ -1413,6 +1415,8 @@ uint64_t SharedArbitrator::reclaim(
                       << " stats " << succinctBytes(stats.reclaimedBytes)
                       << " numNonReclaimableAttempts "
                       << stats.numNonReclaimableAttempts;
+
+  freeCapacity(reclaimedBytes);
   if (reclaimedBytes == 0) {
     FB_LOG_EVERY_MS(WARNING, 1'000) << fmt::format(
         "Nothing reclaimed from memory pool {} with reclaim target {},  memory pool stats:\n{}\n{}",

velox/common/memory/tests/MemoryCapExceededTest.cpp

@@ -69,7 +69,7 @@ TEST_P(MemoryCapExceededTest, singleDriver) {
   // why).
   std::vector<std::string> expectedTexts = {
       "Can't grow ",
-      "capacity with 2.00MB. This will exceed its max capacity 5.00MB, current "
+      "capacity with 2.00MB. This will exceed its memory pool capacity 5.00MB, current "
       "capacity 5.00MB.\n"
       "ARBITRATOR[SHARED CAPACITY[6.00GB] STATS[numRequests 1 numRunning 1 "
       "numSucceded 0 numAborted 0 numFailures 0 numNonReclaimableAttempts 0 "

velox/common/memory/tests/SharedArbitratorTest.cpp

@@ -760,7 +760,7 @@ DEBUG_ONLY_TEST_P(
     folly::EventCount taskPauseWait;
     auto taskPauseWaitKey = taskPauseWait.prepareWait();
 
-    const auto fakeAllocationSize = kMemoryCapacity - (32L << 20);
+    const auto fakeAllocationSize = kMemoryCapacity - (2L << 20);
 
     std::atomic<bool> injectAllocationOnce{true};
     fakeOperatorFactory_->setAllocationCallback([&](Operator* op) {
@@ -1379,6 +1379,7 @@ TEST_P(
           if (e.errorCode() != error_code::kMemCapExceeded.c_str() &&
               e.errorCode() != error_code::kMemAborted.c_str() &&
               e.errorCode() != error_code::kMemAllocError.c_str() &&
+              e.errorCode() != error_code::kMemArbitrationTimeout.c_str() &&
               (e.message() != "Aborted for external error")) {
             std::rethrow_exception(std::current_exception());
           }