From 03ed1c082046f9c09930cb1d282d0c7967d1b9d8 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Wed, 26 Mar 2025 15:34:12 +0300 Subject: [PATCH 01/29] Add scripts for create ubuntu jammy docker file --- .../docker-cbdb-build-containers.yml | 4 +- .../cloudberry/scripts/build-cloudberry.sh | 6 +- .../cloudberry/scripts/cloudberry-utils.sh | 14 +- .../scripts/configure-cloudberry.sh | 19 +- .../scripts/create-cloudberry-demo-cluster.sh | 6 +- .../destroy-cloudberry-demo-cluster.sh | 2 +- .../cloudberry/scripts/test-cloudberry.sh | 2 +- .../cloudberry/scripts/unittest-cloudberry.sh | 4 +- images/docker/cbdb/build/jammy/Dockerfile | 203 ++++++++++++++++ .../cbdb/build/jammy/configs/init_system.sh | 192 +++++++++++++++ images/docker/cbdb/test/jammy/Dockerfile | 119 ++++++++++ .../cbdb/test/jammy/configs/init_system.sh | 221 ++++++++++++++++++ packaging/deb/jammy/debian/changelog | 5 + packaging/deb/jammy/debian/compat | 1 + packaging/deb/jammy/debian/control | 111 +++++++++ packaging/deb/jammy/debian/install | 1 + packaging/deb/jammy/debian/postinst | 12 + packaging/deb/jammy/debian/preinst | 13 ++ packaging/deb/jammy/debian/rules | 49 ++++ packaging/deb/jammy/debian/source/format | 1 + .../deb/jammy/debian/source/local-options | 2 + scripts/build-deb.sh | 144 ++++++++++++ 22 files changed, 1109 insertions(+), 22 deletions(-) create mode 100644 images/docker/cbdb/build/jammy/Dockerfile create mode 100644 images/docker/cbdb/build/jammy/configs/init_system.sh create mode 100644 images/docker/cbdb/test/jammy/Dockerfile create mode 100644 images/docker/cbdb/test/jammy/configs/init_system.sh create mode 100644 packaging/deb/jammy/debian/changelog create mode 100644 packaging/deb/jammy/debian/compat create mode 100644 packaging/deb/jammy/debian/control create mode 100644 packaging/deb/jammy/debian/install create mode 100644 packaging/deb/jammy/debian/postinst create mode 100644 packaging/deb/jammy/debian/preinst create mode 100755 packaging/deb/jammy/debian/rules create mode 100644 packaging/deb/jammy/debian/source/format create mode 100644 packaging/deb/jammy/debian/source/local-options create mode 100644 scripts/build-deb.sh diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 2a9e77b..edbad06 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -76,7 +76,7 @@ jobs: # Matrix strategy to build for both Rocky Linux 8 and 9 strategy: matrix: - platform: ['rocky8', 'rocky9'] + platform: ['rocky8', 'rocky9', 'jammy'] steps: # Checkout repository code with full history @@ -103,6 +103,8 @@ jobs: - 'images/docker/cbdb/build/rocky8/**' rocky9: - 'images/docker/cbdb/build/rocky9/**' + jammy: + - 'images/docker/cbdb/build/jammy/**' # Set up QEMU for multi-architecture support # This allows building ARM64 images on AMD64 infrastructure and vice versa diff --git a/build_automation/cloudberry/scripts/build-cloudberry.sh b/build_automation/cloudberry/scripts/build-cloudberry.sh index db04f1b..4ce4752 100755 --- a/build_automation/cloudberry/scripts/build-cloudberry.sh +++ b/build_automation/cloudberry/scripts/build-cloudberry.sh @@ -45,7 +45,7 @@ # Prerequisites: # - configure-cloudberry.sh must be run first # - Required build dependencies must be installed -# - /usr/local/cloudberry-db/lib must exist and be writable +# - ${BUILD_DESTINATION}/lib must exist and be writable # # Exit Codes: # 0 - Build and installation completed successfully @@ -67,11 +67,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" BUILD_LOG="${LOG_DIR}/build.log" # Initialize environment -init_environment "Cloudberry Build Script" "${BUILD_LOG}" +init_environment "Cloudberry Build Script" "${BUILD_LOG}" "${BUILD_DESTINATION}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Build process diff --git a/build_automation/cloudberry/scripts/cloudberry-utils.sh b/build_automation/cloudberry/scripts/cloudberry-utils.sh index a0cc1b8..a3c4b61 100755 --- a/build_automation/cloudberry/scripts/cloudberry-utils.sh +++ b/build_automation/cloudberry/scripts/cloudberry-utils.sh @@ -30,11 +30,12 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Functions: -# init_environment "Script Name" "Log File" +# init_environment "Script Name" "Log File" "Build Destination" # - Initialize logging and verify environment # - Parameters: # * script_name: Name of the calling script # * log_file: Path to log file +# * build_destination: Path to Cloudberry destination, by default is /usr/local/cloudberry-db # - Returns: 0 on success, 1 on failure # # execute_cmd command [args...] @@ -70,7 +71,7 @@ # # Example: # source ./cloudberry-utils.sh -# init_environment "My Script" "${LOG_FILE}" +# init_environment "My Script" "${LOG_FILE}" "${BUILD_DESTINATION}" # execute_cmd make clean # log_section "Build Process" # execute_cmd make -j$(nproc) @@ -79,10 +80,18 @@ # # -------------------------------------------------------------------- +DEFAULT_BUILD_DESTINATION=/usr/local/cloudberry-db + # Initialize logging and environment init_environment() { local script_name=$1 local log_file=$2 + local build_destination=$3 + + if [ -z "$build_destination" ]; then + build_destination=${DEFAULT_BUILD_DESTINATION} + fi + export BUILD_DESTINATION=$build_destination echo "=== Initializing environment for ${script_name} ===" echo "${script_name} executed at $(date)" | tee -a "${log_file}" @@ -91,6 +100,7 @@ init_environment() { echo "Working directory: $(pwd)" | tee -a "${log_file}" echo "Source directory: ${SRC_DIR}" | tee -a "${log_file}" echo "Log directory: ${LOG_DIR}" | tee -a "${log_file}" + echo "Build destination: ${BUILD_DESTINATION}" | tee -a "${log_file}" if [ -z "${SRC_DIR:-}" ]; then echo "Error: SRC_DIR environment variable is not set" | tee -a "${log_file}" diff --git a/build_automation/cloudberry/scripts/configure-cloudberry.sh b/build_automation/cloudberry/scripts/configure-cloudberry.sh index 587ba10..6ac287b 100755 --- a/build_automation/cloudberry/scripts/configure-cloudberry.sh +++ b/build_automation/cloudberry/scripts/configure-cloudberry.sh @@ -23,7 +23,7 @@ # Description: Configures Apache Cloudberry build environment and runs # ./configure with optimized settings. Performs the # following: -# 1. Prepares /usr/local/cloudberry-db directory +# 1. Prepares ${BUILD_DESTINATION} directory # 2. Sets up library dependencies # 3. Configures build with required features enabled # @@ -96,22 +96,23 @@ export LOG_DIR="${SRC_DIR}/build-logs" CONFIGURE_LOG="${LOG_DIR}/configure.log" # Initialize environment -init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" +init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" "${BUILD_DESTINATION}" # Initial setup log_section "Initial Setup" -execute_cmd sudo rm -rf /usr/local/cloudberry-db || exit 2 +execute_cmd sudo rm -rf ${BUILD_DESTINATION}/* || exit 2 execute_cmd sudo chmod a+w /usr/local || exit 2 -execute_cmd mkdir -p /usr/local/cloudberry-db/lib || exit 2 +execute_cmd sudo mkdir -p ${BUILD_DESTINATION}/lib || exit 2 +execute_cmd sudo chown -R gpadmin:gpadmin ${BUILD_DESTINATION} || exit 2 execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ /usr/local/xerces-c/lib/libxerces-c-3.3.so \ - /usr/local/cloudberry-db/lib || exit 3 -execute_cmd sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db || exit 2 + ${BUILD_DESTINATION}/lib || exit 3 +execute_cmd sudo chown -R gpadmin:gpadmin ${BUILD_DESTINATION} || exit 2 log_section_end "Initial Setup" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Add debug options if ENABLE_DEBUG is set to "true" @@ -126,7 +127,7 @@ fi # Configure build log_section "Configure" -execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ +execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --disable-external-fts \ --enable-gpcloud \ --enable-ic-proxy \ @@ -150,7 +151,7 @@ execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ --with-openssl \ --with-uuid=e2fs \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=/usr/local/cloudberry-db/lib || exit 4 + --with-libraries=${BUILD_DESTINATION}/lib || exit 4 log_section_end "Configure" # Capture version information diff --git a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh index 396780b..88d51ef 100755 --- a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh @@ -35,7 +35,7 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Prerequisites: -# - Apache Cloudberry must be installed (/usr/local/cloudberry-db) +# - Apache Cloudberry must be installed (${BUILD_DESTINATION}) # - SSH must be configured for passwordless access to localhost # - User must have permissions to create cluster directories # - PostgreSQL client tools (psql) must be available @@ -75,11 +75,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/cluster.log" # Initialize environment -init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" +init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" "${BUILD_DESTINATION}" # Setup environment log_section "Environment Setup" -source /usr/local/cloudberry-db/greenplum_path.sh || exit 1 +source ${BUILD_DESTINATION}/greenplum_path.sh || exit 1 log_section_end "Environment Setup" # Verify SSH access diff --git a/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh index 3d4ce24..ae2bdc0 100755 --- a/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh @@ -71,7 +71,7 @@ export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/destroy-cluster.log" # Initialize environment -init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" +init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" "${BUILD_DESTINATION}" # Source Cloudberry environment log_section "Environment Setup" diff --git a/build_automation/cloudberry/scripts/test-cloudberry.sh b/build_automation/cloudberry/scripts/test-cloudberry.sh index 411f16c..e3ff951 100755 --- a/build_automation/cloudberry/scripts/test-cloudberry.sh +++ b/build_automation/cloudberry/scripts/test-cloudberry.sh @@ -60,7 +60,7 @@ export LOG_DIR="build-logs" TEST_LOG="${LOG_DIR}/test.log" # Initialize environment -init_environment "Cloudberry Test Script" "${TEST_LOG}" +init_environment "Cloudberry Test Script" "${TEST_LOG}" "${BUILD_DESTINATION}" # Source Cloudberry environment log_section "Environment Setup" diff --git a/build_automation/cloudberry/scripts/unittest-cloudberry.sh b/build_automation/cloudberry/scripts/unittest-cloudberry.sh index f7bc120..7e2283e 100755 --- a/build_automation/cloudberry/scripts/unittest-cloudberry.sh +++ b/build_automation/cloudberry/scripts/unittest-cloudberry.sh @@ -52,11 +52,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" UNITTEST_LOG="${LOG_DIR}/unittest.log" # Initialize environment -init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" +init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" "${BUILD_DESTINATION}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Unittest process diff --git a/images/docker/cbdb/build/jammy/Dockerfile b/images/docker/cbdb/build/jammy/Dockerfile new file mode 100644 index 0000000..268ef8d --- /dev/null +++ b/images/docker/cbdb/build/jammy/Dockerfile @@ -0,0 +1,203 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Cloudberry Database Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as +# a base environment for evaluating the Cloudberry Database. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Cloudberry +# Database DEB can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +FROM ubuntu:jammy + +ARG accessKeyId +ARG secretAccessKey +ARG bucketName + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="Europe/London" + + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive + +RUN stat -fc %T /sys/fs/cgroup/ + +RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo $TIMEZONE_VAR > /etc/timezone + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- +RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ + build-essential \ + libssl-dev \ + gnupg \ + openssl \ + debhelper \ + libfakeroot \ + debootstrap \ + devscripts \ + make \ + equivs \ + bison \ + ca-certificates-java \ + ca-certificates \ + cmake \ + curl \ + cgroup-tools \ + flex \ + gcc-11 \ + g++-11 \ + g++-11-multilib \ + git \ + krb5-multidev \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libpstreams-dev \ + libxerces-c-dev \ + libsystemd-dev \ + libevent-dev \ + libkrb5-dev \ + libldap2-dev \ + libperl-dev \ + libreadline-dev \ + libssl-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + libaprutil1-dev \ + libpam0g-dev \ + libpam0g \ + libcgroup1 \ + libyaml-0-2 \ + libldap-2.5-0 \ + libssl3 \ + ninja-build \ + python-setuptools \ + python3-setuptools \ + quilt \ + unzip \ + wget \ + zlib1g-dev \ + libuv1-dev \ + libgpgme-dev \ + libgpgme11 \ + python2.7 \ + python2.7-dev \ + pkg-config \ + python3.10 \ + python3.10-dev \ + python3-distutils + +RUN apt-get install -y locales \ +&& locale-gen "en_US.UTF-8" \ +&& update-locale LC_ALL="en_US.UTF-8" + +RUN cd \ +&& XERCES_LATEST_RELEASE=3.3.0 \ +&& wget -nv "https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" \ +&& echo "$(curl -sL https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz.sha256)" | sha256sum -c - \ +&& tar xf "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz"; rm "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" \ +&& cd xerces-c-${XERCES_LATEST_RELEASE} \ +&& ./configure --prefix=/usr/local/xerces-c \ +&& make -j \ +&& make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} \ +&& rm -rf ~/xerces-c* + +RUN cd \ +&& GO_VERSION="go1.23.4" \ +&& ARCH=$(uname -m) \ +&& if [ "${ARCH}" = "aarch64" ]; then \ + GO_ARCH="arm64" && \ + GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + elif [ "${ARCH}" = "x86_64" ]; then \ + GO_ARCH="amd64" && \ + GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + else \ + echo "Unsupported architecture: ${ARCH}" && exit 1; \ + fi \ +&& GO_URL="https://go.dev/dl/${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ +&& wget -nv "${GO_URL}" \ +&& echo "${GO_SHA256} ${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | sha256sum -c - \ +&& tar xf "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ +&& mv go "/usr/local/${GO_VERSION}" \ +&& ln -s "/usr/local/${GO_VERSION}" /usr/local/go \ +&& rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ +&& echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null + + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- + +RUN ln -s /usr/bin/python2.7 /usr/bin/python + +COPY ./configs/* /tmp/ + +RUN echo ${TIMEZONE_VAR} > /etc/timezone && \ + chmod 777 /tmp/init_system.sh && \ + groupadd gpadmin && \ + useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin && \ + echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers && \ + locale-gen "en_US.UTF-8" && \ + update-locale LC_ALL="en_US.UTF-8" + +USER gpadmin +WORKDIR /home/gpadmin + +RUN sudo DEBIAN_FRONTEND=noninteractive apt install -y libhyperic-sigar-java libaprutil1-dev libuv1-dev + +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/images/docker/cbdb/build/jammy/configs/init_system.sh b/images/docker/cbdb/build/jammy/configs/init_system.sh new file mode 100644 index 0000000..d8c4a00 --- /dev/null +++ b/images/docker/cbdb/build/jammy/configs/init_system.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +## Container Initialization Script +# -------------------------------------------------------------------- +## This script sets up the environment inside the Docker container for +## the Apache Cloudberry Build Environment. It performs the following +## tasks: +## +## 1. Verifies that the container is running with the expected hostname. +## 2. Starts the SSH daemon to allow SSH access to the container. +## 3. Configures passwordless SSH access for the 'gpadmin' user. +## 4. Displays a welcome banner and system information. +## 5. Starts an interactive bash shell. +## +## This script is intended to be used as an entrypoint or initialization +## script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +cat <<-EOF +Welcome to the Apache Cloudberry Build Environment! + +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/images/docker/cbdb/test/jammy/Dockerfile b/images/docker/cbdb/test/jammy/Dockerfile new file mode 100644 index 0000000..1255e21 --- /dev/null +++ b/images/docker/cbdb/test/jammy/Dockerfile @@ -0,0 +1,119 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Cloudberry Database Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as +# a base environment for evaluating the Cloudberry Database. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Cloudberry +# Database DEB can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +FROM ubuntu:jammy + +ARG accessKeyId +ARG secretAccessKey +ARG bucketName + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="Europe/London" + + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive + +RUN stat -fc %T /sys/fs/cgroup/ + +RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo $TIMEZONE_VAR > /etc/timezone + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- +RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ + file \ + gdb \ + glibc-locale-source \ + make \ + openssh \ + openssh-clients \ + openssh-server \ + procps-ng \ + sudo \ + which + +RUN apt-get install -y locales \ +&& locale-gen "en_US.UTF-8" \ +&& update-locale LC_ALL="en_US.UTF-8" + + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- + +RUN ln -s /usr/bin/python2.7 /usr/bin/python + +COPY ./configs/* /tmp/ + +RUN echo ${TIMEZONE_VAR} > /etc/timezone && \ + chmod 777 /tmp/init_system.sh && \ + groupadd gpadmin && \ + useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin && \ + echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers && \ + locale-gen "en_US.UTF-8" && \ + update-locale LC_ALL="en_US.UTF-8" + +USER gpadmin +WORKDIR /home/gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/images/docker/cbdb/test/jammy/configs/init_system.sh b/images/docker/cbdb/test/jammy/configs/init_system.sh new file mode 100644 index 0000000..749d789 --- /dev/null +++ b/images/docker/cbdb/test/jammy/configs/init_system.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# Container Initialization Script +# -------------------------------------------------------------------- +# This script sets up the environment inside the Docker container for +# the Apache Cloudberry Build Environment. It performs the following +# tasks: +# +# 1. Verifies that the container is running with the expected hostname. +# 2. Starts the SSH daemon to allow SSH access to the container. +# 3. Configures passwordless SSH access for the 'gpadmin' user. +# 4. Sets up the necessary directories and configuration files for +# Apache Cloudberry. +# 5. Displays a welcome banner and system information. +# 6. Starts an interactive bash shell. +# +# This script is intended to be used as an entrypoint or initialization +# script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# -------------------------------------------------------------------- +# Cloudberry Data Directories Setup +# -------------------------------------------------------------------- +# The script sets up the necessary directories for Apache Cloudberry, +# including directories for the coordinator, standby coordinator, primary +# segments, and mirror segments. It also sets up the configuration files +# required for initializing the database. +# -------------------------------------------------------------------- +sudo rm -rf /data1/* +sudo mkdir -p /data1/coordinator /data1/standby_coordinator /data1/primary /data1/mirror +sudo chown -R gpadmin.gpadmin /data1 + +# Copy the gpinitsystem configuration file to the home directory +cp /tmp/gpinitsystem.conf /home/gpadmin + +# Set up the hostfile for cluster initialization +echo $(hostname) > /home/gpadmin/hostfile_gpinitsystem + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# - Cloudberry version (if installed) +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +# Check if Apache Cloudberry is installed and display its version +if dpkg -l apache-cloudberry-db-incubating > /dev/null 2>&1; then + CBDB_VERSION=$(/usr/local/cbdb/bin/postgres --gp-version) +else + CBDB_VERSION="Not installed" +fi + +cat <<-EOF +Welcome to the Apache Cloudberry Test Environment! + +Cloudberry version .. : $CBDB_VERSION +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/packaging/deb/jammy/debian/changelog b/packaging/deb/jammy/debian/changelog new file mode 100644 index 0000000..211d271 --- /dev/null +++ b/packaging/deb/jammy/debian/changelog @@ -0,0 +1,5 @@ +apache-cloudberry-db-incubating (2.0.0) jammy; urgency=medium + + * Initial release. + + -- Cloudberry Team Wed, 26 Mar 2025 11:10:44 +0000 diff --git a/packaging/deb/jammy/debian/compat b/packaging/deb/jammy/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/packaging/deb/jammy/debian/compat @@ -0,0 +1 @@ +9 diff --git a/packaging/deb/jammy/debian/control b/packaging/deb/jammy/debian/control new file mode 100644 index 0000000..a8ae4e7 --- /dev/null +++ b/packaging/deb/jammy/debian/control @@ -0,0 +1,111 @@ +Source: apache-cloudberry-db-incubating +Maintainer: open-gpdb dev team https://github.com/open-gpdb/gpdb +Section: database +Build-Depends: debhelper (>= 9), + bison, + ca-certificates-java, + ca-certificates, + cmake, + curl, + cgroup-tools, + flex, + gcc-11, + g++-11, + g++-11-multilib, + git, + krb5-multidev, + libapr1-dev, + libbz2-dev, + libcurl4-gnutls-dev, + libevent-dev, + libkrb5-dev, + libldap2-dev, + libperl-dev, + libreadline6-dev, + libssl-dev, + libxml2-dev, + libyaml-dev, + libzstd-dev, + libaprutil1-dev, + libpam0g-dev, + libpam0g, + libcgroup1, + libyaml-0-2, + libldap-2.5-0, + libssl3, + ninja-build, + python2.7-dev, + python-setuptools, + quilt, + unzip, + wget, + zlib1g-dev, + libuv1-dev + +Package: apache-cloudberry-db-incubating +Architecture: amd64 +Depends: curl, + cgroup-tools, + iputils-ping, + krb5-multidev, + less, + libapr1, + libbz2-1.0, + libcurl4, + libcurl3-gnutls, + libevent-2.1-7, + libreadline8, + libxml2, + libyaml-0-2, + libldap-2.5-0, + libzstd1, + libcgroup1, + libssl3, + libpam0g, + locales, + net-tools, + openssh-client, + openssh-server, + openssl, + python2.7, + python2.7-dev, + python-lockfile, + python-paramiko, + python-psutil, + python-yaml, + python-pip, + python-behave, + python-epydoc, + python-ldap, + python-mock, + python-six, + python-parse, + python-is-python2, + rsync, + wget, + zlib1g, + libuv1 +Description: Apache Cloudberry (incubating) is an advanced, open-source, massively + parallel processing (MPP) data warehouse developed from PostgreSQL and + Greenplum. It is designed for high-performance analytics on + large-scale data sets, offering powerful analytical capabilities and + enhanced security features. + Key Features: + - Massively parallel processing for optimized performance + - Advanced analytics for complex data processing + - Integration with ETL and BI tools + - Compatibility with multiple data sources and formats + - Enhanced security features + Apache Cloudberry supports both batch processing and real-time data + warehousing, making it a versatile solution for modern data + environments. + Apache Cloudberry (incubating) is an effort undergoing incubation at + the Apache Software Foundation (ASF), sponsored by the Apache + Incubator PMC. + Incubation is required of all newly accepted projects until a further + review indicates that the infrastructure, communications, and decision + making process have stabilized in a manner consistent with other + successful ASF projects. + While incubation status is not necessarily a reflection of the + completeness or stability of the code, it does indicate that the + project has yet to be fully endorsed by the ASF. diff --git a/packaging/deb/jammy/debian/install b/packaging/deb/jammy/debian/install new file mode 100644 index 0000000..4ce7955 --- /dev/null +++ b/packaging/deb/jammy/debian/install @@ -0,0 +1 @@ +debian/build/* /usr/local/cloudberry-db diff --git a/packaging/deb/jammy/debian/postinst b/packaging/deb/jammy/debian/postinst new file mode 100644 index 0000000..9a4318e --- /dev/null +++ b/packaging/deb/jammy/debian/postinst @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin +GPHOME=/usr/local/cloudberry-db + +if [ "$1" = configure ]; then + + chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} + +fi diff --git a/packaging/deb/jammy/debian/preinst b/packaging/deb/jammy/debian/preinst new file mode 100644 index 0000000..29d6211 --- /dev/null +++ b/packaging/deb/jammy/debian/preinst @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin + +case $1 in + install|upgrade) + id "${GPADMIN}" >> /dev/null 2>&1 || \ + useradd -r -m -d /home/${GPADMIN} -s /bin/bash ${GPADMIN} + ;; +esac + diff --git a/packaging/deb/jammy/debian/rules b/packaging/deb/jammy/debian/rules new file mode 100755 index 0000000..fa5f5a4 --- /dev/null +++ b/packaging/deb/jammy/debian/rules @@ -0,0 +1,49 @@ +#!/usr/bin/make -f + +DH_VERBOSE = 1 +DPKG_EXPORT_BUILDFLAGS = 1 + +GPDB_BIN_PATH := /usr/local/cloudberry-db +DEBIAN_DESTINATION := $(shell pwd)/debian/build + +# assumes that CWD is root of gpdb source +GPDB_PKG_VERSION := $(GPDB_PKG_VERSION) +PACKAGE_GPDB := $(shell cat debian/control | egrep "^Package: " | cut -d " " -f 2) +PATH := ${DEBIAN_DESTINATION}/bin:${PATH} + +.PHONY: gpinstall + +include /usr/share/dpkg/default.mk + +%: + dh $@ --parallel + +gpinstall: + make install + +override_dh_auto_install: gpinstall + # the staging directory for creating a debian is NOT the right GPHOME. + # change GPHOME to point to the post-install target install directory. + sed -i "s#GPHOME=.*#GPHOME=${GPDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh + +override_dh_auto_build: + echo "Skipping build" + +override_dh_auto_clean: + echo "Skipping clean" + +override_dh_auto_configure: + echo "Skipping configure" + +override_dh_auto_test: + echo "Skipping auto test" + +override_dh_gencontrol: + echo "using version ${GPDB_PKG_VERSION} for binary GPDB" + dh_gencontrol -- -v${GPDB_PKG_VERSION} -p${PACKAGE_GPDB} + +override_dh_shlibdeps: + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/libfakeroot:${DEBIAN_DESTINATION}/lib dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info + +clean_dev_local: + rm -rf ${DEBIAN_DESTINATION} diff --git a/packaging/deb/jammy/debian/source/format b/packaging/deb/jammy/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/packaging/deb/jammy/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/packaging/deb/jammy/debian/source/local-options b/packaging/deb/jammy/debian/source/local-options new file mode 100644 index 0000000..00131ee --- /dev/null +++ b/packaging/deb/jammy/debian/source/local-options @@ -0,0 +1,2 @@ +#abort-on-upstream-changes +#unapply-patches diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh new file mode 100644 index 0000000..9c32247 --- /dev/null +++ b/scripts/build-deb.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# Script Name: build-deb.sh +# +# Description: +# This script automates the process of building an DEB package using a specified +# version number. It ensures that the necessary tools are installed +# and that the control file exists before attempting to build the DEB. The script +# also includes error handling to provide meaningful feedback in case of failure. +# +# Usage: +# ./build-deb.sh [-v ] [-h] [--dry-run] +# +# Options: +# -v, --version : Specify the version (required) +# -h, --help : Display this help and exit +# -n, --dry-run : Show what would be done, without making any changes +# +# Example: +# ./build-deb.sh -v 1.5.5 # Build with version 1.5.5 +# +# Prerequisites: +# - The dpkg-buildpackage package must be installed (provides the dpkg-buildpackage command). +# - The control file must exist at debian/control. +# +# Error Handling: +# The script includes checks to ensure: +# - The version option (-v or --version) is provided. +# - The necessary commands are available. +# - The control file exists at the specified location. +# If any of these checks fail, the script exits with an appropriate error message. + +# Enable strict mode for better error handling +set -euo pipefail + +# Default values +VERSION="" +RELEASE="1" +DEBUG_BUILD=false + +# Function to display usage information +usage() { + echo "Usage: $0 [-v ] [-h] [--dry-run]" + echo " -v, --version : Specify the version (optional)" + echo " -h, --help : Display this help and exit" + echo " -n, --dry-run : Show what would be done, without making any changes" + exit 1 +} + +# Function to check if required commands are available +check_commands() { + local cmds=("dpkg-buildpackage") + for cmd in "${cmds[@]}"; do + if ! command -v "$cmd" &> /dev/null; then + echo "Error: Required command '$cmd' not found. Please install it before running the script." + exit 1 + fi + done +} + +function print_changelog() { +cat < $(date +'%a, %d %b %Y %H:%M:%S %z') +EOF +} + +# Parse options +while [[ "$#" -gt 0 ]]; do + case $1 in + -v|--version) + VERSION="$2" + shift 2 + ;; + -h|--help) + usage + ;; + -n|--dry-run) + DRY_RUN=true + shift + ;; + *) + echo "Unknown option: ($1)" + shift + ;; + esac +done + +export GPDB_FULL_VERSION=$VERSION + +# Set version if not provided +if [ -z "${VERSION}" ]; then + export GPDB_FULL_VERSION=$(./getversion | cut -d'-' -f 1 | cut -d'+' -f 1) +fi + +if [ -z ${BUILD_NUMBER+x} ]; then + export BUILD_NUMBER=1 +fi + +if [ -z ${BUILD_USER+x} ]; then + export BUILD_USER=github +fi + +export GPDB_PKG_VERSION=${GPDB_FULL_VERSION}-${BUILD_NUMBER}-$(git --git-dir=.git rev-list HEAD --count).$(git --git-dir=.git rev-parse --short HEAD) + +# Check if required commands are available +check_commands + +# Define the control file path +CONTROL_FILE=debian/control + +# Check if the spec file exists +if [ ! -f "$CONTROL_FILE" ]; then + echo "Error: Control file not found at $CONTROL_FILE." + exit 1 +fi + +# Build the rpmbuild command based on options +DEBBUILD_CMD="dpkg-buildpackage -us -uc" + +# Dry-run mode +if [ "${DRY_RUN:-false}" = true ]; then + echo "Dry-run mode: This is what would be done:" + print_changelog + echo "" + echo "$DEBBUILD_CMD" + exit 0 +fi + +# Run debbuild with the provided options +echo "Building DEB with Version $GPDB_FULL_VERSION ..." + +print_changelog > debian/changelog + +if ! eval "$DEBBUILD_CMD"; then + echo "Error: deb build failed." + exit 1 +fi + +# Print completion message +echo "DEB build completed successfully with package $GPDB_PKG_VERSION" From b09962e66e9a9e6524378d84113d7c4ed6e16e3e Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Wed, 26 Mar 2025 22:28:30 +0300 Subject: [PATCH 02/29] Remove internal yandex dependencies --- packaging/deb/jammy/debian/control | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packaging/deb/jammy/debian/control b/packaging/deb/jammy/debian/control index a8ae4e7..bdc545c 100644 --- a/packaging/deb/jammy/debian/control +++ b/packaging/deb/jammy/debian/control @@ -69,18 +69,8 @@ Depends: curl, openssl, python2.7, python2.7-dev, - python-lockfile, - python-paramiko, - python-psutil, - python-yaml, python-pip, - python-behave, - python-epydoc, - python-ldap, - python-mock, python-six, - python-parse, - python-is-python2, rsync, wget, zlib1g, From 779dafdff72ab0af8eac4af494398b8442d8b651 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Wed, 26 Mar 2025 22:29:43 +0300 Subject: [PATCH 03/29] Deb version always should start with a number --- scripts/build-deb.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh index 9c32247..2896668 100644 --- a/scripts/build-deb.sh +++ b/scripts/build-deb.sh @@ -96,6 +96,10 @@ if [ -z "${VERSION}" ]; then export GPDB_FULL_VERSION=$(./getversion | cut -d'-' -f 1 | cut -d'+' -f 1) fi +if [[ ! $GPDB_FULL_VERSION =~ ^[0-9] ]]; then + export GPDB_FULL_VERSION="0.$GPDB_FULL_VERSION" +fi + if [ -z ${BUILD_NUMBER+x} ]; then export BUILD_NUMBER=1 fi From f2ab2ed6f02db725b1f568ba8c79a67919b198c5 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Wed, 26 Mar 2025 22:51:06 +0300 Subject: [PATCH 04/29] Install sudo - need for various scripts --- images/docker/cbdb/build/jammy/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/docker/cbdb/build/jammy/Dockerfile b/images/docker/cbdb/build/jammy/Dockerfile index 268ef8d..f25e8f7 100644 --- a/images/docker/cbdb/build/jammy/Dockerfile +++ b/images/docker/cbdb/build/jammy/Dockerfile @@ -140,7 +140,8 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install pkg-config \ python3.10 \ python3.10-dev \ - python3-distutils + python3-distutils \ + sudo RUN apt-get install -y locales \ && locale-gen "en_US.UTF-8" \ From 212980c2a054b80b95c2cb8e2b641ea72a4fbcb7 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Thu, 27 Mar 2025 17:48:20 +0300 Subject: [PATCH 05/29] Resolve review notes --- .../docker-cbdb-build-containers.yml | 7 ++- .../workflows/docker-cbdb-test-containers.yml | 7 ++- .../scripts/configure-cloudberry.sh | 1 - .../build/{jammy => ubuntu22.04}/Dockerfile | 63 +++++++++---------- .../configs/init_system.sh | 0 .../test/{jammy => ubuntu22.04}/Dockerfile | 14 ++--- .../configs/init_system.sh | 0 .../{jammy => ubuntu22.04}/debian/changelog | 0 .../deb/{jammy => ubuntu22.04}/debian/compat | 0 .../deb/{jammy => ubuntu22.04}/debian/control | 2 +- .../deb/{jammy => ubuntu22.04}/debian/install | 0 .../{jammy => ubuntu22.04}/debian/postinst | 0 .../deb/{jammy => ubuntu22.04}/debian/preinst | 0 .../deb/{jammy => ubuntu22.04}/debian/rules | 14 ++--- .../debian/source/format | 0 .../debian/source/local-options | 0 scripts/build-deb.sh | 16 ++--- 17 files changed, 63 insertions(+), 61 deletions(-) rename images/docker/cbdb/build/{jammy => ubuntu22.04}/Dockerfile (77%) rename images/docker/cbdb/build/{jammy => ubuntu22.04}/configs/init_system.sh (100%) rename images/docker/cbdb/test/{jammy => ubuntu22.04}/Dockerfile (91%) rename images/docker/cbdb/test/{jammy => ubuntu22.04}/configs/init_system.sh (100%) rename packaging/deb/{jammy => ubuntu22.04}/debian/changelog (100%) rename packaging/deb/{jammy => ubuntu22.04}/debian/compat (100%) rename packaging/deb/{jammy => ubuntu22.04}/debian/control (97%) rename packaging/deb/{jammy => ubuntu22.04}/debian/install (100%) rename packaging/deb/{jammy => ubuntu22.04}/debian/postinst (100%) rename packaging/deb/{jammy => ubuntu22.04}/debian/preinst (100%) rename packaging/deb/{jammy => ubuntu22.04}/debian/rules (71%) rename packaging/deb/{jammy => ubuntu22.04}/debian/source/format (100%) rename packaging/deb/{jammy => ubuntu22.04}/debian/source/local-options (100%) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index edbad06..214cc68 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -60,6 +60,7 @@ on: paths: - 'images/docker/cbdb/build/rocky8/**' - 'images/docker/cbdb/build/rocky9/**' + - 'images/docker/cbdb/build/ubuntu22.04/**' workflow_dispatch: # Manual trigger # Prevent multiple workflow runs from interfering with each other @@ -76,7 +77,7 @@ jobs: # Matrix strategy to build for both Rocky Linux 8 and 9 strategy: matrix: - platform: ['rocky8', 'rocky9', 'jammy'] + platform: ['rocky8', 'rocky9', 'ubuntu22.04'] steps: # Checkout repository code with full history @@ -103,8 +104,8 @@ jobs: - 'images/docker/cbdb/build/rocky8/**' rocky9: - 'images/docker/cbdb/build/rocky9/**' - jammy: - - 'images/docker/cbdb/build/jammy/**' + ubuntu22.04: + - 'images/docker/cbdb/build/ubuntu22.04/**' # Set up QEMU for multi-architecture support # This allows building ARM64 images on AMD64 infrastructure and vice versa diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 5b2ab15..37422f0 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -49,6 +49,7 @@ on: paths: - 'images/docker/cbdb/test/rocky8/**' - 'images/docker/cbdb/test/rocky9/**' + - 'images/docker/cbdb/test/ubuntu22.04/**' workflow_dispatch: # Manual trigger # Prevent multiple workflow runs from interfering with each other @@ -62,8 +63,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Build for both Rocky Linux 8 and 9 - platform: ['rocky8', 'rocky9'] + # Build for both Rocky Linux 8 and 9, ubuntu 22.04 + platform: ['rocky8', 'rocky9', 'ubuntu22.04'] steps: # Checkout repository code @@ -87,6 +88,8 @@ jobs: - 'images/docker/cbdb/test/rocky8/**' rocky9: - 'images/docker/cbdb/test/rocky9/**' + ubuntu22.04: + - 'images/docker/cbdb/test/ubuntu22.04/**' # Skip if no changes for current platform - name: Skip if not relevant diff --git a/build_automation/cloudberry/scripts/configure-cloudberry.sh b/build_automation/cloudberry/scripts/configure-cloudberry.sh index 6ac287b..14dec02 100755 --- a/build_automation/cloudberry/scripts/configure-cloudberry.sh +++ b/build_automation/cloudberry/scripts/configure-cloudberry.sh @@ -103,7 +103,6 @@ log_section "Initial Setup" execute_cmd sudo rm -rf ${BUILD_DESTINATION}/* || exit 2 execute_cmd sudo chmod a+w /usr/local || exit 2 execute_cmd sudo mkdir -p ${BUILD_DESTINATION}/lib || exit 2 -execute_cmd sudo chown -R gpadmin:gpadmin ${BUILD_DESTINATION} || exit 2 execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ /usr/local/xerces-c/lib/libxerces-c-3.3.so \ ${BUILD_DESTINATION}/lib || exit 3 diff --git a/images/docker/cbdb/build/jammy/Dockerfile b/images/docker/cbdb/build/ubuntu22.04/Dockerfile similarity index 77% rename from images/docker/cbdb/build/jammy/Dockerfile rename to images/docker/cbdb/build/ubuntu22.04/Dockerfile index f25e8f7..d584ac0 100644 --- a/images/docker/cbdb/build/jammy/Dockerfile +++ b/images/docker/cbdb/build/ubuntu22.04/Dockerfile @@ -32,13 +32,13 @@ # project has yet to be fully endorsed by the ASF. # # -------------------------------------------------------------------- -# Dockerfile for Cloudberry Database Base Environment +# Dockerfile for Apache Cloudberry Base Environment # -------------------------------------------------------------------- # This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as -# a base environment for evaluating the Cloudberry Database. It installs +# a base environment for evaluating the Apache Cloudberry. It installs # necessary system utilities, configures the environment for SSH access, -# and sets up a 'gpadmin' user with sudo privileges. The Cloudberry -# Database DEB can be installed into this container for testing and +# and sets up a 'gpadmin' user with sudo privileges. The Apache Cloudberry +# DEB can be installed into this container for testing and # functional verification. # # Key Features: @@ -143,25 +143,24 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install python3-distutils \ sudo -RUN apt-get install -y locales \ -&& locale-gen "en_US.UTF-8" \ -&& update-locale LC_ALL="en_US.UTF-8" - -RUN cd \ -&& XERCES_LATEST_RELEASE=3.3.0 \ -&& wget -nv "https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" \ -&& echo "$(curl -sL https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz.sha256)" | sha256sum -c - \ -&& tar xf "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz"; rm "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" \ -&& cd xerces-c-${XERCES_LATEST_RELEASE} \ -&& ./configure --prefix=/usr/local/xerces-c \ -&& make -j \ -&& make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} \ -&& rm -rf ~/xerces-c* - -RUN cd \ -&& GO_VERSION="go1.23.4" \ -&& ARCH=$(uname -m) \ -&& if [ "${ARCH}" = "aarch64" ]; then \ +RUN apt-get install -y locales && \ + locale-gen "en_US.UTF-8" && \ + update-locale LC_ALL="en_US.UTF-8" + +RUN cd && \ + XERCES_LATEST_RELEASE=3.3.0 && \ + wget -nv "https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" && \ + echo "$(curl -sL https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz.sha256)" | sha256sum -c - && \ + tar xf "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz"; rm "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" && \ + cd xerces-c-${XERCES_LATEST_RELEASE} && \ + ./configure --prefix=/usr/local/xerces-c && \ + make -j$(nproc) && \ + make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} && \ + rm -rf ~/xerces-c* + +RUN cd && GO_VERSION="go1.23.4" && \ + ARCH=$(uname -m) && \ + if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ elif [ "${ARCH}" = "x86_64" ]; then \ @@ -169,15 +168,15 @@ RUN cd \ GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ - fi \ -&& GO_URL="https://go.dev/dl/${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ -&& wget -nv "${GO_URL}" \ -&& echo "${GO_SHA256} ${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | sha256sum -c - \ -&& tar xf "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ -&& mv go "/usr/local/${GO_VERSION}" \ -&& ln -s "/usr/local/${GO_VERSION}" /usr/local/go \ -&& rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ -&& echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null + fi && \ + GO_URL="https://go.dev/dl/${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + wget -nv "${GO_URL}" && \ + echo "${GO_SHA256} ${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | sha256sum -c - && \ + tar xf "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + mv go "/usr/local/${GO_VERSION}" && \ + ln -s "/usr/local/${GO_VERSION}" /usr/local/go && \ + rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null # -------------------------------------------------------------------- diff --git a/images/docker/cbdb/build/jammy/configs/init_system.sh b/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh similarity index 100% rename from images/docker/cbdb/build/jammy/configs/init_system.sh rename to images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh diff --git a/images/docker/cbdb/test/jammy/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile similarity index 91% rename from images/docker/cbdb/test/jammy/Dockerfile rename to images/docker/cbdb/test/ubuntu22.04/Dockerfile index 1255e21..6e2244f 100644 --- a/images/docker/cbdb/test/jammy/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -32,13 +32,13 @@ # project has yet to be fully endorsed by the ASF. # # -------------------------------------------------------------------- -# Dockerfile for Cloudberry Database Base Environment +# Dockerfile for Apache Cloudberry Base Environment # -------------------------------------------------------------------- # This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as -# a base environment for evaluating the Cloudberry Database. It installs +# a base environment for evaluating the Apache Cloudberry. It installs # necessary system utilities, configures the environment for SSH access, -# and sets up a 'gpadmin' user with sudo privileges. The Cloudberry -# Database DEB can be installed into this container for testing and +# and sets up a 'gpadmin' user with sudo privileges. The Apache +# Cloudberry DEB can be installed into this container for testing and # functional verification. # # Key Features: @@ -92,9 +92,9 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install sudo \ which -RUN apt-get install -y locales \ -&& locale-gen "en_US.UTF-8" \ -&& update-locale LC_ALL="en_US.UTF-8" +RUN apt-get install -y locales && \ + locale-gen "en_US.UTF-8" && \ + update-locale LC_ALL="en_US.UTF-8" # -------------------------------------------------------------------- diff --git a/images/docker/cbdb/test/jammy/configs/init_system.sh b/images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh similarity index 100% rename from images/docker/cbdb/test/jammy/configs/init_system.sh rename to images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh diff --git a/packaging/deb/jammy/debian/changelog b/packaging/deb/ubuntu22.04/debian/changelog similarity index 100% rename from packaging/deb/jammy/debian/changelog rename to packaging/deb/ubuntu22.04/debian/changelog diff --git a/packaging/deb/jammy/debian/compat b/packaging/deb/ubuntu22.04/debian/compat similarity index 100% rename from packaging/deb/jammy/debian/compat rename to packaging/deb/ubuntu22.04/debian/compat diff --git a/packaging/deb/jammy/debian/control b/packaging/deb/ubuntu22.04/debian/control similarity index 97% rename from packaging/deb/jammy/debian/control rename to packaging/deb/ubuntu22.04/debian/control index bdc545c..37af9f0 100644 --- a/packaging/deb/jammy/debian/control +++ b/packaging/deb/ubuntu22.04/debian/control @@ -1,5 +1,5 @@ Source: apache-cloudberry-db-incubating -Maintainer: open-gpdb dev team https://github.com/open-gpdb/gpdb +Maintainer: Apache Cloudberry (Incubating) https://cloudberry.apache.org Section: database Build-Depends: debhelper (>= 9), bison, diff --git a/packaging/deb/jammy/debian/install b/packaging/deb/ubuntu22.04/debian/install similarity index 100% rename from packaging/deb/jammy/debian/install rename to packaging/deb/ubuntu22.04/debian/install diff --git a/packaging/deb/jammy/debian/postinst b/packaging/deb/ubuntu22.04/debian/postinst similarity index 100% rename from packaging/deb/jammy/debian/postinst rename to packaging/deb/ubuntu22.04/debian/postinst diff --git a/packaging/deb/jammy/debian/preinst b/packaging/deb/ubuntu22.04/debian/preinst similarity index 100% rename from packaging/deb/jammy/debian/preinst rename to packaging/deb/ubuntu22.04/debian/preinst diff --git a/packaging/deb/jammy/debian/rules b/packaging/deb/ubuntu22.04/debian/rules similarity index 71% rename from packaging/deb/jammy/debian/rules rename to packaging/deb/ubuntu22.04/debian/rules index fa5f5a4..518bcf7 100755 --- a/packaging/deb/jammy/debian/rules +++ b/packaging/deb/ubuntu22.04/debian/rules @@ -3,12 +3,12 @@ DH_VERBOSE = 1 DPKG_EXPORT_BUILDFLAGS = 1 -GPDB_BIN_PATH := /usr/local/cloudberry-db +CBDB_BIN_PATH := /usr/local/cloudberry-db DEBIAN_DESTINATION := $(shell pwd)/debian/build -# assumes that CWD is root of gpdb source -GPDB_PKG_VERSION := $(GPDB_PKG_VERSION) -PACKAGE_GPDB := $(shell cat debian/control | egrep "^Package: " | cut -d " " -f 2) +# assumes that CWD is root of cbdb source +CBDB_PKG_VERSION := $(CBDB_PKG_VERSION) +PACKAGE_CBDB := $(shell cat debian/control | egrep "^Package: " | cut -d " " -f 2) PATH := ${DEBIAN_DESTINATION}/bin:${PATH} .PHONY: gpinstall @@ -24,7 +24,7 @@ gpinstall: override_dh_auto_install: gpinstall # the staging directory for creating a debian is NOT the right GPHOME. # change GPHOME to point to the post-install target install directory. - sed -i "s#GPHOME=.*#GPHOME=${GPDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh + sed -i "s#GPHOME=.*#GPHOME=${CBDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh override_dh_auto_build: echo "Skipping build" @@ -39,8 +39,8 @@ override_dh_auto_test: echo "Skipping auto test" override_dh_gencontrol: - echo "using version ${GPDB_PKG_VERSION} for binary GPDB" - dh_gencontrol -- -v${GPDB_PKG_VERSION} -p${PACKAGE_GPDB} + echo "using version ${CBDB_PKG_VERSION} for binary Cloudberry" + dh_gencontrol -- -v${CBDB_PKG_VERSION} -p${PACKAGE_CBDB} override_dh_shlibdeps: LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/libfakeroot:${DEBIAN_DESTINATION}/lib dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info diff --git a/packaging/deb/jammy/debian/source/format b/packaging/deb/ubuntu22.04/debian/source/format similarity index 100% rename from packaging/deb/jammy/debian/source/format rename to packaging/deb/ubuntu22.04/debian/source/format diff --git a/packaging/deb/jammy/debian/source/local-options b/packaging/deb/ubuntu22.04/debian/source/local-options similarity index 100% rename from packaging/deb/jammy/debian/source/local-options rename to packaging/deb/ubuntu22.04/debian/source/local-options diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh index 2896668..4dbff91 100644 --- a/scripts/build-deb.sh +++ b/scripts/build-deb.sh @@ -60,7 +60,7 @@ check_commands() { function print_changelog() { cat < debian/changelog @@ -145,4 +145,4 @@ if ! eval "$DEBBUILD_CMD"; then fi # Print completion message -echo "DEB build completed successfully with package $GPDB_PKG_VERSION" +echo "DEB build completed successfully with package $CBDB_PKG_VERSION" From 044362fc30553bf4464ae2f94422381193ddc5af Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:23:32 +0300 Subject: [PATCH 06/29] Update images/docker/cbdb/test/ubuntu22.04/Dockerfile Co-authored-by: Dianjin Wang --- images/docker/cbdb/test/ubuntu22.04/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index 6e2244f..d857d03 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -83,7 +83,7 @@ RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo $TIMEZONE_V RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ file \ gdb \ - glibc-locale-source \ + glibc-source \ make \ openssh \ openssh-clients \ From 181f398997ffe772ce7ad63935a0d719edfbfe46 Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:23:42 +0300 Subject: [PATCH 07/29] Update images/docker/cbdb/test/ubuntu22.04/Dockerfile Co-authored-by: Dianjin Wang --- images/docker/cbdb/test/ubuntu22.04/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index d857d03..dbdc35d 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -86,7 +86,7 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install glibc-source \ make \ openssh \ - openssh-clients \ + openssh-client \ openssh-server \ procps-ng \ sudo \ From 843e6679df6e4168eda678864fc366114834ba01 Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:23:53 +0300 Subject: [PATCH 08/29] Update images/docker/cbdb/test/ubuntu22.04/Dockerfile Co-authored-by: Dianjin Wang --- images/docker/cbdb/test/ubuntu22.04/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index dbdc35d..b4b49e3 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -88,7 +88,7 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install openssh \ openssh-client \ openssh-server \ - procps-ng \ + procps \ sudo \ which From 0e777df7d62546d4e6d37c4300268b866a666c35 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Fri, 4 Apr 2025 13:22:37 +0300 Subject: [PATCH 09/29] Combed the Dockerfile based on the comments in PR --- .../docker/cbdb/build/ubuntu22.04/Dockerfile | 40 +++++++++++-------- .../docker/cbdb/test/ubuntu22.04/Dockerfile | 23 +++++------ 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/images/docker/cbdb/build/ubuntu22.04/Dockerfile b/images/docker/cbdb/build/ubuntu22.04/Dockerfile index d584ac0..4023f43 100644 --- a/images/docker/cbdb/build/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/build/ubuntu22.04/Dockerfile @@ -60,29 +60,24 @@ # docker run -h cdw -it cloudberry-db-base-env # -------------------------------------------------------------------- -FROM ubuntu:jammy - -ARG accessKeyId -ARG secretAccessKey -ARG bucketName +FROM ubuntu:22.04 # Argument for configuring the timezone ARG TIMEZONE_VAR="Europe/London" - SHELL ["/bin/bash", "-o", "pipefail", "-c"] ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NOWARNINGS="yes" RUN stat -fc %T /sys/fs/cgroup/ -RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo $TIMEZONE_VAR > /etc/timezone +RUN ln -snf /usr/share/zoneinfo/${TIMEZONE_VAR} /etc/localtime && echo "${TIMEZONE_VAR}" > /etc/timezone # -------------------------------------------------------------------- # Install Development Tools and Utilities # -------------------------------------------------------------------- -RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ - libssl-dev \ gnupg \ openssl \ debhelper \ @@ -141,7 +136,16 @@ RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install python3.10 \ python3.10-dev \ python3-distutils \ - sudo + sudo \ + liblz4-dev \ + iputils-ping \ + iproute2 \ + rsync \ + less \ + openssh-client \ + openssh-server \ + lsof \ + apt-utils RUN apt-get install -y locales && \ locale-gen "en_US.UTF-8" && \ @@ -183,21 +187,23 @@ RUN cd && GO_VERSION="go1.23.4" && \ # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- -RUN ln -s /usr/bin/python2.7 /usr/bin/python +RUN ln -s /usr/bin/python3.10 /usr/bin/python COPY ./configs/* /tmp/ -RUN echo ${TIMEZONE_VAR} > /etc/timezone && \ - chmod 777 /tmp/init_system.sh && \ +RUN chmod 777 /tmp/init_system.sh && \ groupadd gpadmin && \ useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin && \ echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers && \ - locale-gen "en_US.UTF-8" && \ - update-locale LC_ALL="en_US.UTF-8" + mkdir -p /run/sshd && chmod 0755 /run/sshd + +RUN apt-get install -y libhyperic-sigar-java +# Clean up APT when done. +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/tmp/* + +ENV USER=gpadmin USER gpadmin WORKDIR /home/gpadmin -RUN sudo DEBIAN_FRONTEND=noninteractive apt install -y libhyperic-sigar-java libaprutil1-dev libuv1-dev - CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index b4b49e3..5b01404 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -60,43 +60,36 @@ # docker run -h cdw -it cloudberry-db-base-env # -------------------------------------------------------------------- -FROM ubuntu:jammy - -ARG accessKeyId -ARG secretAccessKey -ARG bucketName +FROM ubuntu:22.04 # Argument for configuring the timezone ARG TIMEZONE_VAR="Europe/London" - SHELL ["/bin/bash", "-o", "pipefail", "-c"] ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NOWARNINGS="yes" RUN stat -fc %T /sys/fs/cgroup/ -RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo $TIMEZONE_VAR > /etc/timezone +RUN ln -snf /usr/share/zoneinfo/${TIMEZONE_VAR} /etc/localtime && echo "${TIMEZONE_VAR}" > /etc/timezone # -------------------------------------------------------------------- # Install Development Tools and Utilities # -------------------------------------------------------------------- -RUN apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ +RUN apt-get update && apt-get install -y --no-install-recommends \ file \ gdb \ glibc-source \ make \ - openssh \ openssh-client \ openssh-server \ procps \ - sudo \ - which + sudo RUN apt-get install -y locales && \ locale-gen "en_US.UTF-8" && \ update-locale LC_ALL="en_US.UTF-8" - # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- @@ -110,9 +103,11 @@ RUN echo ${TIMEZONE_VAR} > /etc/timezone && \ groupadd gpadmin && \ useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin && \ echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers && \ - locale-gen "en_US.UTF-8" && \ - update-locale LC_ALL="en_US.UTF-8" + mkdir -p /run/sshd && chmod 0755 /run/sshd + +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/tmp/* +ENV USER=gpadmin USER gpadmin WORKDIR /home/gpadmin From 9dd4b6f3715cbb5b2dda0f8a4319a2a6f0bec623 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Fri, 4 Apr 2025 17:58:19 +0300 Subject: [PATCH 10/29] install into /usr/cloudberry-db --- images/docker/cbdb/build/ubuntu22.04/Dockerfile | 6 +++++- images/docker/cbdb/test/ubuntu22.04/Dockerfile | 5 ++++- packaging/deb/ubuntu22.04/debian/install | 2 +- packaging/deb/ubuntu22.04/debian/postinst | 4 +++- scripts/build-deb.sh | 0 5 files changed, 13 insertions(+), 4 deletions(-) mode change 100644 => 100755 scripts/build-deb.sh diff --git a/images/docker/cbdb/build/ubuntu22.04/Dockerfile b/images/docker/cbdb/build/ubuntu22.04/Dockerfile index 4023f43..761f030 100644 --- a/images/docker/cbdb/build/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/build/ubuntu22.04/Dockerfile @@ -145,7 +145,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ openssh-client \ openssh-server \ lsof \ - apt-utils + apt-utils \ + silversearcher-ag \ + net-tools \ + python-pip \ + python-six RUN apt-get install -y locales && \ locale-gen "en_US.UTF-8" && \ diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index 5b01404..8b02b05 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -84,7 +84,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ openssh-client \ openssh-server \ procps \ - sudo + sudo \ + net-tools \ + python-pip \ + python-six RUN apt-get install -y locales && \ locale-gen "en_US.UTF-8" && \ diff --git a/packaging/deb/ubuntu22.04/debian/install b/packaging/deb/ubuntu22.04/debian/install index 4ce7955..3e29bb0 100644 --- a/packaging/deb/ubuntu22.04/debian/install +++ b/packaging/deb/ubuntu22.04/debian/install @@ -1 +1 @@ -debian/build/* /usr/local/cloudberry-db +debian/build/* /usr/cloudberry-db diff --git a/packaging/deb/ubuntu22.04/debian/postinst b/packaging/deb/ubuntu22.04/debian/postinst index 9a4318e..2ecc307 100644 --- a/packaging/deb/ubuntu22.04/debian/postinst +++ b/packaging/deb/ubuntu22.04/debian/postinst @@ -3,10 +3,12 @@ set -e GPADMIN=gpadmin -GPHOME=/usr/local/cloudberry-db +GPHOME=/usr/cloudberry-db if [ "$1" = configure ]; then chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} + ln -s ${GPHOME} /usr/local/cloudberry-db + fi diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh old mode 100644 new mode 100755 From 38b9d5b65ace14025ef7e4fd41d57300181cec36 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Mon, 7 Apr 2025 15:28:14 +0300 Subject: [PATCH 11/29] Move debian package creation changes from https://github.com/apache/cloudberry-devops-release/pull/13 to separate PR --- .../cloudberry/scripts/build-cloudberry.sh | 6 +- .../cloudberry/scripts/cloudberry-utils.sh | 14 +- .../scripts/configure-cloudberry.sh | 18 +-- .../scripts/create-cloudberry-demo-cluster.sh | 6 +- .../destroy-cloudberry-demo-cluster.sh | 2 +- .../cloudberry/scripts/test-cloudberry.sh | 2 +- .../cloudberry/scripts/unittest-cloudberry.sh | 4 +- packaging/deb/ubuntu22.04/debian/changelog | 5 + packaging/deb/ubuntu22.04/debian/compat | 1 + packaging/deb/ubuntu22.04/debian/control | 101 ++++++++++++ packaging/deb/ubuntu22.04/debian/install | 1 + packaging/deb/ubuntu22.04/debian/postinst | 14 ++ packaging/deb/ubuntu22.04/debian/preinst | 13 ++ packaging/deb/ubuntu22.04/debian/rules | 49 ++++++ .../deb/ubuntu22.04/debian/source/format | 1 + .../ubuntu22.04/debian/source/local-options | 2 + scripts/build-deb.sh | 148 ++++++++++++++++++ 17 files changed, 366 insertions(+), 21 deletions(-) create mode 100644 packaging/deb/ubuntu22.04/debian/changelog create mode 100644 packaging/deb/ubuntu22.04/debian/compat create mode 100644 packaging/deb/ubuntu22.04/debian/control create mode 100644 packaging/deb/ubuntu22.04/debian/install create mode 100644 packaging/deb/ubuntu22.04/debian/postinst create mode 100644 packaging/deb/ubuntu22.04/debian/preinst create mode 100755 packaging/deb/ubuntu22.04/debian/rules create mode 100644 packaging/deb/ubuntu22.04/debian/source/format create mode 100644 packaging/deb/ubuntu22.04/debian/source/local-options create mode 100755 scripts/build-deb.sh diff --git a/build_automation/cloudberry/scripts/build-cloudberry.sh b/build_automation/cloudberry/scripts/build-cloudberry.sh index db04f1b..4ce4752 100755 --- a/build_automation/cloudberry/scripts/build-cloudberry.sh +++ b/build_automation/cloudberry/scripts/build-cloudberry.sh @@ -45,7 +45,7 @@ # Prerequisites: # - configure-cloudberry.sh must be run first # - Required build dependencies must be installed -# - /usr/local/cloudberry-db/lib must exist and be writable +# - ${BUILD_DESTINATION}/lib must exist and be writable # # Exit Codes: # 0 - Build and installation completed successfully @@ -67,11 +67,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" BUILD_LOG="${LOG_DIR}/build.log" # Initialize environment -init_environment "Cloudberry Build Script" "${BUILD_LOG}" +init_environment "Cloudberry Build Script" "${BUILD_LOG}" "${BUILD_DESTINATION}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Build process diff --git a/build_automation/cloudberry/scripts/cloudberry-utils.sh b/build_automation/cloudberry/scripts/cloudberry-utils.sh index a0cc1b8..a3c4b61 100755 --- a/build_automation/cloudberry/scripts/cloudberry-utils.sh +++ b/build_automation/cloudberry/scripts/cloudberry-utils.sh @@ -30,11 +30,12 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Functions: -# init_environment "Script Name" "Log File" +# init_environment "Script Name" "Log File" "Build Destination" # - Initialize logging and verify environment # - Parameters: # * script_name: Name of the calling script # * log_file: Path to log file +# * build_destination: Path to Cloudberry destination, by default is /usr/local/cloudberry-db # - Returns: 0 on success, 1 on failure # # execute_cmd command [args...] @@ -70,7 +71,7 @@ # # Example: # source ./cloudberry-utils.sh -# init_environment "My Script" "${LOG_FILE}" +# init_environment "My Script" "${LOG_FILE}" "${BUILD_DESTINATION}" # execute_cmd make clean # log_section "Build Process" # execute_cmd make -j$(nproc) @@ -79,10 +80,18 @@ # # -------------------------------------------------------------------- +DEFAULT_BUILD_DESTINATION=/usr/local/cloudberry-db + # Initialize logging and environment init_environment() { local script_name=$1 local log_file=$2 + local build_destination=$3 + + if [ -z "$build_destination" ]; then + build_destination=${DEFAULT_BUILD_DESTINATION} + fi + export BUILD_DESTINATION=$build_destination echo "=== Initializing environment for ${script_name} ===" echo "${script_name} executed at $(date)" | tee -a "${log_file}" @@ -91,6 +100,7 @@ init_environment() { echo "Working directory: $(pwd)" | tee -a "${log_file}" echo "Source directory: ${SRC_DIR}" | tee -a "${log_file}" echo "Log directory: ${LOG_DIR}" | tee -a "${log_file}" + echo "Build destination: ${BUILD_DESTINATION}" | tee -a "${log_file}" if [ -z "${SRC_DIR:-}" ]; then echo "Error: SRC_DIR environment variable is not set" | tee -a "${log_file}" diff --git a/build_automation/cloudberry/scripts/configure-cloudberry.sh b/build_automation/cloudberry/scripts/configure-cloudberry.sh index 587ba10..14dec02 100755 --- a/build_automation/cloudberry/scripts/configure-cloudberry.sh +++ b/build_automation/cloudberry/scripts/configure-cloudberry.sh @@ -23,7 +23,7 @@ # Description: Configures Apache Cloudberry build environment and runs # ./configure with optimized settings. Performs the # following: -# 1. Prepares /usr/local/cloudberry-db directory +# 1. Prepares ${BUILD_DESTINATION} directory # 2. Sets up library dependencies # 3. Configures build with required features enabled # @@ -96,22 +96,22 @@ export LOG_DIR="${SRC_DIR}/build-logs" CONFIGURE_LOG="${LOG_DIR}/configure.log" # Initialize environment -init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" +init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" "${BUILD_DESTINATION}" # Initial setup log_section "Initial Setup" -execute_cmd sudo rm -rf /usr/local/cloudberry-db || exit 2 +execute_cmd sudo rm -rf ${BUILD_DESTINATION}/* || exit 2 execute_cmd sudo chmod a+w /usr/local || exit 2 -execute_cmd mkdir -p /usr/local/cloudberry-db/lib || exit 2 +execute_cmd sudo mkdir -p ${BUILD_DESTINATION}/lib || exit 2 execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ /usr/local/xerces-c/lib/libxerces-c-3.3.so \ - /usr/local/cloudberry-db/lib || exit 3 -execute_cmd sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db || exit 2 + ${BUILD_DESTINATION}/lib || exit 3 +execute_cmd sudo chown -R gpadmin:gpadmin ${BUILD_DESTINATION} || exit 2 log_section_end "Initial Setup" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Add debug options if ENABLE_DEBUG is set to "true" @@ -126,7 +126,7 @@ fi # Configure build log_section "Configure" -execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ +execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --disable-external-fts \ --enable-gpcloud \ --enable-ic-proxy \ @@ -150,7 +150,7 @@ execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ --with-openssl \ --with-uuid=e2fs \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=/usr/local/cloudberry-db/lib || exit 4 + --with-libraries=${BUILD_DESTINATION}/lib || exit 4 log_section_end "Configure" # Capture version information diff --git a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh index 396780b..88d51ef 100755 --- a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh @@ -35,7 +35,7 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Prerequisites: -# - Apache Cloudberry must be installed (/usr/local/cloudberry-db) +# - Apache Cloudberry must be installed (${BUILD_DESTINATION}) # - SSH must be configured for passwordless access to localhost # - User must have permissions to create cluster directories # - PostgreSQL client tools (psql) must be available @@ -75,11 +75,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/cluster.log" # Initialize environment -init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" +init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" "${BUILD_DESTINATION}" # Setup environment log_section "Environment Setup" -source /usr/local/cloudberry-db/greenplum_path.sh || exit 1 +source ${BUILD_DESTINATION}/greenplum_path.sh || exit 1 log_section_end "Environment Setup" # Verify SSH access diff --git a/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh index 3d4ce24..ae2bdc0 100755 --- a/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh @@ -71,7 +71,7 @@ export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/destroy-cluster.log" # Initialize environment -init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" +init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" "${BUILD_DESTINATION}" # Source Cloudberry environment log_section "Environment Setup" diff --git a/build_automation/cloudberry/scripts/test-cloudberry.sh b/build_automation/cloudberry/scripts/test-cloudberry.sh index 411f16c..e3ff951 100755 --- a/build_automation/cloudberry/scripts/test-cloudberry.sh +++ b/build_automation/cloudberry/scripts/test-cloudberry.sh @@ -60,7 +60,7 @@ export LOG_DIR="build-logs" TEST_LOG="${LOG_DIR}/test.log" # Initialize environment -init_environment "Cloudberry Test Script" "${TEST_LOG}" +init_environment "Cloudberry Test Script" "${TEST_LOG}" "${BUILD_DESTINATION}" # Source Cloudberry environment log_section "Environment Setup" diff --git a/build_automation/cloudberry/scripts/unittest-cloudberry.sh b/build_automation/cloudberry/scripts/unittest-cloudberry.sh index f7bc120..7e2283e 100755 --- a/build_automation/cloudberry/scripts/unittest-cloudberry.sh +++ b/build_automation/cloudberry/scripts/unittest-cloudberry.sh @@ -52,11 +52,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" UNITTEST_LOG="${LOG_DIR}/unittest.log" # Initialize environment -init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" +init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" "${BUILD_DESTINATION}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Unittest process diff --git a/packaging/deb/ubuntu22.04/debian/changelog b/packaging/deb/ubuntu22.04/debian/changelog new file mode 100644 index 0000000..211d271 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/changelog @@ -0,0 +1,5 @@ +apache-cloudberry-db-incubating (2.0.0) jammy; urgency=medium + + * Initial release. + + -- Cloudberry Team Wed, 26 Mar 2025 11:10:44 +0000 diff --git a/packaging/deb/ubuntu22.04/debian/compat b/packaging/deb/ubuntu22.04/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/compat @@ -0,0 +1 @@ +9 diff --git a/packaging/deb/ubuntu22.04/debian/control b/packaging/deb/ubuntu22.04/debian/control new file mode 100644 index 0000000..37af9f0 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/control @@ -0,0 +1,101 @@ +Source: apache-cloudberry-db-incubating +Maintainer: Apache Cloudberry (Incubating) https://cloudberry.apache.org +Section: database +Build-Depends: debhelper (>= 9), + bison, + ca-certificates-java, + ca-certificates, + cmake, + curl, + cgroup-tools, + flex, + gcc-11, + g++-11, + g++-11-multilib, + git, + krb5-multidev, + libapr1-dev, + libbz2-dev, + libcurl4-gnutls-dev, + libevent-dev, + libkrb5-dev, + libldap2-dev, + libperl-dev, + libreadline6-dev, + libssl-dev, + libxml2-dev, + libyaml-dev, + libzstd-dev, + libaprutil1-dev, + libpam0g-dev, + libpam0g, + libcgroup1, + libyaml-0-2, + libldap-2.5-0, + libssl3, + ninja-build, + python2.7-dev, + python-setuptools, + quilt, + unzip, + wget, + zlib1g-dev, + libuv1-dev + +Package: apache-cloudberry-db-incubating +Architecture: amd64 +Depends: curl, + cgroup-tools, + iputils-ping, + krb5-multidev, + less, + libapr1, + libbz2-1.0, + libcurl4, + libcurl3-gnutls, + libevent-2.1-7, + libreadline8, + libxml2, + libyaml-0-2, + libldap-2.5-0, + libzstd1, + libcgroup1, + libssl3, + libpam0g, + locales, + net-tools, + openssh-client, + openssh-server, + openssl, + python2.7, + python2.7-dev, + python-pip, + python-six, + rsync, + wget, + zlib1g, + libuv1 +Description: Apache Cloudberry (incubating) is an advanced, open-source, massively + parallel processing (MPP) data warehouse developed from PostgreSQL and + Greenplum. It is designed for high-performance analytics on + large-scale data sets, offering powerful analytical capabilities and + enhanced security features. + Key Features: + - Massively parallel processing for optimized performance + - Advanced analytics for complex data processing + - Integration with ETL and BI tools + - Compatibility with multiple data sources and formats + - Enhanced security features + Apache Cloudberry supports both batch processing and real-time data + warehousing, making it a versatile solution for modern data + environments. + Apache Cloudberry (incubating) is an effort undergoing incubation at + the Apache Software Foundation (ASF), sponsored by the Apache + Incubator PMC. + Incubation is required of all newly accepted projects until a further + review indicates that the infrastructure, communications, and decision + making process have stabilized in a manner consistent with other + successful ASF projects. + While incubation status is not necessarily a reflection of the + completeness or stability of the code, it does indicate that the + project has yet to be fully endorsed by the ASF. diff --git a/packaging/deb/ubuntu22.04/debian/install b/packaging/deb/ubuntu22.04/debian/install new file mode 100644 index 0000000..3e29bb0 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/install @@ -0,0 +1 @@ +debian/build/* /usr/cloudberry-db diff --git a/packaging/deb/ubuntu22.04/debian/postinst b/packaging/deb/ubuntu22.04/debian/postinst new file mode 100644 index 0000000..2ecc307 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/postinst @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin +GPHOME=/usr/cloudberry-db + +if [ "$1" = configure ]; then + + chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} + + ln -s ${GPHOME} /usr/local/cloudberry-db + +fi diff --git a/packaging/deb/ubuntu22.04/debian/preinst b/packaging/deb/ubuntu22.04/debian/preinst new file mode 100644 index 0000000..29d6211 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/preinst @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin + +case $1 in + install|upgrade) + id "${GPADMIN}" >> /dev/null 2>&1 || \ + useradd -r -m -d /home/${GPADMIN} -s /bin/bash ${GPADMIN} + ;; +esac + diff --git a/packaging/deb/ubuntu22.04/debian/rules b/packaging/deb/ubuntu22.04/debian/rules new file mode 100755 index 0000000..518bcf7 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/rules @@ -0,0 +1,49 @@ +#!/usr/bin/make -f + +DH_VERBOSE = 1 +DPKG_EXPORT_BUILDFLAGS = 1 + +CBDB_BIN_PATH := /usr/local/cloudberry-db +DEBIAN_DESTINATION := $(shell pwd)/debian/build + +# assumes that CWD is root of cbdb source +CBDB_PKG_VERSION := $(CBDB_PKG_VERSION) +PACKAGE_CBDB := $(shell cat debian/control | egrep "^Package: " | cut -d " " -f 2) +PATH := ${DEBIAN_DESTINATION}/bin:${PATH} + +.PHONY: gpinstall + +include /usr/share/dpkg/default.mk + +%: + dh $@ --parallel + +gpinstall: + make install + +override_dh_auto_install: gpinstall + # the staging directory for creating a debian is NOT the right GPHOME. + # change GPHOME to point to the post-install target install directory. + sed -i "s#GPHOME=.*#GPHOME=${CBDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh + +override_dh_auto_build: + echo "Skipping build" + +override_dh_auto_clean: + echo "Skipping clean" + +override_dh_auto_configure: + echo "Skipping configure" + +override_dh_auto_test: + echo "Skipping auto test" + +override_dh_gencontrol: + echo "using version ${CBDB_PKG_VERSION} for binary Cloudberry" + dh_gencontrol -- -v${CBDB_PKG_VERSION} -p${PACKAGE_CBDB} + +override_dh_shlibdeps: + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/libfakeroot:${DEBIAN_DESTINATION}/lib dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info + +clean_dev_local: + rm -rf ${DEBIAN_DESTINATION} diff --git a/packaging/deb/ubuntu22.04/debian/source/format b/packaging/deb/ubuntu22.04/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/packaging/deb/ubuntu22.04/debian/source/local-options b/packaging/deb/ubuntu22.04/debian/source/local-options new file mode 100644 index 0000000..00131ee --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/source/local-options @@ -0,0 +1,2 @@ +#abort-on-upstream-changes +#unapply-patches diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh new file mode 100755 index 0000000..4dbff91 --- /dev/null +++ b/scripts/build-deb.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Script Name: build-deb.sh +# +# Description: +# This script automates the process of building an DEB package using a specified +# version number. It ensures that the necessary tools are installed +# and that the control file exists before attempting to build the DEB. The script +# also includes error handling to provide meaningful feedback in case of failure. +# +# Usage: +# ./build-deb.sh [-v ] [-h] [--dry-run] +# +# Options: +# -v, --version : Specify the version (required) +# -h, --help : Display this help and exit +# -n, --dry-run : Show what would be done, without making any changes +# +# Example: +# ./build-deb.sh -v 1.5.5 # Build with version 1.5.5 +# +# Prerequisites: +# - The dpkg-buildpackage package must be installed (provides the dpkg-buildpackage command). +# - The control file must exist at debian/control. +# +# Error Handling: +# The script includes checks to ensure: +# - The version option (-v or --version) is provided. +# - The necessary commands are available. +# - The control file exists at the specified location. +# If any of these checks fail, the script exits with an appropriate error message. + +# Enable strict mode for better error handling +set -euo pipefail + +# Default values +VERSION="" +RELEASE="1" +DEBUG_BUILD=false + +# Function to display usage information +usage() { + echo "Usage: $0 [-v ] [-h] [--dry-run]" + echo " -v, --version : Specify the version (optional)" + echo " -h, --help : Display this help and exit" + echo " -n, --dry-run : Show what would be done, without making any changes" + exit 1 +} + +# Function to check if required commands are available +check_commands() { + local cmds=("dpkg-buildpackage") + for cmd in "${cmds[@]}"; do + if ! command -v "$cmd" &> /dev/null; then + echo "Error: Required command '$cmd' not found. Please install it before running the script." + exit 1 + fi + done +} + +function print_changelog() { +cat < $(date +'%a, %d %b %Y %H:%M:%S %z') +EOF +} + +# Parse options +while [[ "$#" -gt 0 ]]; do + case $1 in + -v|--version) + VERSION="$2" + shift 2 + ;; + -h|--help) + usage + ;; + -n|--dry-run) + DRY_RUN=true + shift + ;; + *) + echo "Unknown option: ($1)" + shift + ;; + esac +done + +export CBDB_FULL_VERSION=$VERSION + +# Set version if not provided +if [ -z "${VERSION}" ]; then + export CBDB_FULL_VERSION=$(./getversion | cut -d'-' -f 1 | cut -d'+' -f 1) +fi + +if [[ ! $CBDB_FULL_VERSION =~ ^[0-9] ]]; then + export CBDB_FULL_VERSION="0.$CBDB_FULL_VERSION" +fi + +if [ -z ${BUILD_NUMBER+x} ]; then + export BUILD_NUMBER=1 +fi + +if [ -z ${BUILD_USER+x} ]; then + export BUILD_USER=github +fi + +export CBDB_PKG_VERSION=${CBDB_FULL_VERSION}-${BUILD_NUMBER}-$(git --git-dir=.git rev-list HEAD --count).$(git --git-dir=.git rev-parse --short HEAD) + +# Check if required commands are available +check_commands + +# Define the control file path +CONTROL_FILE=debian/control + +# Check if the spec file exists +if [ ! -f "$CONTROL_FILE" ]; then + echo "Error: Control file not found at $CONTROL_FILE." + exit 1 +fi + +# Build the rpmbuild command based on options +DEBBUILD_CMD="dpkg-buildpackage -us -uc" + +# Dry-run mode +if [ "${DRY_RUN:-false}" = true ]; then + echo "Dry-run mode: This is what would be done:" + print_changelog + echo "" + echo "$DEBBUILD_CMD" + exit 0 +fi + +# Run debbuild with the provided options +echo "Building DEB with Version $CBDB_FULL_VERSION ..." + +print_changelog > debian/changelog + +if ! eval "$DEBBUILD_CMD"; then + echo "Error: deb build failed." + exit 1 +fi + +# Print completion message +echo "DEB build completed successfully with package $CBDB_PKG_VERSION" From 60bbbb7d2f409ee8f85cdd3df061c4d4896779f5 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Mon, 7 Apr 2025 15:30:08 +0300 Subject: [PATCH 12/29] Move some changes to https://github.com/apache/cloudberry-devops-release/pull/15 --- packaging/deb/ubuntu22.04/debian/changelog | 5 - packaging/deb/ubuntu22.04/debian/compat | 1 - packaging/deb/ubuntu22.04/debian/control | 101 ------------ packaging/deb/ubuntu22.04/debian/install | 1 - packaging/deb/ubuntu22.04/debian/postinst | 14 -- packaging/deb/ubuntu22.04/debian/preinst | 13 -- packaging/deb/ubuntu22.04/debian/rules | 49 ------ .../deb/ubuntu22.04/debian/source/format | 1 - .../ubuntu22.04/debian/source/local-options | 2 - scripts/build-deb.sh | 148 ------------------ 10 files changed, 335 deletions(-) delete mode 100644 packaging/deb/ubuntu22.04/debian/changelog delete mode 100644 packaging/deb/ubuntu22.04/debian/compat delete mode 100644 packaging/deb/ubuntu22.04/debian/control delete mode 100644 packaging/deb/ubuntu22.04/debian/install delete mode 100644 packaging/deb/ubuntu22.04/debian/postinst delete mode 100644 packaging/deb/ubuntu22.04/debian/preinst delete mode 100755 packaging/deb/ubuntu22.04/debian/rules delete mode 100644 packaging/deb/ubuntu22.04/debian/source/format delete mode 100644 packaging/deb/ubuntu22.04/debian/source/local-options delete mode 100755 scripts/build-deb.sh diff --git a/packaging/deb/ubuntu22.04/debian/changelog b/packaging/deb/ubuntu22.04/debian/changelog deleted file mode 100644 index 211d271..0000000 --- a/packaging/deb/ubuntu22.04/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -apache-cloudberry-db-incubating (2.0.0) jammy; urgency=medium - - * Initial release. - - -- Cloudberry Team Wed, 26 Mar 2025 11:10:44 +0000 diff --git a/packaging/deb/ubuntu22.04/debian/compat b/packaging/deb/ubuntu22.04/debian/compat deleted file mode 100644 index ec63514..0000000 --- a/packaging/deb/ubuntu22.04/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/packaging/deb/ubuntu22.04/debian/control b/packaging/deb/ubuntu22.04/debian/control deleted file mode 100644 index 37af9f0..0000000 --- a/packaging/deb/ubuntu22.04/debian/control +++ /dev/null @@ -1,101 +0,0 @@ -Source: apache-cloudberry-db-incubating -Maintainer: Apache Cloudberry (Incubating) https://cloudberry.apache.org -Section: database -Build-Depends: debhelper (>= 9), - bison, - ca-certificates-java, - ca-certificates, - cmake, - curl, - cgroup-tools, - flex, - gcc-11, - g++-11, - g++-11-multilib, - git, - krb5-multidev, - libapr1-dev, - libbz2-dev, - libcurl4-gnutls-dev, - libevent-dev, - libkrb5-dev, - libldap2-dev, - libperl-dev, - libreadline6-dev, - libssl-dev, - libxml2-dev, - libyaml-dev, - libzstd-dev, - libaprutil1-dev, - libpam0g-dev, - libpam0g, - libcgroup1, - libyaml-0-2, - libldap-2.5-0, - libssl3, - ninja-build, - python2.7-dev, - python-setuptools, - quilt, - unzip, - wget, - zlib1g-dev, - libuv1-dev - -Package: apache-cloudberry-db-incubating -Architecture: amd64 -Depends: curl, - cgroup-tools, - iputils-ping, - krb5-multidev, - less, - libapr1, - libbz2-1.0, - libcurl4, - libcurl3-gnutls, - libevent-2.1-7, - libreadline8, - libxml2, - libyaml-0-2, - libldap-2.5-0, - libzstd1, - libcgroup1, - libssl3, - libpam0g, - locales, - net-tools, - openssh-client, - openssh-server, - openssl, - python2.7, - python2.7-dev, - python-pip, - python-six, - rsync, - wget, - zlib1g, - libuv1 -Description: Apache Cloudberry (incubating) is an advanced, open-source, massively - parallel processing (MPP) data warehouse developed from PostgreSQL and - Greenplum. It is designed for high-performance analytics on - large-scale data sets, offering powerful analytical capabilities and - enhanced security features. - Key Features: - - Massively parallel processing for optimized performance - - Advanced analytics for complex data processing - - Integration with ETL and BI tools - - Compatibility with multiple data sources and formats - - Enhanced security features - Apache Cloudberry supports both batch processing and real-time data - warehousing, making it a versatile solution for modern data - environments. - Apache Cloudberry (incubating) is an effort undergoing incubation at - the Apache Software Foundation (ASF), sponsored by the Apache - Incubator PMC. - Incubation is required of all newly accepted projects until a further - review indicates that the infrastructure, communications, and decision - making process have stabilized in a manner consistent with other - successful ASF projects. - While incubation status is not necessarily a reflection of the - completeness or stability of the code, it does indicate that the - project has yet to be fully endorsed by the ASF. diff --git a/packaging/deb/ubuntu22.04/debian/install b/packaging/deb/ubuntu22.04/debian/install deleted file mode 100644 index 3e29bb0..0000000 --- a/packaging/deb/ubuntu22.04/debian/install +++ /dev/null @@ -1 +0,0 @@ -debian/build/* /usr/cloudberry-db diff --git a/packaging/deb/ubuntu22.04/debian/postinst b/packaging/deb/ubuntu22.04/debian/postinst deleted file mode 100644 index 2ecc307..0000000 --- a/packaging/deb/ubuntu22.04/debian/postinst +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -e - -GPADMIN=gpadmin -GPHOME=/usr/cloudberry-db - -if [ "$1" = configure ]; then - - chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} - - ln -s ${GPHOME} /usr/local/cloudberry-db - -fi diff --git a/packaging/deb/ubuntu22.04/debian/preinst b/packaging/deb/ubuntu22.04/debian/preinst deleted file mode 100644 index 29d6211..0000000 --- a/packaging/deb/ubuntu22.04/debian/preinst +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -e - -GPADMIN=gpadmin - -case $1 in - install|upgrade) - id "${GPADMIN}" >> /dev/null 2>&1 || \ - useradd -r -m -d /home/${GPADMIN} -s /bin/bash ${GPADMIN} - ;; -esac - diff --git a/packaging/deb/ubuntu22.04/debian/rules b/packaging/deb/ubuntu22.04/debian/rules deleted file mode 100755 index 518bcf7..0000000 --- a/packaging/deb/ubuntu22.04/debian/rules +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/make -f - -DH_VERBOSE = 1 -DPKG_EXPORT_BUILDFLAGS = 1 - -CBDB_BIN_PATH := /usr/local/cloudberry-db -DEBIAN_DESTINATION := $(shell pwd)/debian/build - -# assumes that CWD is root of cbdb source -CBDB_PKG_VERSION := $(CBDB_PKG_VERSION) -PACKAGE_CBDB := $(shell cat debian/control | egrep "^Package: " | cut -d " " -f 2) -PATH := ${DEBIAN_DESTINATION}/bin:${PATH} - -.PHONY: gpinstall - -include /usr/share/dpkg/default.mk - -%: - dh $@ --parallel - -gpinstall: - make install - -override_dh_auto_install: gpinstall - # the staging directory for creating a debian is NOT the right GPHOME. - # change GPHOME to point to the post-install target install directory. - sed -i "s#GPHOME=.*#GPHOME=${CBDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh - -override_dh_auto_build: - echo "Skipping build" - -override_dh_auto_clean: - echo "Skipping clean" - -override_dh_auto_configure: - echo "Skipping configure" - -override_dh_auto_test: - echo "Skipping auto test" - -override_dh_gencontrol: - echo "using version ${CBDB_PKG_VERSION} for binary Cloudberry" - dh_gencontrol -- -v${CBDB_PKG_VERSION} -p${PACKAGE_CBDB} - -override_dh_shlibdeps: - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/libfakeroot:${DEBIAN_DESTINATION}/lib dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info - -clean_dev_local: - rm -rf ${DEBIAN_DESTINATION} diff --git a/packaging/deb/ubuntu22.04/debian/source/format b/packaging/deb/ubuntu22.04/debian/source/format deleted file mode 100644 index 89ae9db..0000000 --- a/packaging/deb/ubuntu22.04/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/packaging/deb/ubuntu22.04/debian/source/local-options b/packaging/deb/ubuntu22.04/debian/source/local-options deleted file mode 100644 index 00131ee..0000000 --- a/packaging/deb/ubuntu22.04/debian/source/local-options +++ /dev/null @@ -1,2 +0,0 @@ -#abort-on-upstream-changes -#unapply-patches diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh deleted file mode 100755 index 4dbff91..0000000 --- a/scripts/build-deb.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# -# Script Name: build-deb.sh -# -# Description: -# This script automates the process of building an DEB package using a specified -# version number. It ensures that the necessary tools are installed -# and that the control file exists before attempting to build the DEB. The script -# also includes error handling to provide meaningful feedback in case of failure. -# -# Usage: -# ./build-deb.sh [-v ] [-h] [--dry-run] -# -# Options: -# -v, --version : Specify the version (required) -# -h, --help : Display this help and exit -# -n, --dry-run : Show what would be done, without making any changes -# -# Example: -# ./build-deb.sh -v 1.5.5 # Build with version 1.5.5 -# -# Prerequisites: -# - The dpkg-buildpackage package must be installed (provides the dpkg-buildpackage command). -# - The control file must exist at debian/control. -# -# Error Handling: -# The script includes checks to ensure: -# - The version option (-v or --version) is provided. -# - The necessary commands are available. -# - The control file exists at the specified location. -# If any of these checks fail, the script exits with an appropriate error message. - -# Enable strict mode for better error handling -set -euo pipefail - -# Default values -VERSION="" -RELEASE="1" -DEBUG_BUILD=false - -# Function to display usage information -usage() { - echo "Usage: $0 [-v ] [-h] [--dry-run]" - echo " -v, --version : Specify the version (optional)" - echo " -h, --help : Display this help and exit" - echo " -n, --dry-run : Show what would be done, without making any changes" - exit 1 -} - -# Function to check if required commands are available -check_commands() { - local cmds=("dpkg-buildpackage") - for cmd in "${cmds[@]}"; do - if ! command -v "$cmd" &> /dev/null; then - echo "Error: Required command '$cmd' not found. Please install it before running the script." - exit 1 - fi - done -} - -function print_changelog() { -cat < $(date +'%a, %d %b %Y %H:%M:%S %z') -EOF -} - -# Parse options -while [[ "$#" -gt 0 ]]; do - case $1 in - -v|--version) - VERSION="$2" - shift 2 - ;; - -h|--help) - usage - ;; - -n|--dry-run) - DRY_RUN=true - shift - ;; - *) - echo "Unknown option: ($1)" - shift - ;; - esac -done - -export CBDB_FULL_VERSION=$VERSION - -# Set version if not provided -if [ -z "${VERSION}" ]; then - export CBDB_FULL_VERSION=$(./getversion | cut -d'-' -f 1 | cut -d'+' -f 1) -fi - -if [[ ! $CBDB_FULL_VERSION =~ ^[0-9] ]]; then - export CBDB_FULL_VERSION="0.$CBDB_FULL_VERSION" -fi - -if [ -z ${BUILD_NUMBER+x} ]; then - export BUILD_NUMBER=1 -fi - -if [ -z ${BUILD_USER+x} ]; then - export BUILD_USER=github -fi - -export CBDB_PKG_VERSION=${CBDB_FULL_VERSION}-${BUILD_NUMBER}-$(git --git-dir=.git rev-list HEAD --count).$(git --git-dir=.git rev-parse --short HEAD) - -# Check if required commands are available -check_commands - -# Define the control file path -CONTROL_FILE=debian/control - -# Check if the spec file exists -if [ ! -f "$CONTROL_FILE" ]; then - echo "Error: Control file not found at $CONTROL_FILE." - exit 1 -fi - -# Build the rpmbuild command based on options -DEBBUILD_CMD="dpkg-buildpackage -us -uc" - -# Dry-run mode -if [ "${DRY_RUN:-false}" = true ]; then - echo "Dry-run mode: This is what would be done:" - print_changelog - echo "" - echo "$DEBBUILD_CMD" - exit 0 -fi - -# Run debbuild with the provided options -echo "Building DEB with Version $CBDB_FULL_VERSION ..." - -print_changelog > debian/changelog - -if ! eval "$DEBBUILD_CMD"; then - echo "Error: deb build failed." - exit 1 -fi - -# Print completion message -echo "DEB build completed successfully with package $CBDB_PKG_VERSION" From 0e5dd66e5a482e71f38fefa85ddf1492718a304e Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Mon, 7 Apr 2025 15:32:56 +0300 Subject: [PATCH 13/29] Do not touch build_automation/cloudberry/scripts in this PR --- .../cloudberry/scripts/build-cloudberry.sh | 6 +++--- .../cloudberry/scripts/cloudberry-utils.sh | 14 ++------------ .../cloudberry/scripts/configure-cloudberry.sh | 18 +++++++++--------- .../scripts/create-cloudberry-demo-cluster.sh | 6 +++--- .../scripts/destroy-cloudberry-demo-cluster.sh | 2 +- .../cloudberry/scripts/test-cloudberry.sh | 2 +- .../cloudberry/scripts/unittest-cloudberry.sh | 4 ++-- 7 files changed, 21 insertions(+), 31 deletions(-) diff --git a/build_automation/cloudberry/scripts/build-cloudberry.sh b/build_automation/cloudberry/scripts/build-cloudberry.sh index 4ce4752..db04f1b 100755 --- a/build_automation/cloudberry/scripts/build-cloudberry.sh +++ b/build_automation/cloudberry/scripts/build-cloudberry.sh @@ -45,7 +45,7 @@ # Prerequisites: # - configure-cloudberry.sh must be run first # - Required build dependencies must be installed -# - ${BUILD_DESTINATION}/lib must exist and be writable +# - /usr/local/cloudberry-db/lib must exist and be writable # # Exit Codes: # 0 - Build and installation completed successfully @@ -67,11 +67,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" BUILD_LOG="${LOG_DIR}/build.log" # Initialize environment -init_environment "Cloudberry Build Script" "${BUILD_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Build Script" "${BUILD_LOG}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Build process diff --git a/build_automation/cloudberry/scripts/cloudberry-utils.sh b/build_automation/cloudberry/scripts/cloudberry-utils.sh index a3c4b61..a0cc1b8 100755 --- a/build_automation/cloudberry/scripts/cloudberry-utils.sh +++ b/build_automation/cloudberry/scripts/cloudberry-utils.sh @@ -30,12 +30,11 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Functions: -# init_environment "Script Name" "Log File" "Build Destination" +# init_environment "Script Name" "Log File" # - Initialize logging and verify environment # - Parameters: # * script_name: Name of the calling script # * log_file: Path to log file -# * build_destination: Path to Cloudberry destination, by default is /usr/local/cloudberry-db # - Returns: 0 on success, 1 on failure # # execute_cmd command [args...] @@ -71,7 +70,7 @@ # # Example: # source ./cloudberry-utils.sh -# init_environment "My Script" "${LOG_FILE}" "${BUILD_DESTINATION}" +# init_environment "My Script" "${LOG_FILE}" # execute_cmd make clean # log_section "Build Process" # execute_cmd make -j$(nproc) @@ -80,18 +79,10 @@ # # -------------------------------------------------------------------- -DEFAULT_BUILD_DESTINATION=/usr/local/cloudberry-db - # Initialize logging and environment init_environment() { local script_name=$1 local log_file=$2 - local build_destination=$3 - - if [ -z "$build_destination" ]; then - build_destination=${DEFAULT_BUILD_DESTINATION} - fi - export BUILD_DESTINATION=$build_destination echo "=== Initializing environment for ${script_name} ===" echo "${script_name} executed at $(date)" | tee -a "${log_file}" @@ -100,7 +91,6 @@ init_environment() { echo "Working directory: $(pwd)" | tee -a "${log_file}" echo "Source directory: ${SRC_DIR}" | tee -a "${log_file}" echo "Log directory: ${LOG_DIR}" | tee -a "${log_file}" - echo "Build destination: ${BUILD_DESTINATION}" | tee -a "${log_file}" if [ -z "${SRC_DIR:-}" ]; then echo "Error: SRC_DIR environment variable is not set" | tee -a "${log_file}" diff --git a/build_automation/cloudberry/scripts/configure-cloudberry.sh b/build_automation/cloudberry/scripts/configure-cloudberry.sh index 14dec02..587ba10 100755 --- a/build_automation/cloudberry/scripts/configure-cloudberry.sh +++ b/build_automation/cloudberry/scripts/configure-cloudberry.sh @@ -23,7 +23,7 @@ # Description: Configures Apache Cloudberry build environment and runs # ./configure with optimized settings. Performs the # following: -# 1. Prepares ${BUILD_DESTINATION} directory +# 1. Prepares /usr/local/cloudberry-db directory # 2. Sets up library dependencies # 3. Configures build with required features enabled # @@ -96,22 +96,22 @@ export LOG_DIR="${SRC_DIR}/build-logs" CONFIGURE_LOG="${LOG_DIR}/configure.log" # Initialize environment -init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" # Initial setup log_section "Initial Setup" -execute_cmd sudo rm -rf ${BUILD_DESTINATION}/* || exit 2 +execute_cmd sudo rm -rf /usr/local/cloudberry-db || exit 2 execute_cmd sudo chmod a+w /usr/local || exit 2 -execute_cmd sudo mkdir -p ${BUILD_DESTINATION}/lib || exit 2 +execute_cmd mkdir -p /usr/local/cloudberry-db/lib || exit 2 execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ /usr/local/xerces-c/lib/libxerces-c-3.3.so \ - ${BUILD_DESTINATION}/lib || exit 3 -execute_cmd sudo chown -R gpadmin:gpadmin ${BUILD_DESTINATION} || exit 2 + /usr/local/cloudberry-db/lib || exit 3 +execute_cmd sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db || exit 2 log_section_end "Initial Setup" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Add debug options if ENABLE_DEBUG is set to "true" @@ -126,7 +126,7 @@ fi # Configure build log_section "Configure" -execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ +execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ --disable-external-fts \ --enable-gpcloud \ --enable-ic-proxy \ @@ -150,7 +150,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --with-openssl \ --with-uuid=e2fs \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=${BUILD_DESTINATION}/lib || exit 4 + --with-libraries=/usr/local/cloudberry-db/lib || exit 4 log_section_end "Configure" # Capture version information diff --git a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh index 88d51ef..396780b 100755 --- a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh @@ -35,7 +35,7 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Prerequisites: -# - Apache Cloudberry must be installed (${BUILD_DESTINATION}) +# - Apache Cloudberry must be installed (/usr/local/cloudberry-db) # - SSH must be configured for passwordless access to localhost # - User must have permissions to create cluster directories # - PostgreSQL client tools (psql) must be available @@ -75,11 +75,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/cluster.log" # Initialize environment -init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" # Setup environment log_section "Environment Setup" -source ${BUILD_DESTINATION}/greenplum_path.sh || exit 1 +source /usr/local/cloudberry-db/greenplum_path.sh || exit 1 log_section_end "Environment Setup" # Verify SSH access diff --git a/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh index ae2bdc0..3d4ce24 100755 --- a/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh @@ -71,7 +71,7 @@ export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/destroy-cluster.log" # Initialize environment -init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" "${BUILD_DESTINATION}" +init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" # Source Cloudberry environment log_section "Environment Setup" diff --git a/build_automation/cloudberry/scripts/test-cloudberry.sh b/build_automation/cloudberry/scripts/test-cloudberry.sh index e3ff951..411f16c 100755 --- a/build_automation/cloudberry/scripts/test-cloudberry.sh +++ b/build_automation/cloudberry/scripts/test-cloudberry.sh @@ -60,7 +60,7 @@ export LOG_DIR="build-logs" TEST_LOG="${LOG_DIR}/test.log" # Initialize environment -init_environment "Cloudberry Test Script" "${TEST_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Test Script" "${TEST_LOG}" # Source Cloudberry environment log_section "Environment Setup" diff --git a/build_automation/cloudberry/scripts/unittest-cloudberry.sh b/build_automation/cloudberry/scripts/unittest-cloudberry.sh index 7e2283e..f7bc120 100755 --- a/build_automation/cloudberry/scripts/unittest-cloudberry.sh +++ b/build_automation/cloudberry/scripts/unittest-cloudberry.sh @@ -52,11 +52,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" UNITTEST_LOG="${LOG_DIR}/unittest.log" # Initialize environment -init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Unittest process From b30da1fec9e5b619d2bb38c1e66f609ef04f44d7 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Mon, 7 Apr 2025 19:03:05 +0300 Subject: [PATCH 14/29] Add test_cloudberry_db_env --- .../docker/cbdb/build/ubuntu22.04/Dockerfile | 23 ++- .../build/ubuntu22.04/configs/90-cbdb-limits | 32 ++++ .../build/ubuntu22.04/configs/init_system.sh | 0 .../build/ubuntu22.04/tests/requirements.txt | 3 + .../tests/testinfra/test_cloudberry_db_env.py | 158 ++++++++++++++++++ .../docker/cbdb/test/ubuntu22.04/Dockerfile | 3 +- .../test/ubuntu22.04/configs/90-cbdb-limits | 32 ++++ .../ubuntu22.04/configs/gpinitsystem.conf | 87 ++++++++++ 8 files changed, 331 insertions(+), 7 deletions(-) create mode 100644 images/docker/cbdb/build/ubuntu22.04/configs/90-cbdb-limits mode change 100644 => 100755 images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh create mode 100644 images/docker/cbdb/build/ubuntu22.04/tests/requirements.txt create mode 100644 images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py create mode 100644 images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits create mode 100644 images/docker/cbdb/test/ubuntu22.04/configs/gpinitsystem.conf diff --git a/images/docker/cbdb/build/ubuntu22.04/Dockerfile b/images/docker/cbdb/build/ubuntu22.04/Dockerfile index 761f030..ee57e95 100644 --- a/images/docker/cbdb/build/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/build/ubuntu22.04/Dockerfile @@ -91,6 +91,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ cmake \ curl \ + htop \ cgroup-tools \ flex \ gcc-11 \ @@ -98,6 +99,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ g++-11-multilib \ git \ krb5-multidev \ + libprotobuf-dev \ + protobuf-compiler \ libapr1-dev \ libbz2-dev \ libcurl4-gnutls-dev \ @@ -121,8 +124,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libldap-2.5-0 \ libssl3 \ ninja-build \ - python-setuptools \ - python3-setuptools \ quilt \ unzip \ wget \ @@ -136,6 +137,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3.10 \ python3.10-dev \ python3-distutils \ + python3-pip \ + python3-setuptools \ sudo \ liblz4-dev \ iputils-ping \ @@ -148,8 +151,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ apt-utils \ silversearcher-ag \ net-tools \ - python-pip \ - python-six + python-six \ + tzdata \ + vim RUN apt-get install -y locales && \ locale-gen "en_US.UTF-8" && \ @@ -195,9 +199,10 @@ RUN ln -s /usr/bin/python3.10 /usr/bin/python COPY ./configs/* /tmp/ -RUN chmod 777 /tmp/init_system.sh && \ +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + chmod 755 /tmp/init_system.sh && \ groupadd gpadmin && \ - useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin && \ + useradd -rm -d /home/gpadmin -s /bin/bash -g gpadmin -G sudo -G root -u 1001 gpadmin && \ echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers && \ mkdir -p /run/sshd && chmod 0755 /run/sshd @@ -206,6 +211,12 @@ RUN apt-get install -y libhyperic-sigar-java # Clean up APT when done. RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/tmp/* +# Install testinfra via pip +RUN pip3 install pytest-testinfra + +# Example: Copying test files into the container +COPY tests /tests + ENV USER=gpadmin USER gpadmin WORKDIR /home/gpadmin diff --git a/images/docker/cbdb/build/ubuntu22.04/configs/90-cbdb-limits b/images/docker/cbdb/build/ubuntu22.04/configs/90-cbdb-limits new file mode 100644 index 0000000..8ea1d9e --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits + # -------------------------------------------------------------------- + # + # Licensed to the Apache Software Foundation (ASF) under one or more + # contributor license agreements. See the NOTICE file distributed + # with this work for additional information regarding copyright + # ownership. The ASF licenses this file to You under the Apache + # License, Version 2.0 (the "License"); you may not use this file + # except in compliance with the License. You may obtain a copy of the + # License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + # implied. See the License for the specific language governing + # permissions and limitations under the License. + # + # -------------------------------------------------------------------- + + # Core dump file size limits for gpadmin + gpadmin soft core unlimited + gpadmin hard core unlimited + + # Open file limits for gpadmin + gpadmin soft nofile 524288 + gpadmin hard nofile 524288 + + # Process limits for gpadmin + gpadmin soft nproc 131072 + gpadmin hard nproc 131072 diff --git a/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh b/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh old mode 100644 new mode 100755 diff --git a/images/docker/cbdb/build/ubuntu22.04/tests/requirements.txt b/images/docker/cbdb/build/ubuntu22.04/tests/requirements.txt new file mode 100644 index 0000000..b9711ed --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/tests/requirements.txt @@ -0,0 +1,3 @@ +testinfra +pytest-testinfra +paramiko diff --git a/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py b/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py new file mode 100644 index 0000000..09c3e33 --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py @@ -0,0 +1,158 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +import testinfra + +def test_installed_packages(host): + """ + Test if the essential packages are installed. + """ + packages = [ + "apt-utils", + "bison", + "build-essential", + "ca-certificates", + "ca-certificates-java", + "cgroup-tools", + "cmake", + "curl", + "debhelper", + "debootstrap", + "devscripts", + "equivs", + "flex", + "flex", + "g++-11", + "g++-11-multilib", + "gcc-11", + "git", + "gnupg", + "htop", + "iproute2", + "iputils-ping", + "krb5-multidev", + "less", + "libapr1-dev", + "libaprutil1-dev", + "libbz2-dev", + "libcgroup1", + "libcurl4-gnutls-dev", + "libevent-dev", + "libfakeroot", + "libgpgme11", + "libgpgme-dev", + "libkrb5-dev", + "libldap-2.5-0", + "libldap2-dev", + "liblz4-dev", + "libpam0g", + "libpam0g-dev", + "libperl-dev", + "libprotobuf-dev", + "libpstreams-dev", + "libreadline-dev", + "libssl3", + "libssl-dev", + "libsystemd-dev", + "libuv1-dev", + "libxerces-c-dev", + "libxml2-dev", + "libyaml-0-2", + "libyaml-dev", + "libzstd-dev", + "lsof", + "make", + "net-tools", + "ninja-build", + "openssh-client", + "openssh-server", + "openssl", + "pkg-config", + "protobuf-compiler", + "python3.10", + "python3.10-dev", + "python3-distutils", + "python3-pip", + "python3-setuptools", + "python-six", + "quilt", + "rsync", + "silversearcher-ag", + "sudo", + "tzdata", + "unzip", + "vim", + "wget", + "zlib1g-dev", + ] + for package in packages: + pkg = host.package(package) + assert pkg.is_installed + + +def test_user_gpadmin_exists(host): + """ + Test if the gpadmin user exists and is configured properly. + """ + user = host.user("gpadmin") + assert user.exists + assert "gpadmin" in user.groups + + +def test_ssh_service(host): + """ + Test if SSH service is configured correctly. + """ + sshd_config = host.file("/etc/ssh/sshd_config") + assert sshd_config.exists + + +def test_timezone(host): + """ + Test if the timezone is configured correctly. + """ + localtime = host.file("/etc/localtime") + assert localtime.exists or localtime.is_symlink + + +def test_system_limits_configured(host): + """ + Test if the custom system limits are applied. + """ + limits_file = host.file("/etc/security/limits.d/90-cbdb-limits") + assert limits_file.exists + + +def test_init_system_script(host): + """ + Test if the init_system.sh script is present and executable. + """ + script = host.file("/tmp/init_system.sh") + assert script.exists + assert script.mode == 0o755 + + +def test_locale_generated(host): + """ + Test if the en_US.UTF-8 locale is correctly generated. + """ + locale = host.run("locale -a | grep en_US.utf8") + assert locale.exit_status == 0 + assert "en_US.utf8" in locale.stdout diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index 8b02b05..e90f5b9 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -101,7 +101,8 @@ RUN ln -s /usr/bin/python2.7 /usr/bin/python COPY ./configs/* /tmp/ -RUN echo ${TIMEZONE_VAR} > /etc/timezone && \ +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + echo ${TIMEZONE_VAR} > /etc/timezone && \ chmod 777 /tmp/init_system.sh && \ groupadd gpadmin && \ useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin && \ diff --git a/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits b/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits new file mode 100644 index 0000000..8ea1d9e --- /dev/null +++ b/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits + # -------------------------------------------------------------------- + # + # Licensed to the Apache Software Foundation (ASF) under one or more + # contributor license agreements. See the NOTICE file distributed + # with this work for additional information regarding copyright + # ownership. The ASF licenses this file to You under the Apache + # License, Version 2.0 (the "License"); you may not use this file + # except in compliance with the License. You may obtain a copy of the + # License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + # implied. See the License for the specific language governing + # permissions and limitations under the License. + # + # -------------------------------------------------------------------- + + # Core dump file size limits for gpadmin + gpadmin soft core unlimited + gpadmin hard core unlimited + + # Open file limits for gpadmin + gpadmin soft nofile 524288 + gpadmin hard nofile 524288 + + # Process limits for gpadmin + gpadmin soft nproc 131072 + gpadmin hard nproc 131072 diff --git a/images/docker/cbdb/test/ubuntu22.04/configs/gpinitsystem.conf b/images/docker/cbdb/test/ubuntu22.04/configs/gpinitsystem.conf new file mode 100644 index 0000000..896c8c7 --- /dev/null +++ b/images/docker/cbdb/test/ubuntu22.04/configs/gpinitsystem.conf @@ -0,0 +1,87 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# ---------------------------------------------------------------------- +# gpinitsystem Configuration File for Cloudberry Database +# ---------------------------------------------------------------------- +# This configuration file is used to initialize a Cloudberry Database +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# ---------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# ---------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# ---------------------------------------------------------------------- From 1114b87bd5898a71d7c94a44777553a9e4433bbd Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Mon, 7 Apr 2025 19:12:01 +0300 Subject: [PATCH 15/29] Add gpinitsystem.conf --- .../ubuntu22.04/configs/gpinitsystem.conf | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 images/docker/cbdb/build/ubuntu22.04/configs/gpinitsystem.conf diff --git a/images/docker/cbdb/build/ubuntu22.04/configs/gpinitsystem.conf b/images/docker/cbdb/build/ubuntu22.04/configs/gpinitsystem.conf new file mode 100644 index 0000000..d4d3122 --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/configs/gpinitsystem.conf @@ -0,0 +1,89 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# gpinitsystem Configuration File for Apache Cloudberry +# -------------------------------------------------------------------- +# This configuration file is used to initialize an Apache Cloudberry +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# -------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# -------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# -------------------------------------------------------------------- From aa2f271e9d145508251969f5cd2da528b8eab4ab Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Tue, 8 Apr 2025 12:20:30 +0300 Subject: [PATCH 16/29] Use python3 in test container --- images/docker/cbdb/test/ubuntu22.04/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index e90f5b9..c496e24 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -86,7 +86,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ procps \ sudo \ net-tools \ - python-pip \ + python3.10 \ + python3.10-dev \ + python3-distutils \ + python3-pip \ python-six RUN apt-get install -y locales && \ @@ -97,15 +100,15 @@ RUN apt-get install -y locales && \ # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- -RUN ln -s /usr/bin/python2.7 /usr/bin/python +RUN ln -s /usr/bin/python3.10 /usr/bin/python COPY ./configs/* /tmp/ RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ echo ${TIMEZONE_VAR} > /etc/timezone && \ - chmod 777 /tmp/init_system.sh && \ + chmod 755 /tmp/init_system.sh && \ groupadd gpadmin && \ - useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin && \ + useradd -rm -d /home/gpadmin -s /bin/bash -g gpadmin -G root -G sudo -u 1001 gpadmin && \ echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers && \ mkdir -p /run/sshd && chmod 0755 /run/sshd From ff80ee03f2f480ac783021620cb4111aa83db15e Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Fri, 9 May 2025 16:08:00 +0300 Subject: [PATCH 17/29] Fix test debian build --- .../tests/testinfra/test_cloudberry_db_env.py | 74 +++++---------- .../docker/cbdb/test/ubuntu22.04/Dockerfile | 90 +++++++++++-------- .../test/ubuntu22.04/configs/90-cbdb-limits | 62 ++++++------- .../test/ubuntu22.04/configs/init_system.sh | 3 + 4 files changed, 110 insertions(+), 119 deletions(-) diff --git a/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py b/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py index 09c3e33..d7f018a 100644 --- a/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py +++ b/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py @@ -17,96 +17,66 @@ # permissions and limitations under the License. # # -------------------------------------------------------------------- - + import testinfra - + def test_installed_packages(host): """ Test if the essential packages are installed. """ packages = [ - "apt-utils", + "bat", "bison", - "build-essential", - "ca-certificates", - "ca-certificates-java", - "cgroup-tools", "cmake", - "curl", - "debhelper", - "debootstrap", - "devscripts", - "equivs", - "flex", "flex", "g++-11", - "g++-11-multilib", "gcc-11", "git", - "gnupg", "htop", "iproute2", "iputils-ping", - "krb5-multidev", - "less", "libapr1-dev", - "libaprutil1-dev", "libbz2-dev", - "libcgroup1", "libcurl4-gnutls-dev", "libevent-dev", - "libfakeroot", - "libgpgme11", - "libgpgme-dev", + "libipc-run-perl", "libkrb5-dev", - "libldap-2.5-0", - "libldap2-dev", + "libldap-dev", "liblz4-dev", - "libpam0g", "libpam0g-dev", "libperl-dev", "libprotobuf-dev", - "libpstreams-dev", "libreadline-dev", - "libssl3", "libssl-dev", - "libsystemd-dev", "libuv1-dev", "libxerces-c-dev", "libxml2-dev", - "libyaml-0-2", "libyaml-dev", "libzstd-dev", + "locales", "lsof", "make", - "net-tools", - "ninja-build", - "openssh-client", "openssh-server", - "openssl", "pkg-config", "protobuf-compiler", - "python3.10", - "python3.10-dev", "python3-distutils", "python3-pip", "python3-setuptools", - "python-six", - "quilt", + "python3.10", + "python3.10-dev", "rsync", "silversearcher-ag", "sudo", "tzdata", - "unzip", "vim", "wget", - "zlib1g-dev", + "zlib1g-dev" ] for package in packages: pkg = host.package(package) assert pkg.is_installed - - + + def test_user_gpadmin_exists(host): """ Test if the gpadmin user exists and is configured properly. @@ -114,32 +84,32 @@ def test_user_gpadmin_exists(host): user = host.user("gpadmin") assert user.exists assert "gpadmin" in user.groups - - + + def test_ssh_service(host): """ Test if SSH service is configured correctly. """ sshd_config = host.file("/etc/ssh/sshd_config") assert sshd_config.exists - - + + def test_timezone(host): """ Test if the timezone is configured correctly. """ localtime = host.file("/etc/localtime") - assert localtime.exists or localtime.is_symlink - - + assert localtime.exists + + def test_system_limits_configured(host): """ Test if the custom system limits are applied. """ limits_file = host.file("/etc/security/limits.d/90-cbdb-limits") assert limits_file.exists - - + + def test_init_system_script(host): """ Test if the init_system.sh script is present and executable. @@ -147,8 +117,8 @@ def test_init_system_script(host): script = host.file("/tmp/init_system.sh") assert script.exists assert script.mode == 0o755 - - + + def test_locale_generated(host): """ Test if the en_US.UTF-8 locale is correctly generated. diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index c496e24..f9d160b 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -37,8 +37,8 @@ # This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as # a base environment for evaluating the Apache Cloudberry. It installs # necessary system utilities, configures the environment for SSH access, -# and sets up a 'gpadmin' user with sudo privileges. The Apache -# Cloudberry DEB can be installed into this container for testing and +# and sets up a 'gpadmin' user with sudo privileges. The Apache Cloudberry +# DEB can be installed into this container for testing and # functional verification. # # Key Features: @@ -65,56 +65,74 @@ FROM ubuntu:22.04 # Argument for configuring the timezone ARG TIMEZONE_VAR="Europe/London" -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin +ENV TZ=${TIMEZONE_VAR} ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" -RUN stat -fc %T /sys/fs/cgroup/ - -RUN ln -snf /usr/share/zoneinfo/${TIMEZONE_VAR} /etc/localtime && echo "${TIMEZONE_VAR}" > /etc/timezone - # -------------------------------------------------------------------- # Install Development Tools and Utilities # -------------------------------------------------------------------- -RUN apt-get update && apt-get install -y --no-install-recommends \ - file \ - gdb \ - glibc-source \ - make \ - openssh-client \ - openssh-server \ - procps \ - sudo \ - net-tools \ - python3.10 \ - python3.10-dev \ - python3-distutils \ - python3-pip \ - python-six -RUN apt-get install -y locales && \ - locale-gen "en_US.UTF-8" && \ - update-locale LC_ALL="en_US.UTF-8" +RUN apt-get update && \ + apt-get install -y -qq \ + htop \ + bat \ + silversearcher-ag \ + vim \ + wget \ + git \ + iproute2 \ + iputils-ping \ + lsof \ + openssh-server \ + pkg-config \ + python3.10 \ + python3-distutils \ + python3-pip \ + python3-setuptools \ + rsync \ + sudo \ + tzdata && \ + apt-get install -y -qq locales && \ + locale-gen ${LANG} && \ + update-locale LANG=${LANG} && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/bin/python3.10 /usr/bin/python # -------------------------------------------------------------------- -# Copy Configuration Files and Setup the Environment +# User Creation and Configuration # -------------------------------------------------------------------- +# - Create the 'gpadmin' user and group. +# - Configure the 'gpadmin' user with passwordless sudo privileges. +# - Add Cloudberry-specific entries to the gpadmin's .bashrc. +# -------------------------------------------------------------------- +RUN /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd -m -g gpadmin gpadmin && \ + echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ + chmod 0440 /etc/sudoers.d/90-gpadmin && \ + echo '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/greenplum_path.sh ]; then\n source /usr/local/cloudberry/greenplum_path.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc -RUN ln -s /usr/bin/python3.10 /usr/bin/python - +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- +# - Copy custom configuration files from the build context to /tmp/. +# - Apply custom system limits and timezone. +# - Set up SSH for password-based authentication. +# - Generate locale and set the default locale to en_US.UTF-8. +# -------------------------------------------------------------------- COPY ./configs/* /tmp/ RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ - echo ${TIMEZONE_VAR} > /etc/timezone && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ chmod 755 /tmp/init_system.sh && \ - groupadd gpadmin && \ - useradd -rm -d /home/gpadmin -s /bin/bash -g gpadmin -G root -G sudo -u 1001 gpadmin && \ - echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers && \ - mkdir -p /run/sshd && chmod 0755 /run/sshd - -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/tmp/* + ssh-keygen -A -ENV USER=gpadmin USER gpadmin WORKDIR /home/gpadmin diff --git a/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits b/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits index 8ea1d9e..474957c 100644 --- a/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits +++ b/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits @@ -1,32 +1,32 @@ # /etc/security/limits.d/90-db-limits - # -------------------------------------------------------------------- - # - # Licensed to the Apache Software Foundation (ASF) under one or more - # contributor license agreements. See the NOTICE file distributed - # with this work for additional information regarding copyright - # ownership. The ASF licenses this file to You under the Apache - # License, Version 2.0 (the "License"); you may not use this file - # except in compliance with the License. You may obtain a copy of the - # License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - # implied. See the License for the specific language governing - # permissions and limitations under the License. - # - # -------------------------------------------------------------------- - - # Core dump file size limits for gpadmin - gpadmin soft core unlimited - gpadmin hard core unlimited - - # Open file limits for gpadmin - gpadmin soft nofile 524288 - gpadmin hard nofile 524288 - - # Process limits for gpadmin - gpadmin soft nproc 131072 - gpadmin hard nproc 131072 +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited + +# Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 + +# Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 diff --git a/images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh b/images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh index 749d789..1928fe5 100644 --- a/images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh +++ b/images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh @@ -57,6 +57,9 @@ fi # SSH. This is useful for development and debugging purposes. If the SSH # daemon fails to start, the script exits with an error. # -------------------------------------------------------------------- +sudo mkdir -p /run/sshd +sudo chmod 755 /run/sshd + if ! sudo /usr/sbin/sshd; then echo "Failed to start SSH daemon" >&2 exit 1 From 70a7ef88937667151705ff8847330af8ebe98879 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Fri, 6 Jun 2025 14:07:16 +0000 Subject: [PATCH 18/29] Cope edespino files --- .../cloudberry/scripts/cloudberry-utils.sh | 11 ++ .../scripts/configure-cloudberry.sh | 18 +- .../docker/cbdb/build/ubuntu22.04/Dockerfile | 187 +++++++----------- 3 files changed, 98 insertions(+), 118 deletions(-) diff --git a/build_automation/cloudberry/scripts/cloudberry-utils.sh b/build_automation/cloudberry/scripts/cloudberry-utils.sh index a0cc1b8..2a8aa21 100755 --- a/build_automation/cloudberry/scripts/cloudberry-utils.sh +++ b/build_automation/cloudberry/scripts/cloudberry-utils.sh @@ -146,3 +146,14 @@ log_completion() { local timestamp=$(date "+%Y.%m.%d-%H.%M.%S") echo "${script_name} execution completed successfully at ${timestamp}" | tee -a "${log_file}" } + +detect_os() { + if [ -f /etc/os-release ]; then + . /etc/os-release + OS_ID=$ID + OS_VERSION=$VERSION_ID + else + echo "Unsupported system: cannot detect OS" >&2 + exit 99 + fi +} diff --git a/build_automation/cloudberry/scripts/configure-cloudberry.sh b/build_automation/cloudberry/scripts/configure-cloudberry.sh index 4d84c3b..21d6c77 100755 --- a/build_automation/cloudberry/scripts/configure-cloudberry.sh +++ b/build_automation/cloudberry/scripts/configure-cloudberry.sh @@ -33,7 +33,6 @@ # - MapReduce Processing # - Oracle Compatibility (orafce) # - ORCA Query Optimizer -# - PAX Access Method # - PXF External Table Access # - Test Automation Support (tap-tests) # @@ -92,6 +91,11 @@ set -euo pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "${SCRIPT_DIR}/cloudberry-utils.sh" +# Call it before conditional logic +detect_os + +echo "Detected OS: $OS_ID $OS_VERSION" + # Define log directory and files export LOG_DIR="${SRC_DIR}/build-logs" CONFIGURE_LOG="${LOG_DIR}/configure.log" @@ -104,9 +108,12 @@ log_section "Initial Setup" execute_cmd sudo rm -rf /usr/local/cloudberry-db || exit 2 execute_cmd sudo chmod a+w /usr/local || exit 2 execute_cmd mkdir -p /usr/local/cloudberry-db/lib || exit 2 -execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ - /usr/local/xerces-c/lib/libxerces-c-3.3.so \ - /usr/local/cloudberry-db/lib || exit 3 +if [[ "$OS_ID" == "rocky" && "$OS_VERSION" =~ ^(8|9) ]]; then + execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ + /usr/local/xerces-c/lib/libxerces-c-3.3.so \ + /usr/local/cloudberry-db/lib || exit 3 +fi + execute_cmd sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db || exit 2 log_section_end "Initial Setup" @@ -120,7 +127,6 @@ CONFIGURE_DEBUG_OPTS="" if [ "${ENABLE_DEBUG:-false}" = "true" ]; then CONFIGURE_DEBUG_OPTS="--enable-debug \ - --enable-profiling \ --enable-cassert \ --enable-debug-extensions" fi @@ -134,7 +140,7 @@ execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ --enable-mapreduce \ --enable-orafce \ --enable-orca \ - --enable-pax \ + --enable-pax \ --enable-pxf \ --enable-tap-tests \ ${CONFIGURE_DEBUG_OPTS} \ diff --git a/images/docker/cbdb/build/ubuntu22.04/Dockerfile b/images/docker/cbdb/build/ubuntu22.04/Dockerfile index ee57e95..c157d1a 100644 --- a/images/docker/cbdb/build/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/build/ubuntu22.04/Dockerfile @@ -65,112 +65,74 @@ FROM ubuntu:22.04 # Argument for configuring the timezone ARG TIMEZONE_VAR="Europe/London" -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin +ENV TZ=${TIMEZONE_VAR} ENV DEBIAN_FRONTEND=noninteractive -ENV DEBCONF_NOWARNINGS="yes" - -RUN stat -fc %T /sys/fs/cgroup/ - -RUN ln -snf /usr/share/zoneinfo/${TIMEZONE_VAR} /etc/localtime && echo "${TIMEZONE_VAR}" > /etc/timezone # -------------------------------------------------------------------- # Install Development Tools and Utilities # -------------------------------------------------------------------- -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - gnupg \ - openssl \ - debhelper \ - libfakeroot \ - debootstrap \ - devscripts \ - make \ - equivs \ - bison \ - ca-certificates-java \ - ca-certificates \ - cmake \ - curl \ - htop \ - cgroup-tools \ - flex \ - gcc-11 \ - g++-11 \ - g++-11-multilib \ - git \ - krb5-multidev \ - libprotobuf-dev \ - protobuf-compiler \ - libapr1-dev \ - libbz2-dev \ - libcurl4-gnutls-dev \ - libpstreams-dev \ - libxerces-c-dev \ - libsystemd-dev \ - libevent-dev \ - libkrb5-dev \ - libldap2-dev \ - libperl-dev \ - libreadline-dev \ - libssl-dev \ - libxml2-dev \ - libyaml-dev \ - libzstd-dev \ - libaprutil1-dev \ - libpam0g-dev \ - libpam0g \ - libcgroup1 \ - libyaml-0-2 \ - libldap-2.5-0 \ - libssl3 \ - ninja-build \ - quilt \ - unzip \ - wget \ - zlib1g-dev \ - libuv1-dev \ - libgpgme-dev \ - libgpgme11 \ - python2.7 \ - python2.7-dev \ - pkg-config \ - python3.10 \ - python3.10-dev \ - python3-distutils \ - python3-pip \ - python3-setuptools \ - sudo \ - liblz4-dev \ - iputils-ping \ - iproute2 \ - rsync \ - less \ - openssh-client \ - openssh-server \ - lsof \ - apt-utils \ - silversearcher-ag \ - net-tools \ - python-six \ - tzdata \ - vim -RUN apt-get install -y locales && \ +RUN sed -i "s/archive.ubuntu.com/mirror.yandex.ru/g" /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y -qq \ + htop \ + bat \ + silversearcher-ag \ + vim \ + wget && \ + apt-get install -y -qq locales && \ locale-gen "en_US.UTF-8" && \ - update-locale LC_ALL="en_US.UTF-8" - -RUN cd && \ - XERCES_LATEST_RELEASE=3.3.0 && \ - wget -nv "https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" && \ - echo "$(curl -sL https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz.sha256)" | sha256sum -c - && \ - tar xf "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz"; rm "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" && \ - cd xerces-c-${XERCES_LATEST_RELEASE} && \ - ./configure --prefix=/usr/local/xerces-c && \ - make -j$(nproc) && \ - make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} && \ - rm -rf ~/xerces-c* - -RUN cd && GO_VERSION="go1.23.4" && \ + update-locale LANG="en_US.UTF-8" && \ + apt-get install -y -qq \ + bison \ + build-essential \ + cmake \ + dpkg-dev \ + fakeroot \ + flex \ + g++-11 \ + gcc-11 \ + git \ + iproute2 \ + iputils-ping \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libipc-run-perl \ + libkrb5-dev \ + libldap-dev \ + liblz4-dev \ + libpam0g-dev \ + libperl-dev \ + libprotobuf-dev \ + libreadline-dev \ + libssl-dev \ + libuv1-dev \ + libxerces-c-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + lsof \ + make \ + openssh-server \ + pkg-config \ + protobuf-compiler \ + python3-distutils \ + python3-pip \ + python3-setuptools \ + python3.10 \ + python3.10-dev \ + rsync \ + sudo \ + tzdata \ + zlib1g-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + cd && GO_VERSION="go1.23.4" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ @@ -190,34 +152,35 @@ RUN cd && GO_VERSION="go1.23.4" && \ rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ + update-alternatives --install /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/gcc-11 100 && \ + update-alternatives --set gcc /usr/bin/gcc-11 && \ + update-alternatives --set g++ /usr/bin/g++-11 # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment # -------------------------------------------------------------------- -RUN ln -s /usr/bin/python3.10 /usr/bin/python - COPY ./configs/* /tmp/ RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ chmod 755 /tmp/init_system.sh && \ - groupadd gpadmin && \ - useradd -rm -d /home/gpadmin -s /bin/bash -g gpadmin -G sudo -G root -u 1001 gpadmin && \ - echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers && \ - mkdir -p /run/sshd && chmod 0755 /run/sshd - -RUN apt-get install -y libhyperic-sigar-java - -# Clean up APT when done. -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/tmp/* + /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd -m -g gpadmin gpadmin && \ + echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/90-gpadmin && \ + chmod 0440 /etc/sudoers.d/90-gpadmin && \ + ssh-keygen -A && \ + mkdir /var/run/sshd && chmod 0755 /var/run/sshd # Install testinfra via pip RUN pip3 install pytest-testinfra - + # Example: Copying test files into the container COPY tests /tests -ENV USER=gpadmin USER gpadmin WORKDIR /home/gpadmin From 55c129d9720dd0cd2e9b0103897671fa5d29973f Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Fri, 6 Jun 2025 14:08:30 +0000 Subject: [PATCH 19/29] Add go to path --- images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh b/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh index d8c4a00..52a4446 100755 --- a/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh +++ b/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh @@ -183,6 +183,9 @@ Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total EOF +# Add go to PATH +source /etc/profile.d/go.sh + # -------------------------------------------------------------------- # Start an interactive bash shell # -------------------------------------------------------------------- From 19a6754ff993ec92085307302e134e66d6ba8a1f Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Fri, 6 Jun 2025 15:21:20 +0000 Subject: [PATCH 20/29] Change docker key --- .github/workflows/docker-cbdb-build-containers.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 214cc68..ee66f5a 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -113,14 +113,13 @@ jobs: if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} uses: docker/setup-qemu-action@v3 - # Login to DockerHub for pushing images - # Requires DOCKERHUB_USER and DOCKERHUB_TOKEN secrets to be set - - name: Login to Docker Hub - if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} + # Login to GitHub Container Registry for pushing images + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} # Setup Docker Buildx for efficient builds # Enable debug mode for better troubleshooting From bfd7d3cbaf352934232b377bac9307b9fe63bd66 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Fri, 6 Jun 2025 15:41:39 +0000 Subject: [PATCH 21/29] Push to ghcr.io registry --- .github/workflows/docker-cbdb-build-containers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index ee66f5a..dee8ff5 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -172,8 +172,8 @@ jobs: platforms: linux/amd64,linux/arm64 # Tag with both latest and version-specific tags tags: | - apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest - apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} + ghcr.io/leborchuk:cbdb-build-${{ matrix.platform }}-latest + ghcr.io/leborchuk:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} # Add standard Open Container Initiative (OCI) labels labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} From 2db34e16ac9594284e9bc6b783029af5c96c31c5 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Tue, 10 Jun 2025 13:29:12 +0000 Subject: [PATCH 22/29] Remove excessive dependencies --- images/docker/cbdb/build/ubuntu22.04/Dockerfile | 10 ++++++++++ packaging/deb/ubuntu22.04/debian/control | 5 +---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/images/docker/cbdb/build/ubuntu22.04/Dockerfile b/images/docker/cbdb/build/ubuntu22.04/Dockerfile index c157d1a..bb68496 100644 --- a/images/docker/cbdb/build/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/build/ubuntu22.04/Dockerfile @@ -131,6 +131,16 @@ RUN sed -i "s/archive.ubuntu.com/mirror.yandex.ru/g" /etc/apt/sources.list && \ sudo \ tzdata \ zlib1g-dev && \ + apt-get install -y -qq \ + ca-certificates-java \ + cgroup-tools \ + curl \ + debhelper \ + libaprutil1-dev \ + libcgroup1 \ + ninja-build \ + quilt \ + unzip && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ cd && GO_VERSION="go1.23.4" && \ ARCH=$(uname -m) && \ diff --git a/packaging/deb/ubuntu22.04/debian/control b/packaging/deb/ubuntu22.04/debian/control index 37af9f0..6087f3f 100644 --- a/packaging/deb/ubuntu22.04/debian/control +++ b/packaging/deb/ubuntu22.04/debian/control @@ -11,7 +11,6 @@ Build-Depends: debhelper (>= 9), flex, gcc-11, g++-11, - g++-11-multilib, git, krb5-multidev, libapr1-dev, @@ -34,7 +33,6 @@ Build-Depends: debhelper (>= 9), libldap-2.5-0, libssl3, ninja-build, - python2.7-dev, python-setuptools, quilt, unzip, @@ -67,10 +65,9 @@ Depends: curl, openssh-client, openssh-server, openssl, + python-six, python2.7, python2.7-dev, - python-pip, - python-six, rsync, wget, zlib1g, From 8549e40a9c506c51218fccf30b4416e756cfa007 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Tue, 10 Jun 2025 14:37:56 +0000 Subject: [PATCH 23/29] Tru to set correct repository in tag --- .github/workflows/docker-cbdb-build-containers.yml | 14 +++++++------- .github/workflows/docker-cbdb-test-containers.yml | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index dee8ff5..b098184 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -172,8 +172,8 @@ jobs: platforms: linux/amd64,linux/arm64 # Tag with both latest and version-specific tags tags: | - ghcr.io/leborchuk:cbdb-build-${{ matrix.platform }}-latest - ghcr.io/leborchuk:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} + ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest + ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} # Add standard Open Container Initiative (OCI) labels labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} @@ -196,15 +196,15 @@ jobs: echo "- **Build Date**: ${{ steps.version.outputs.BUILD_DATE }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY - echo "- Latest tag: \`apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest\`" >> $GITHUB_STEP_SUMMARY - echo "- Version tag: \`apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Latest tag: \`ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest\`" >> $GITHUB_STEP_SUMMARY + echo "- Version tag: \`ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }}\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### 📋 Quick Reference" >> $GITHUB_STEP_SUMMARY echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY echo "# Pull the image (automatically selects correct architecture)" >> $GITHUB_STEP_SUMMARY - echo "docker pull apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "# Pull specific architecture if needed" >> $GITHUB_STEP_SUMMARY - echo "docker pull --platform linux/amd64 apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY - echo "docker pull --platform linux/arm64 apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull --platform linux/amd64 ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull --platform linux/arm64 ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 37422f0..e077b0a 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -147,8 +147,8 @@ jobs: type=gha,scope=docker-cbdb-test-${{ matrix.platform }} # Tag with both latest and version-specific tags tags: | - apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest - apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} + ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest + ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} # Add metadata labels for better image tracking labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} @@ -171,15 +171,15 @@ jobs: echo "- **Build Date**: ${{ steps.version.outputs.BUILD_DATE }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY - echo "- Latest tag: \`apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest\`" >> $GITHUB_STEP_SUMMARY - echo "- Version tag: \`apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Latest tag: \`ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest\`" >> $GITHUB_STEP_SUMMARY + echo "- Version tag: \`ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }}\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### 📋 Quick Reference" >> $GITHUB_STEP_SUMMARY echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY echo "# Pull the image (automatically selects correct architecture)" >> $GITHUB_STEP_SUMMARY - echo "docker pull apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "# Pull specific architecture if needed" >> $GITHUB_STEP_SUMMARY - echo "docker pull --platform linux/amd64 apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY - echo "docker pull --platform linux/arm64 apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull --platform linux/amd64 ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull --platform linux/arm64 ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY From a4bf28859333c06e61346b24a819695b6ffd14ab Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Fri, 13 Jun 2025 20:42:47 +0000 Subject: [PATCH 24/29] Add libxerces to dependencies --- packaging/deb/ubuntu22.04/debian/control | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/deb/ubuntu22.04/debian/control b/packaging/deb/ubuntu22.04/debian/control index 6087f3f..aa87347 100644 --- a/packaging/deb/ubuntu22.04/debian/control +++ b/packaging/deb/ubuntu22.04/debian/control @@ -32,6 +32,8 @@ Build-Depends: debhelper (>= 9), libyaml-0-2, libldap-2.5-0, libssl3, + libxerces-c-dev, + libxerces-c3.2, ninja-build, python-setuptools, quilt, @@ -60,6 +62,7 @@ Depends: curl, libcgroup1, libssl3, libpam0g, + libxerces-c3.2, locales, net-tools, openssh-client, From f06fa77c06ee5be96cd91769a1a3d9ef91c7ef82 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Sat, 14 Jun 2025 19:21:54 +0000 Subject: [PATCH 25/29] Do not need python-setuptools to build deb --- packaging/deb/ubuntu22.04/debian/control | 1 - 1 file changed, 1 deletion(-) diff --git a/packaging/deb/ubuntu22.04/debian/control b/packaging/deb/ubuntu22.04/debian/control index aa87347..0689108 100644 --- a/packaging/deb/ubuntu22.04/debian/control +++ b/packaging/deb/ubuntu22.04/debian/control @@ -35,7 +35,6 @@ Build-Depends: debhelper (>= 9), libxerces-c-dev, libxerces-c3.2, ninja-build, - python-setuptools, quilt, unzip, wget, From 9d4400b3a12e4c6142e9b416966ff1dc92b15fb3 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Mon, 16 Jun 2025 03:27:46 +0000 Subject: [PATCH 26/29] Move back to incubator-cloudberry docker --- .../docker-cbdb-build-containers.yml | 19 +++++++++---------- packaging/deb/ubuntu22.04/debian/postinst | 1 + 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index b098184..1e9ea27 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -117,9 +117,8 @@ jobs: - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} # Setup Docker Buildx for efficient builds # Enable debug mode for better troubleshooting @@ -172,8 +171,8 @@ jobs: platforms: linux/amd64,linux/arm64 # Tag with both latest and version-specific tags tags: | - ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest - ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} + apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest + apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} # Add standard Open Container Initiative (OCI) labels labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} @@ -196,15 +195,15 @@ jobs: echo "- **Build Date**: ${{ steps.version.outputs.BUILD_DATE }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY - echo "- Latest tag: \`ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest\`" >> $GITHUB_STEP_SUMMARY - echo "- Version tag: \`ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Latest tag: \`apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest\`" >> $GITHUB_STEP_SUMMARY + echo "- Version tag: \`apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }}\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### 📋 Quick Reference" >> $GITHUB_STEP_SUMMARY echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY echo "# Pull the image (automatically selects correct architecture)" >> $GITHUB_STEP_SUMMARY - echo "docker pull ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "# Pull specific architecture if needed" >> $GITHUB_STEP_SUMMARY - echo "docker pull --platform linux/amd64 ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY - echo "docker pull --platform linux/arm64 ghcr.io/${{ github.repository }}:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull --platform linux/amd64 apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull --platform linux/arm64 apache/incubator-cloudberry:cbdb-build-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/packaging/deb/ubuntu22.04/debian/postinst b/packaging/deb/ubuntu22.04/debian/postinst index 2ecc307..e9b8619 100644 --- a/packaging/deb/ubuntu22.04/debian/postinst +++ b/packaging/deb/ubuntu22.04/debian/postinst @@ -10,5 +10,6 @@ if [ "$1" = configure ]; then chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} ln -s ${GPHOME} /usr/local/cloudberry-db + ln -s ${GPHOME} /usr/local/cloudberry fi From c49b173da9a8721639c6d50dbc9ea2bfe2d44b28 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Wed, 18 Jun 2025 10:13:05 +0000 Subject: [PATCH 27/29] Revert registry to apache incubator --- .github/workflows/docker-cbdb-build-containers.yml | 6 ++++-- .github/workflows/docker-cbdb-test-containers.yml | 14 +++++++------- images/docker/cbdb/build/ubuntu22.04/Dockerfile | 3 +-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 1e9ea27..214cc68 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -113,8 +113,10 @@ jobs: if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} uses: docker/setup-qemu-action@v3 - # Login to GitHub Container Registry for pushing images - - name: Login to GitHub Container Registry + # Login to DockerHub for pushing images + # Requires DOCKERHUB_USER and DOCKERHUB_TOKEN secrets to be set + - name: Login to Docker Hub + if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USER }} diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index e077b0a..37422f0 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -147,8 +147,8 @@ jobs: type=gha,scope=docker-cbdb-test-${{ matrix.platform }} # Tag with both latest and version-specific tags tags: | - ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest - ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} + apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest + apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }} # Add metadata labels for better image tracking labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} @@ -171,15 +171,15 @@ jobs: echo "- **Build Date**: ${{ steps.version.outputs.BUILD_DATE }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY - echo "- Latest tag: \`ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest\`" >> $GITHUB_STEP_SUMMARY - echo "- Version tag: \`ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Latest tag: \`apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest\`" >> $GITHUB_STEP_SUMMARY + echo "- Version tag: \`apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-${{ steps.version.outputs.BUILD_DATE }}-${{ steps.version.outputs.SHA_SHORT }}\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### 📋 Quick Reference" >> $GITHUB_STEP_SUMMARY echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY echo "# Pull the image (automatically selects correct architecture)" >> $GITHUB_STEP_SUMMARY - echo "docker pull ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "# Pull specific architecture if needed" >> $GITHUB_STEP_SUMMARY - echo "docker pull --platform linux/amd64 ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY - echo "docker pull --platform linux/arm64 ghcr.io/${{ github.repository }}:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull --platform linux/amd64 apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY + echo "docker pull --platform linux/arm64 apache/incubator-cloudberry:cbdb-test-${{ matrix.platform }}-latest" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/images/docker/cbdb/build/ubuntu22.04/Dockerfile b/images/docker/cbdb/build/ubuntu22.04/Dockerfile index bb68496..7a3abd6 100644 --- a/images/docker/cbdb/build/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/build/ubuntu22.04/Dockerfile @@ -76,8 +76,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Install Development Tools and Utilities # -------------------------------------------------------------------- -RUN sed -i "s/archive.ubuntu.com/mirror.yandex.ru/g" /etc/apt/sources.list && \ - apt-get update && \ +RUN apt-get update && \ apt-get install -y -qq \ htop \ bat \ From 90f06a6ebaad1d266b71b40940a7224b50a18e09 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Thu, 26 Jun 2025 08:32:02 +0000 Subject: [PATCH 28/29] Do not change rights to binary files --- packaging/deb/ubuntu22.04/debian/postinst | 2 -- 1 file changed, 2 deletions(-) diff --git a/packaging/deb/ubuntu22.04/debian/postinst b/packaging/deb/ubuntu22.04/debian/postinst index e9b8619..5382505 100644 --- a/packaging/deb/ubuntu22.04/debian/postinst +++ b/packaging/deb/ubuntu22.04/debian/postinst @@ -7,8 +7,6 @@ GPHOME=/usr/cloudberry-db if [ "$1" = configure ]; then - chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} - ln -s ${GPHOME} /usr/local/cloudberry-db ln -s ${GPHOME} /usr/local/cloudberry From f506a9ca52142361e778cd9cc3f0264fc2947d63 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Sat, 7 Jun 2025 08:51:09 +0800 Subject: [PATCH 29/29] Add jammy build to cloudberry-devops-release --- .../workflows/docker-cbdb-build-containers.yml | 2 +- .../workflows/docker-cbdb-test-containers.yml | 2 +- .../cloudberry/scripts/build-cloudberry.sh | 6 +++--- .../cloudberry/scripts/cloudberry-utils.sh | 16 +++------------- .../cloudberry/scripts/configure-cloudberry.sh | 13 ++++++------- .../scripts/create-cloudberry-demo-cluster.sh | 6 +++--- .../scripts/destroy-cloudberry-demo-cluster.sh | 2 +- .../cloudberry/scripts/test-cloudberry.sh | 2 +- .../cloudberry/scripts/unittest-cloudberry.sh | 4 ++-- images/docker/cbdb/build/rocky8/Dockerfile | 2 +- images/docker/cbdb/build/rocky9/Dockerfile | 2 +- images/docker/cbdb/test/rocky8/Dockerfile | 2 +- images/docker/cbdb/test/rocky9/Dockerfile | 2 +- images/docker/cbdb/test/ubuntu22.04/Dockerfile | 4 +++- packaging/deb/ubuntu22.04/debian/control | 2 +- packaging/deb/ubuntu22.04/debian/rules | 2 +- scripts/build-deb.sh | 2 +- 17 files changed, 31 insertions(+), 40 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 214cc68..a3e4dab 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -97,7 +97,7 @@ jobs: # This prevents unnecessary builds if only one platform was modified - name: Determine if platform changed id: platform-filter - uses: dorny/paths-filter@v3 + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 with: filters: | rocky8: diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 37422f0..c385817 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -81,7 +81,7 @@ jobs: # Determine if the current platform's files have changed - name: Determine if platform changed id: platform-filter - uses: dorny/paths-filter@v3 + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 with: filters: | rocky8: diff --git a/build_automation/cloudberry/scripts/build-cloudberry.sh b/build_automation/cloudberry/scripts/build-cloudberry.sh index 4ce4752..db04f1b 100755 --- a/build_automation/cloudberry/scripts/build-cloudberry.sh +++ b/build_automation/cloudberry/scripts/build-cloudberry.sh @@ -45,7 +45,7 @@ # Prerequisites: # - configure-cloudberry.sh must be run first # - Required build dependencies must be installed -# - ${BUILD_DESTINATION}/lib must exist and be writable +# - /usr/local/cloudberry-db/lib must exist and be writable # # Exit Codes: # 0 - Build and installation completed successfully @@ -67,11 +67,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" BUILD_LOG="${LOG_DIR}/build.log" # Initialize environment -init_environment "Cloudberry Build Script" "${BUILD_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Build Script" "${BUILD_LOG}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Build process diff --git a/build_automation/cloudberry/scripts/cloudberry-utils.sh b/build_automation/cloudberry/scripts/cloudberry-utils.sh index 683e166..344b7fd 100755 --- a/build_automation/cloudberry/scripts/cloudberry-utils.sh +++ b/build_automation/cloudberry/scripts/cloudberry-utils.sh @@ -30,12 +30,11 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Functions: -# init_environment "Script Name" "Log File" "Build Destination" +# init_environment "Script Name" "Log File" # - Initialize logging and verify environment # - Parameters: # * script_name: Name of the calling script # * log_file: Path to log file -# * build_destination: Path to Cloudberry destination, by default is /usr/local/cloudberry-db # - Returns: 0 on success, 1 on failure # # execute_cmd command [args...] @@ -71,7 +70,7 @@ # # Example: # source ./cloudberry-utils.sh -# init_environment "My Script" "${LOG_FILE}" "${BUILD_DESTINATION}" +# init_environment "My Script" "${LOG_FILE}" # execute_cmd make clean # log_section "Build Process" # execute_cmd make -j$(nproc) @@ -80,18 +79,10 @@ # # -------------------------------------------------------------------- -DEFAULT_BUILD_DESTINATION=/usr/local/cloudberry-db - # Initialize logging and environment init_environment() { local script_name=$1 local log_file=$2 - local build_destination=$3 - - if [ -z "$build_destination" ]; then - build_destination=${DEFAULT_BUILD_DESTINATION} - fi - export BUILD_DESTINATION=$build_destination echo "=== Initializing environment for ${script_name} ===" echo "${script_name} executed at $(date)" | tee -a "${log_file}" @@ -100,7 +91,6 @@ init_environment() { echo "Working directory: $(pwd)" | tee -a "${log_file}" echo "Source directory: ${SRC_DIR}" | tee -a "${log_file}" echo "Log directory: ${LOG_DIR}" | tee -a "${log_file}" - echo "Build destination: ${BUILD_DESTINATION}" | tee -a "${log_file}" if [ -z "${SRC_DIR:-}" ]; then echo "Error: SRC_DIR environment variable is not set" | tee -a "${log_file}" @@ -131,7 +121,7 @@ run_psql_cmd() { # Function to source Cloudberry environment source_cloudberry_env() { echo "=== Sourcing Cloudberry environment ===" | tee -a "${LOG_DIR}/environment.log" - source /usr/local/cloudberry-db/greenplum_path.sh + source /usr/local/cloudberry-db/cloudberry-env.sh source ${SRC_DIR}/../cloudberry/gpAux/gpdemo/gpdemo-env.sh } diff --git a/build_automation/cloudberry/scripts/configure-cloudberry.sh b/build_automation/cloudberry/scripts/configure-cloudberry.sh index f3b39fb..58f6c97 100755 --- a/build_automation/cloudberry/scripts/configure-cloudberry.sh +++ b/build_automation/cloudberry/scripts/configure-cloudberry.sh @@ -23,7 +23,7 @@ # Description: Configures Apache Cloudberry build environment and runs # ./configure with optimized settings. Performs the # following: -# 1. Prepares ${BUILD_DESTINATION} directory +# 1. Prepares /usr/local/cloudberry-db directory # 2. Sets up library dependencies # 3. Configures build with required features enabled # @@ -101,13 +101,12 @@ export LOG_DIR="${SRC_DIR}/build-logs" CONFIGURE_LOG="${LOG_DIR}/configure.log" # Initialize environment -init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" # Initial setup log_section "Initial Setup" -execute_cmd sudo rm -rf ${BUILD_DESTINATION}/* || exit 2 +execute_cmd sudo rm -rf /usr/local/cloudberry-db || exit 2 execute_cmd sudo chmod a+w /usr/local || exit 2 - execute_cmd sudo mkdir -p ${BUILD_DESTINATION}/lib || exit 2 if [[ "$OS_ID" == "rocky" && "$OS_VERSION" =~ ^(8|9) ]]; then execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ @@ -120,7 +119,7 @@ log_section_end "Initial Setup" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Add debug options if ENABLE_DEBUG is set to "true" @@ -134,7 +133,7 @@ fi # Configure build log_section "Configure" -execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ +execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ --disable-external-fts \ --enable-gpcloud \ --enable-ic-proxy \ @@ -159,7 +158,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --with-openssl \ --with-uuid=e2fs \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=${BUILD_DESTINATION}/lib || exit 4 + --with-libraries=/usr/local/cloudberry-db/lib || exit 4 log_section_end "Configure" # Capture version information diff --git a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh index 88d51ef..61a72ee 100755 --- a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh @@ -35,7 +35,7 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Prerequisites: -# - Apache Cloudberry must be installed (${BUILD_DESTINATION}) +# - Apache Cloudberry must be installed (/usr/local/cloudberry-db) # - SSH must be configured for passwordless access to localhost # - User must have permissions to create cluster directories # - PostgreSQL client tools (psql) must be available @@ -75,11 +75,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/cluster.log" # Initialize environment -init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" # Setup environment log_section "Environment Setup" -source ${BUILD_DESTINATION}/greenplum_path.sh || exit 1 +source ${BUILD_DESTINATION}/cloudberry-env.sh || exit 1 log_section_end "Environment Setup" # Verify SSH access diff --git a/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh index ae2bdc0..3d4ce24 100755 --- a/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/destroy-cloudberry-demo-cluster.sh @@ -71,7 +71,7 @@ export LOG_DIR="${SRC_DIR}/build-logs" CLUSTER_LOG="${LOG_DIR}/destroy-cluster.log" # Initialize environment -init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" "${BUILD_DESTINATION}" +init_environment "Destroy Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" # Source Cloudberry environment log_section "Environment Setup" diff --git a/build_automation/cloudberry/scripts/test-cloudberry.sh b/build_automation/cloudberry/scripts/test-cloudberry.sh index e3ff951..411f16c 100755 --- a/build_automation/cloudberry/scripts/test-cloudberry.sh +++ b/build_automation/cloudberry/scripts/test-cloudberry.sh @@ -60,7 +60,7 @@ export LOG_DIR="build-logs" TEST_LOG="${LOG_DIR}/test.log" # Initialize environment -init_environment "Cloudberry Test Script" "${TEST_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Test Script" "${TEST_LOG}" # Source Cloudberry environment log_section "Environment Setup" diff --git a/build_automation/cloudberry/scripts/unittest-cloudberry.sh b/build_automation/cloudberry/scripts/unittest-cloudberry.sh index 7e2283e..f7bc120 100755 --- a/build_automation/cloudberry/scripts/unittest-cloudberry.sh +++ b/build_automation/cloudberry/scripts/unittest-cloudberry.sh @@ -52,11 +52,11 @@ export LOG_DIR="${SRC_DIR}/build-logs" UNITTEST_LOG="${LOG_DIR}/unittest.log" # Initialize environment -init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" "${BUILD_DESTINATION}" +init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Unittest process diff --git a/images/docker/cbdb/build/rocky8/Dockerfile b/images/docker/cbdb/build/rocky8/Dockerfile index c5e0a67..19d054c 100644 --- a/images/docker/cbdb/build/rocky8/Dockerfile +++ b/images/docker/cbdb/build/rocky8/Dockerfile @@ -186,7 +186,7 @@ RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ setcap cap_net_raw+ep /usr/bin/ping && \ echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ - echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cbdb/greenplum_path.sh ]; then\n source /usr/local/cbdb/greenplum_path.sh\nfi' >> /home/gpadmin/.bashrc && \ + echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cbdb/cloudberry-env.sh ]; then\n source /usr/local/cbdb/cloudberry-env.sh\nfi' >> /home/gpadmin/.bashrc && \ ssh-keygen -A && \ echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config && \ localedef -i en_US -f UTF-8 en_US.UTF-8 && \ diff --git a/images/docker/cbdb/build/rocky9/Dockerfile b/images/docker/cbdb/build/rocky9/Dockerfile index cd1abe7..1da729a 100644 --- a/images/docker/cbdb/build/rocky9/Dockerfile +++ b/images/docker/cbdb/build/rocky9/Dockerfile @@ -189,7 +189,7 @@ RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ setcap cap_net_raw+ep /usr/bin/ping && \ echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ - echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cbdb/greenplum_path.sh ]; then\n source /usr/local/cbdb/greenplum_path.sh\nfi' >> /home/gpadmin/.bashrc && \ + echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cbdb/cloudberry-env.sh ]; then\n source /usr/local/cbdb/cloudberry-env.sh\nfi' >> /home/gpadmin/.bashrc && \ ssh-keygen -A && \ echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config && \ localedef -i en_US -f UTF-8 en_US.UTF-8 && \ diff --git a/images/docker/cbdb/test/rocky8/Dockerfile b/images/docker/cbdb/test/rocky8/Dockerfile index 85f8553..0d19026 100644 --- a/images/docker/cbdb/test/rocky8/Dockerfile +++ b/images/docker/cbdb/test/rocky8/Dockerfile @@ -100,7 +100,7 @@ RUN dnf install -y \ RUN /usr/sbin/groupadd gpadmin && \ /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ - echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/greenplum_path.sh ]; then\n source /usr/local/cloudberry/greenplum_path.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc + echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/cloudberry-env.sh ]; then\n source /usr/local/cloudberry/cloudberry-env.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment diff --git a/images/docker/cbdb/test/rocky9/Dockerfile b/images/docker/cbdb/test/rocky9/Dockerfile index 260d219..245cf91 100644 --- a/images/docker/cbdb/test/rocky9/Dockerfile +++ b/images/docker/cbdb/test/rocky9/Dockerfile @@ -100,7 +100,7 @@ RUN dnf install -y \ RUN /usr/sbin/groupadd gpadmin && \ /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ - echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/greenplum_path.sh ]; then\n source /usr/local/cloudberry/greenplum_path.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc + echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/cloudberry-env.sh ]; then\n source /usr/local/cloudberry/cloudberry-env.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile index f9d160b..2a44c7b 100644 --- a/images/docker/cbdb/test/ubuntu22.04/Dockerfile +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -1,3 +1,4 @@ + # -------------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -115,7 +116,7 @@ RUN /usr/sbin/groupadd gpadmin && \ /usr/sbin/useradd -m -g gpadmin gpadmin && \ echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ chmod 0440 /etc/sudoers.d/90-gpadmin && \ - echo '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/greenplum_path.sh ]; then\n source /usr/local/cloudberry/greenplum_path.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc + echo '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/cloudberry-env.sh ]; then\n source /usr/local/cloudberry/cloudberry-env.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc # -------------------------------------------------------------------- # Copy Configuration Files and Setup the Environment @@ -137,3 +138,4 @@ USER gpadmin WORKDIR /home/gpadmin CMD ["bash","-c","/tmp/init_system.sh"] + diff --git a/packaging/deb/ubuntu22.04/debian/control b/packaging/deb/ubuntu22.04/debian/control index 0689108..c635c53 100644 --- a/packaging/deb/ubuntu22.04/debian/control +++ b/packaging/deb/ubuntu22.04/debian/control @@ -1,5 +1,5 @@ Source: apache-cloudberry-db-incubating -Maintainer: Apache Cloudberry (Incubating) https://cloudberry.apache.org +Maintainer: Apache Cloudberry (Incubating) Section: database Build-Depends: debhelper (>= 9), bison, diff --git a/packaging/deb/ubuntu22.04/debian/rules b/packaging/deb/ubuntu22.04/debian/rules index 518bcf7..6213985 100755 --- a/packaging/deb/ubuntu22.04/debian/rules +++ b/packaging/deb/ubuntu22.04/debian/rules @@ -24,7 +24,7 @@ gpinstall: override_dh_auto_install: gpinstall # the staging directory for creating a debian is NOT the right GPHOME. # change GPHOME to point to the post-install target install directory. - sed -i "s#GPHOME=.*#GPHOME=${CBDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/greenplum_path.sh + sed -i "s#GPHOME=.*#GPHOME=${CBDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/cloudberry-env.sh override_dh_auto_build: echo "Skipping build" diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh index 4dbff91..05b29d5 100755 --- a/scripts/build-deb.sh +++ b/scripts/build-deb.sh @@ -64,7 +64,7 @@ apache-cloudberry-db-incubating (${CBDB_PKG_VERSION}) stable; urgency=low * apache-cloudberry-db autobuild - -- ${BUILD_USER} <${BUILD_USER}@$(hostname)> $(date +'%a, %d %b %Y %H:%M:%S %z') + -- ${BUILD_USER} <${BUILD_USER}@$(hostname)> $(date +'%a, %d %b %Y %H:%M:%S %z') EOF }