diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000000..2f786ac8eef05 --- /dev/null +++ b/.clang-format @@ -0,0 +1,71 @@ +# the official .clang-format style for https://github.com/taocpp +# +# clang-format-4.0 -i -style=file $(find -name '[^.]*.[hc]pp') + +Language: Cpp +Standard: Cpp11 + +AccessModifierOffset: -3 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlinesLeft: false +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: true + AfterControlStatement: false + AfterEnum : true + AfterFunction : true + AfterNamespace : true + AfterStruct : true + AfterUnion : true + BeforeCatch : true + BeforeElse : true + IndentBraces : false +BreakBeforeBinaryOperators: All +BreakBeforeBraces: Custom +BreakBeforeTernaryOperators: false +BreakStringLiterals: false +BreakConstructorInitializersBeforeComma: false +ColumnLimit: 0 +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 3 +ContinuationIndentWidth: 3 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +IndentCaseLabels: true +IndentWidth: 3 +IndentWrappedFunctionNames: false +KeepEmptyLinesAtTheStartOfBlocks: true +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: All +PointerAlignment: Left +ReflowComments: false +SortIncludes: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: Never +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: true +SpacesInCStyleCastParentheses: false +SpacesInContainerLiterals: true +SpacesInParentheses: true +SpacesInSquareBrackets: true +TabWidth: 8 +UseTab: Never diff --git a/.clangd b/.clangd new file mode 100644 index 0000000000000..cdfac3cf93416 --- /dev/null +++ b/.clangd @@ -0,0 +1,3 @@ +CompileFlags: + CompilationDatabase: build/ # Search build/ directory for compile_commands.json + Remove: -Werror diff --git a/.envrc b/.envrc new file mode 100644 index 0000000000000..f5b49c32db284 --- /dev/null +++ b/.envrc @@ -0,0 +1,18 @@ +# Bootstrap nix-direnv (ensures specific version, robust) +if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4=" +fi + +# Watch flake.nix for changes to reload the environment +watch_file flake.nix + +# Use the Nix flake to provide the development environment +use flake + +# Set consistent locale for development environment +#export LANGUAGE=C.utf8 +#export LANG=C.utf8 +#export LC_ALL=C.utf8 + +# IKOS-specific environment variable (keep if needed for your IKOS workflow) +export IKOS_SCAN_NOTIFIER_FILES="" diff --git a/.gitignore b/.gitignore index 4e911395fe3ba..8e429d66ca41f 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,11 @@ lib*.pc /Release/ /tmp_install/ /portlock/ + +build/ +install/ +test-db/ +.direnv/ +.cache/ +.history + diff --git a/.history b/.history new file mode 100644 index 0000000000000..472b0b7cab75a --- /dev/null +++ b/.history @@ -0,0 +1,39 @@ +nix develop +pg-build +pg-indent-changed-files +pg-build +pg-indent-changed-files +pg-build +pg-indent-changed-files +vi src/backend/nodes/bitmapset.c +rg pg_popcount +pg-indent-changed-files +fg +cd ws/postgresql/ +nix develop +cd ws/postgresql/ +git st +nix develop +git switch -c sparsemap +vi ../pg-patch-sets/v5-0001-dev-setup.patch +git am --keep-cr ../pg-patch-sets/v5-0001-dev-setup.patch +nix develop +git co master +git st +git co .history +git co master +git pull +git switch rm-freelist +git rebase origin/master +git branch +git co pg-heap-hot-updates-patch-v17 +git rebase origin/master +git branch -m pg-heap-hot-updates-patch-v17 heap-hot-updates +git co heap-hot-updates +tig +git rebase origin/master +ls +git switch rm-freelist +clion +cd ws/postgresql/ +nix develop diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000000000..13566b81b018a --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/editor.xml b/.idea/editor.xml new file mode 100644 index 0000000000000..1f0ef49b4faf4 --- /dev/null +++ b/.idea/editor.xml @@ -0,0 +1,580 @@ + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000000000..9c69411050eac --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000000000..53624c9e1f9ab --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,18 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/prettier.xml b/.idea/prettier.xml new file mode 100644 index 0000000000000..b0c1c68fbbad6 --- /dev/null +++ b/.idea/prettier.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000000000..35eb1ddfbbc02 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000000..f5d97424c5047 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "(gdb) Attach Postgres", + "type": "cppdbg", + "request": "attach", + "program": "${workspaceRoot}/install/bin/postgres", + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000..cc8a64fa9fa85 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "syscache.h": "c" + } +} \ No newline at end of file diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000000..41bcb32b86834 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1751211869, + "narHash": "sha256-1Cu92i1KSPbhPCKxoiVG5qnoRiKTgR5CcGSRyLpOd7Y=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b43c397f6c213918d6cfe6e3550abfe79b5d1c51", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-25.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000000..0b8a8eb086293 --- /dev/null +++ b/flake.nix @@ -0,0 +1,33 @@ +{ + description = "PostgreSQL development environment"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + }; + + shellConfig = import ./shell.nix { inherit pkgs system; }; + + in { + devShells = { + default = shellConfig.devShell; + gcc = shellConfig.devShell; + clang = shellConfig.clangDevShell; + gcc-musl = shellConfig.muslDevShell; + clang-musl = shellConfig.clangMuslDevShell; + }; + + packages = { + inherit (shellConfig) gdbConfig flameGraphScript pgbenchScript; + }; + } + ); +} diff --git a/pg-aliases.sh b/pg-aliases.sh new file mode 100644 index 0000000000000..6f6b58f4d9dad --- /dev/null +++ b/pg-aliases.sh @@ -0,0 +1,293 @@ +# PostgreSQL Development Aliases + +# Build system management +pg_clean_for_compiler() { + local current_compiler="$(basename $CC)" + local build_dir="$PG_BUILD_DIR" + + if [ -f "$build_dir/compile_commands.json" ]; then + local last_compiler=$(grep -o '/[^/]*/bin/[gc]cc\|/[^/]*/bin/clang' "$build_dir/compile_commands.json" | head -1 | xargs basename 2>/dev/null || echo "unknown") + + if [ "$last_compiler" != "$current_compiler" ] && [ "$last_compiler" != "unknown" ]; then + echo "Detected compiler change from $last_compiler to $current_compiler" + echo "Cleaning build directory..." + rm -rf "$build_dir" + mkdir -p "$build_dir" + fi + fi + + mkdir -p "$build_dir" + echo "$current_compiler" > "$build_dir/.compiler_used" +} + +# Core PostgreSQL commands +alias pg-setup=' + if [ -z "$PERL_CORE_DIR" ]; then + echo "Error: Could not find perl CORE directory" >&2 + return 1 + fi + + pg_clean_for_compiler + + echo "=== PostgreSQL Build Configuration ===" + echo "Compiler: $CC" + echo "LLVM: $(llvm-config --version 2>/dev/null || echo 'disabled')" + echo "Source: $PG_SOURCE_DIR" + echo "Build: $PG_BUILD_DIR" + echo "Install: $PG_INSTALL_DIR" + echo "======================================" + # --fatal-meson-warnings + + env CFLAGS="-I$PERL_CORE_DIR $CFLAGS" \ + LDFLAGS="-L$PERL_CORE_DIR -lperl $LDFLAGS" \ + meson setup --reconfigure \ + -Doptimization=g \ + -Ddebug=true \ + -Db_sanitize=none \ + -Db_lundef=false \ + -Dlz4=enabled \ + -Dzstd=enabled \ + -Dllvm=disabled \ + -Dplperl=enabled \ + -Dplpython=enabled \ + -Dpltcl=enabled \ + -Dlibxml=enabled \ + -Duuid=e2fs \ + -Dlibxslt=enabled \ + -Dssl=openssl \ + -Dldap=disabled \ + -Dcassert=true \ + -Dtap_tests=enabled \ + -Dinjection_points=true \ + -Ddocs_pdf=enabled \ + -Ddocs_html_style=website \ + --prefix="$PG_INSTALL_DIR" \ + "$PG_BUILD_DIR" \ + "$PG_SOURCE_DIR"' + +alias pg-build='meson compile -C "$PG_BUILD_DIR"' +alias pg-install='meson install -C "$PG_BUILD_DIR"' +alias pg-test='meson test -q --print-errorlogs -C "$PG_BUILD_DIR"' +# Clean commands +alias pg-clean='ninja -C "$PG_BUILD_DIR" clean' +alias pg-full-clean='rm -rf "$PG_BUILD_DIR" "$PG_INSTALL_DIR" && echo "Build and install directories cleaned"' + +# Database management +alias pg-init='rm -rf "$PG_DATA_DIR" && "$PG_INSTALL_DIR/bin/initdb" --debug --no-clean "$PG_DATA_DIR"' +alias pg-start='"$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR" -k "$PG_DATA_DIR"' +alias pg-stop='pkill -f "postgres.*-D.*$PG_DATA_DIR" || true' +alias pg-restart='pg-stop && sleep 2 && pg-start' +alias pg-status='pgrep -f "postgres.*-D.*$PG_DATA_DIR" && echo "PostgreSQL is running" || echo "PostgreSQL is not running"' + +# Client connections +alias pg-psql='"$PG_INSTALL_DIR/bin/psql" -h "$PG_DATA_DIR" postgres' +alias pg-createdb='"$PG_INSTALL_DIR/bin/createdb" -h "$PG_DATA_DIR"' +alias pg-dropdb='"$PG_INSTALL_DIR/bin/dropdb" -h "$PG_DATA_DIR"' + +# Debugging +alias pg-debug-gdb='gdb -x "$GDBINIT" "$PG_INSTALL_DIR/bin/postgres"' +alias pg-debug-lldb='lldb "$PG_INSTALL_DIR/bin/postgres"' +alias pg-debug=' + if command -v gdb >/dev/null 2>&1; then + pg-debug-gdb + elif command -v lldb >/dev/null 2>&1; then + pg-debug-lldb + else + echo "No debugger available (gdb or lldb required)" + fi' + +# Attach to running process +alias pg-attach-gdb=' + PG_PID=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1) + if [ -n "$PG_PID" ]; then + echo "Attaching GDB to PostgreSQL process $PG_PID" + gdb -x "$GDBINIT" -p "$PG_PID" + else + echo "No PostgreSQL process found" + fi' + +alias pg-attach-lldb=' + PG_PID=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1) + if [ -n "$PG_PID" ]; then + echo "Attaching LLDB to PostgreSQL process $PG_PID" + lldb -p "$PG_PID" + else + echo "No PostgreSQL process found" + fi' + +alias pg-attach=' + if command -v gdb >/dev/null 2>&1; then + pg-attach-gdb + elif command -v lldb >/dev/null 2>&1; then + pg-attach-lldb + else + echo "No debugger available (gdb or lldb required)" + fi' + +# Performance profiling and analysis +alias pg-valgrind='valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all "$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR"' +alias pg-strace='strace -f -o /tmp/postgres.strace "$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR"' + +# Flame graph generation +alias pg-flame='pg-flame-generate' +alias pg-flame-30='pg-flame-generate 30' +alias pg-flame-60='pg-flame-generate 60' +alias pg-flame-120='pg-flame-generate 120' + +# Custom flame graph with specific duration and output +pg-flame-custom() { + local duration=${1:-30} + local output_dir=${2:-$PG_FLAME_DIR} + echo "Generating flame graph for ${duration}s, output to: $output_dir" + pg-flame-generate "$duration" "$output_dir" +} + +# Benchmarking with pgbench +alias pg-bench='pg-bench-run' +alias pg-bench-quick='pg-bench-run 5 1 100 1 30 select-only' +alias pg-bench-standard='pg-bench-run 10 2 1000 10 60 tpcb-like' +alias pg-bench-heavy='pg-bench-run 50 4 5000 100 300 tpcb-like' +alias pg-bench-readonly='pg-bench-run 20 4 2000 50 120 select-only' + +# Custom benchmark function +pg-bench-custom() { + local clients=${1:-10} + local threads=${2:-2} + local transactions=${3:-1000} + local scale=${4:-10} + local duration=${5:-60} + local test_type=${6:-tpcb-like} + + echo "Running custom benchmark:" + echo " Clients: $clients, Threads: $threads" + echo " Transactions: $transactions, Scale: $scale" + echo " Duration: ${duration}s, Type: $test_type" + + pg-bench-run "$clients" "$threads" "$transactions" "$scale" "$duration" "$test_type" +} + +# Benchmark with flame graph +pg-bench-flame() { + local duration=${1:-60} + local clients=${2:-10} + local scale=${3:-10} + + echo "Running benchmark with flame graph generation" + echo "Duration: ${duration}s, Clients: $clients, Scale: $scale" + + # Start benchmark in background + pg-bench-run "$clients" 2 1000 "$scale" "$duration" tpcb-like & + local bench_pid=$! + + # Wait a bit for benchmark to start + sleep 5 + + # Generate flame graph for most of the benchmark duration + local flame_duration=$((duration - 10)) + if [ $flame_duration -gt 10 ]; then + pg-flame-generate "$flame_duration" & + local flame_pid=$! + fi + + # Wait for benchmark to complete + wait $bench_pid + + # Wait for flame graph if it was started + if [ -n "${flame_pid:-}" ]; then + wait $flame_pid + fi + + echo "Benchmark and flame graph generation completed" +} + +# Performance monitoring +alias pg-perf='perf top -p $(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1)' +alias pg-htop='htop -p $(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | tr "\n" "," | sed "s/,$//")' + +# System performance stats during PostgreSQL operation +pg-stats() { + local duration=${1:-30} + echo "Collecting system stats for ${duration}s..." + + iostat -x 1 "$duration" > "$PG_BENCH_DIR/iostat_$(date +%Y%m%d_%H%M%S).log" & + vmstat 1 "$duration" > "$PG_BENCH_DIR/vmstat_$(date +%Y%m%d_%H%M%S).log" & + + wait + echo "System stats saved to $PG_BENCH_DIR" +} + +# Log management +alias pg-log='tail -f "$PG_DATA_DIR/log/postgresql-$(date +%Y-%m-%d).log" 2>/dev/null || echo "No log file found"' +alias pg-log-errors='grep -i error "$PG_DATA_DIR/log/"*.log 2>/dev/null || echo "No error logs found"' + +# Build logs +alias pg-build-log='cat "$PG_BUILD_DIR/meson-logs/meson-log.txt"' +alias pg-build-errors='grep -i error "$PG_BUILD_DIR/meson-logs/meson-log.txt" 2>/dev/null || echo "No build errors found"' + +# Development helpers +alias pg-format=' + if [ ! -f "$PG_SOURCE_DIR/src/tools/pgindent/pgindent" ]; then + echo "Error: pgindent not found at $PG_SOURCE_DIR/src/tools/pgindent/pgindent" + return 1 + fi + + modified_files=$(git diff --name-only HEAD | egrep "\.c$|\.h$") + + if [ -z "$modified_files" ]; then + echo "No modified .c or .h files found" + return 0 + fi + + echo "Formatting modified files with pgindent:" + for file in $modified_files; do + if [ -f "$file" ]; then + echo " Formatting: $file" + "$PG_SOURCE_DIR/src/tools/pgindent/pgindent" "$file" + else + echo " Warning: File not found: $file" + fi + done + + echo "Formatting completed"' + +alias pg-tidy='find "$PG_SOURCE_DIR" -name "*.c" | head -10 | xargs clang-tidy' + +# Results viewing +alias pg-bench-results='ls -la "$PG_BENCH_DIR" && echo "Latest results:" && tail -20 "$PG_BENCH_DIR"/results_*.txt 2>/dev/null | tail -20' +alias pg-flame-results='ls -la "$PG_FLAME_DIR" && echo "Open flame graphs with: firefox $PG_FLAME_DIR/*.svg"' + +# Clean up old results +pg-clean-results() { + local days=${1:-7} + echo "Cleaning benchmark and flame graph results older than $days days..." + find "$PG_BENCH_DIR" -type f -mtime +$days -delete 2>/dev/null || true + find "$PG_FLAME_DIR" -type f -mtime +$days -delete 2>/dev/null || true + echo "Cleanup completed" +} + +# Information +alias pg-info=' + echo "=== PostgreSQL Development Environment ===" + echo "Source: $PG_SOURCE_DIR" + echo "Build: $PG_BUILD_DIR" + echo "Install: $PG_INSTALL_DIR" + echo "Data: $PG_DATA_DIR" + echo "Benchmarks: $PG_BENCH_DIR" + echo "Flame graphs: $PG_FLAME_DIR" + echo "Compiler: $CC" + echo "" + echo "Available commands:" + echo " Setup: pg-setup, pg-build, pg-install" + echo " Database: pg-init, pg-start, pg-stop, pg-psql" + echo " Debug: pg-debug, pg-attach, pg-valgrind" + echo " Performance: pg-flame, pg-bench, pg-perf" + echo " Benchmarks: pg-bench-quick, pg-bench-standard, pg-bench-heavy" + echo " Flame graphs: pg-flame-30, pg-flame-60, pg-flame-custom" + echo " Combined: pg-bench-flame" + echo " Results: pg-bench-results, pg-flame-results" + echo " Logs: pg-log, pg-build-log" + echo " Clean: pg-clean, pg-full-clean, pg-clean-results" + echo " Code quality: pg-format, pg-tidy" + echo "=========================================="' + +echo "PostgreSQL aliases loaded. Run 'pg-info' for available commands." diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000000..94dc9d3d749a8 --- /dev/null +++ b/shell.nix @@ -0,0 +1,738 @@ +{ pkgs, system }: + +let + # Use LLVM 19 for modern PostgreSQL development + llvmPkgs = pkgs.llvmPackages_19; + + # Configuration constants + config = { + pgSourceDir = "$HOME/ws/postgresql"; + pgBuildDir = "$HOME/ws/postgresql/build"; + pgInstallDir = "$HOME/ws/postgresql/install"; + pgDataDir = "/tmp/test-db"; + pgBenchDir = "/tmp/pgbench-results"; + pgFlameDir = "/tmp/flame-graphs"; + }; + + # Single dependency function that can be used for all environments + getPostgreSQLDeps = muslLibs: with pkgs; [ + # Build system (always use host tools) + meson ninja pkg-config autoconf libtool git which + binutils gnumake + + # Parser/lexer tools + bison flex + + # Perl with required packages + (perl.withPackages (ps: with ps; [ IPCRun ])) + + # Documentation + docbook_xml_dtd_45 docbook-xsl-nons libxslt libxml2 fop + + # Development tools (always use host tools) + coreutils shellcheck ripgrep valgrind curl + gdb lldb strace ltrace + perf-tools linuxPackages.perf flamegraph + htop iotop sysstat + ccache clang-tools cppcheck + + # LLVM toolchain + llvmPkgs.llvm llvmPkgs.llvm.dev + + # Language support + (python3.withPackages (ps: with ps; [ requests browser-cookie3 ])) + tcl + ] ++ (if muslLibs then [ + # Musl target libraries for cross-compilation + pkgs.pkgsMusl.readline + pkgs.pkgsMusl.zlib + pkgs.pkgsMusl.openssl + pkgs.pkgsMusl.icu + pkgs.pkgsMusl.lz4 + pkgs.pkgsMusl.zstd + pkgs.pkgsMusl.libuuid + pkgs.pkgsMusl.libkrb5 + pkgs.pkgsMusl.linux-pam + pkgs.pkgsMusl.libxcrypt + ] else [ + # Glibc target libraries + readline zlib openssl icu lz4 zstd libuuid libkrb5 + linux-pam libxcrypt numactl openldap + liburing libselinux + glibc glibc.dev + ]); + + # GDB configuration for PostgreSQL debugging + gdbConfig = pkgs.writeText "gdbinit-postgres" '' + # PostgreSQL-specific GDB configuration + + # Pretty-print PostgreSQL data structures + define print_node + if $arg0 + printf "Node type: %s\n", nodeTagNames[$arg0->type] + print *$arg0 + else + printf "NULL node\n" + end + end + document print_node + Print a PostgreSQL Node with type information + Usage: print_node + end + + define print_list + set $list = (List*)$arg0 + if $list + printf "List length: %d\n", $list->length + set $cell = $list->head + set $i = 0 + while $cell && $i < $list->length + printf " [%d]: ", $i + print_node $cell->data.ptr_value + set $cell = $cell->next + set $i = $i + 1 + end + else + printf "NULL list\n" + end + end + document print_list + Print a PostgreSQL List structure + Usage: print_list + end + + define print_query + set $query = (Query*)$arg0 + if $query + printf "Query type: %d, command type: %d\n", $query->querySource, $query->commandType + print *$query + else + printf "NULL query\n" + end + end + document print_query + Print a PostgreSQL Query structure + Usage: print_query + end + + define print_relcache + set $rel = (Relation)$arg0 + if $rel + printf "Relation: %s.%s (OID: %u)\n", $rel->rd_rel->relnamespace, $rel->rd_rel->relname.data, $rel->rd_id + printf " natts: %d, relkind: %c\n", $rel->rd_rel->relnatts, $rel->rd_rel->relkind + else + printf "NULL relation\n" + end + end + document print_relcache + Print relation cache entry information + Usage: print_relcache + end + + define print_tupdesc + set $desc = (TupleDesc)$arg0 + if $desc + printf "TupleDesc: %d attributes\n", $desc->natts + set $i = 0 + while $i < $desc->natts + set $attr = $desc->attrs[$i] + printf " [%d]: %s (type: %u, len: %d)\n", $i, $attr->attname.data, $attr->atttypid, $attr->attlen + set $i = $i + 1 + end + else + printf "NULL tuple descriptor\n" + end + end + document print_tupdesc + Print tuple descriptor information + Usage: print_tupdesc + end + + define print_slot + set $slot = (TupleTableSlot*)$arg0 + if $slot + printf "TupleTableSlot: %s\n", $slot->tts_ops->name + printf " empty: %d, shouldFree: %d\n", $slot->tts_empty, $slot->tts_shouldFree + if $slot->tts_tupleDescriptor + print_tupdesc $slot->tts_tupleDescriptor + end + else + printf "NULL slot\n" + end + end + document print_slot + Print tuple table slot information + Usage: print_slot + end + + # Memory context debugging + define print_mcxt + set $context = (MemoryContext)$arg0 + if $context + printf "MemoryContext: %s\n", $context->name + printf " type: %s, parent: %p\n", $context->methods->name, $context->parent + printf " total: %zu, free: %zu\n", $context->mem_allocated, $context->freep - $context->freeptr + else + printf "NULL memory context\n" + end + end + document print_mcxt + Print memory context information + Usage: print_mcxt + end + + # Process debugging + define print_proc + set $proc = (PGPROC*)$arg0 + if $proc + printf "PGPROC: pid=%d, database=%u\n", $proc->pid, $proc->databaseId + printf " waiting: %d, waitStatus: %d\n", $proc->waiting, $proc->waitStatus + else + printf "NULL process\n" + end + end + document print_proc + Print process information + Usage: print_proc + end + + # Set useful defaults + set print pretty on + set print object on + set print static-members off + set print vtbl on + set print demangle on + set demangle-style gnu-v3 + set print sevenbit-strings off + set history save on + set history size 1000 + set history filename ~/.gdb_history_postgres + + # Common breakpoints for PostgreSQL debugging + define pg_break_common + break elog + break errfinish + break ExceptionalCondition + break ProcessInterrupts + end + document pg_break_common + Set common PostgreSQL debugging breakpoints + end + + printf "PostgreSQL GDB configuration loaded.\n" + printf "Available commands: print_node, print_list, print_query, print_relcache,\n" + printf " print_tupdesc, print_slot, print_mcxt, print_proc, pg_break_common\n" + ''; + + # Flame graph generation script + flameGraphScript = pkgs.writeScriptBin "pg-flame-generate" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + DURATION=''${1:-30} + OUTPUT_DIR=''${2:-${config.pgFlameDir}} + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + + mkdir -p "$OUTPUT_DIR" + + echo "Generating flame graph for PostgreSQL (duration: ''${DURATION}s)" + + # Find PostgreSQL processes + PG_PIDS=$(pgrep -f "postgres.*-D.*${config.pgDataDir}" || true) + + if [ -z "$PG_PIDS" ]; then + echo "Error: No PostgreSQL processes found" + exit 1 + fi + + echo "Found PostgreSQL processes: $PG_PIDS" + + # Record perf data + PERF_DATA="$OUTPUT_DIR/perf_$TIMESTAMP.data" + echo "Recording perf data to $PERF_DATA" + + ${pkgs.linuxPackages.perf}/bin/perf record \ + -F 997 \ + -g \ + --call-graph dwarf \ + -p "$(echo $PG_PIDS | tr ' ' ',')" \ + -o "$PERF_DATA" \ + sleep "$DURATION" + + # Generate flame graph + FLAME_SVG="$OUTPUT_DIR/postgres_flame_$TIMESTAMP.svg" + echo "Generating flame graph: $FLAME_SVG" + + ${pkgs.linuxPackages.perf}/bin/perf script -i "$PERF_DATA" | \ + ${pkgs.flamegraph}/bin/stackcollapse-perf.pl | \ + ${pkgs.flamegraph}/bin/flamegraph.pl \ + --title "PostgreSQL Flame Graph ($TIMESTAMP)" \ + --width 1200 \ + --height 800 \ + > "$FLAME_SVG" + + echo "Flame graph generated: $FLAME_SVG" + echo "Perf data saved: $PERF_DATA" + + # Generate summary report + REPORT="$OUTPUT_DIR/report_$TIMESTAMP.txt" + echo "Generating performance report: $REPORT" + + { + echo "PostgreSQL Performance Analysis Report" + echo "Generated: $(date)" + echo "Duration: ''${DURATION}s" + echo "Processes: $PG_PIDS" + echo "" + echo "=== Top Functions ===" + ${pkgs.linuxPackages.perf}/bin/perf report -i "$PERF_DATA" --stdio --sort comm,dso,symbol | head -50 + echo "" + echo "=== Call Graph ===" + ${pkgs.linuxPackages.perf}/bin/perf report -i "$PERF_DATA" --stdio -g --sort comm,dso,symbol | head -100 + } > "$REPORT" + + echo "Report generated: $REPORT" + echo "" + echo "Files created:" + echo " Flame graph: $FLAME_SVG" + echo " Perf data: $PERF_DATA" + echo " Report: $REPORT" + ''; + + # pgbench wrapper script + pgbenchScript = pkgs.writeScriptBin "pg-bench-run" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + # Default parameters + CLIENTS=''${1:-10} + THREADS=''${2:-2} + TRANSACTIONS=''${3:-1000} + SCALE=''${4:-10} + DURATION=''${5:-60} + TEST_TYPE=''${6:-tpcb-like} + + OUTPUT_DIR="${config.pgBenchDir}" + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + + mkdir -p "$OUTPUT_DIR" + + echo "=== PostgreSQL Benchmark Configuration ===" + echo "Clients: $CLIENTS" + echo "Threads: $THREADS" + echo "Transactions: $TRANSACTIONS" + echo "Scale factor: $SCALE" + echo "Duration: ''${DURATION}s" + echo "Test type: $TEST_TYPE" + echo "Output directory: $OUTPUT_DIR" + echo "============================================" + + # Check if PostgreSQL is running + if ! pgrep -f "postgres.*-D.*${config.pgDataDir}" >/dev/null; then + echo "Error: PostgreSQL is not running. Start it with 'pg-start'" + exit 1 + fi + + PGBENCH="${config.pgInstallDir}/bin/pgbench" + PSQL="${config.pgInstallDir}/bin/psql" + CREATEDB="${config.pgInstallDir}/bin/createdb" + DROPDB="${config.pgInstallDir}/bin/dropdb" + + DB_NAME="pgbench_test_$TIMESTAMP" + RESULTS_FILE="$OUTPUT_DIR/results_$TIMESTAMP.txt" + LOG_FILE="$OUTPUT_DIR/pgbench_$TIMESTAMP.log" + + echo "Creating test database: $DB_NAME" + "$CREATEDB" -h "${config.pgDataDir}" "$DB_NAME" || { + echo "Failed to create database" + exit 1 + } + + # Initialize pgbench tables + echo "Initializing pgbench tables (scale factor: $SCALE)" + "$PGBENCH" -h "${config.pgDataDir}" -i -s "$SCALE" "$DB_NAME" || { + echo "Failed to initialize pgbench tables" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + exit 1 + } + + # Run benchmark based on test type + echo "Running benchmark..." + + case "$TEST_TYPE" in + "tpcb-like"|"default") + BENCH_ARGS="" + ;; + "select-only") + BENCH_ARGS="-S" + ;; + "simple-update") + BENCH_ARGS="-N" + ;; + "read-write") + BENCH_ARGS="-b select-only@70 -b tpcb-like@30" + ;; + *) + echo "Unknown test type: $TEST_TYPE" + echo "Available types: tpcb-like, select-only, simple-update, read-write" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + exit 1 + ;; + esac + + { + echo "PostgreSQL Benchmark Results" + echo "Generated: $(date)" + echo "Test type: $TEST_TYPE" + echo "Clients: $CLIENTS, Threads: $THREADS" + echo "Transactions: $TRANSACTIONS, Duration: ''${DURATION}s" + echo "Scale factor: $SCALE" + echo "Database: $DB_NAME" + echo "" + echo "=== System Information ===" + echo "CPU: $(nproc) cores" + echo "Memory: $(free -h | grep '^Mem:' | awk '{print $2}')" + echo "Compiler: $CC" + echo "PostgreSQL version: $("$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -t -c "SELECT version();" | head -1)" + echo "" + echo "=== Benchmark Results ===" + } > "$RESULTS_FILE" + + # Run the actual benchmark + "$PGBENCH" \ + -h "${config.pgDataDir}" \ + -c "$CLIENTS" \ + -j "$THREADS" \ + -T "$DURATION" \ + -P 5 \ + --log \ + --log-prefix="$OUTPUT_DIR/pgbench_$TIMESTAMP" \ + $BENCH_ARGS \ + "$DB_NAME" 2>&1 | tee -a "$RESULTS_FILE" + + # Collect additional statistics + { + echo "" + echo "=== Database Statistics ===" + "$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -c " + SELECT + schemaname, + relname, + n_tup_ins as inserts, + n_tup_upd as updates, + n_tup_del as deletes, + n_live_tup as live_tuples, + n_dead_tup as dead_tuples + FROM pg_stat_user_tables; + " + + echo "" + echo "=== Index Statistics ===" + "$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -c " + SELECT + schemaname, + relname, + indexrelname, + idx_scan, + idx_tup_read, + idx_tup_fetch + FROM pg_stat_user_indexes; + " + } >> "$RESULTS_FILE" + + # Clean up + echo "Cleaning up test database: $DB_NAME" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + + echo "" + echo "Benchmark completed!" + echo "Results saved to: $RESULTS_FILE" + echo "Transaction logs: $OUTPUT_DIR/pgbench_$TIMESTAMP*" + + # Show summary + echo "" + echo "=== Quick Summary ===" + grep -E "(tps|latency)" "$RESULTS_FILE" | tail -5 + ''; + + # Development shell (GCC + glibc) + devShell = pkgs.mkShell { + name = "postgresql-dev"; + buildInputs = (getPostgreSQLDeps false) ++ [ + flameGraphScript + pgbenchScript + ]; + + shellHook = '' + # History configuration + export HISTFILE=.history + export HISTSIZE=1000000 + export HISTFILESIZE=1000000 + + # Clean environment + unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH + + # Essential tools in PATH + export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH" + + # Ccache configuration + export PATH=${pkgs.ccache}/bin:$PATH + export CCACHE_COMPILERCHECK=content + export CCACHE_DIR=$HOME/.ccache_pg_dev + mkdir -p "$CCACHE_DIR" + + # LLVM configuration + export LLVM_CONFIG="${llvmPkgs.llvm}/bin/llvm-config" + export PATH="${llvmPkgs.llvm}/bin:$PATH" + export PKG_CONFIG_PATH="${llvmPkgs.llvm.dev}/lib/pkgconfig:$PKG_CONFIG_PATH" + export LLVM_DIR="${llvmPkgs.llvm.dev}/lib/cmake/llvm" + export LLVM_ROOT="${llvmPkgs.llvm}" + + # Development tools in PATH + export PATH=${pkgs.clang-tools}/bin:$PATH + export PATH=${pkgs.cppcheck}/bin:$PATH + + # Development CFLAGS + # -DRELCACHE_FORCE_RELEASE -DCATCACHE_FORCE_RELEASE -fno-omit-frame-pointer -fno-stack-protector -DUSE_VALGRIND + export CFLAGS="" + export CXXFLAGS="" + + # GCC configuration (default compiler) + export CC="${pkgs.gcc}/bin/gcc" + export CXX="${pkgs.gcc}/bin/g++" + + # PostgreSQL environment + export PG_SOURCE_DIR="${config.pgSourceDir}" + export PG_BUILD_DIR="${config.pgBuildDir}" + export PG_INSTALL_DIR="${config.pgInstallDir}" + export PG_DATA_DIR="${config.pgDataDir}" + export PG_BENCH_DIR="${config.pgBenchDir}" + export PG_FLAME_DIR="${config.pgFlameDir}" + export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d) + + # GDB configuration + export GDBINIT="${gdbConfig}" + + # Performance tools in PATH + export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH" + + # Create output directories + mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR" + + # Compiler verification + echo "Environment configured:" + echo " Compiler: $CC" + echo " LibC: glibc" + echo " LLVM: $(llvm-config --version 2>/dev/null || echo 'not available')" + echo " Ccache: enabled ($CCACHE_DIR)" + + # Load PostgreSQL development aliases + if [ -f ./pg-aliases.sh ]; then + source ./pg-aliases.sh + else + echo "Warning: pg-aliases.sh not found in current directory" + fi + + echo "" + echo "PostgreSQL Development Environment Ready (GCC + glibc)" + echo "Run 'pg-info' for available commands" + ''; + }; + + # Clang + glibc variant + clangDevShell = pkgs.mkShell { + name = "postgresql-clang-glibc"; + buildInputs = (getPostgreSQLDeps false) ++ [ + llvmPkgs.clang + llvmPkgs.lld + llvmPkgs.compiler-rt + flameGraphScript + pgbenchScript + ]; + + shellHook = '' + export HISTFILE=.history + export HISTSIZE=1000000 + export HISTFILESIZE=1000000 + + unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH + + export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH" + + # Ccache configuration + export PATH=${pkgs.ccache}/bin:$PATH + export CCACHE_COMPILERCHECK=content + export CCACHE_DIR=$HOME/.ccache_pg_dev_clang + mkdir -p "$CCACHE_DIR" + + # LLVM configuration + export LLVM_CONFIG="${llvmPkgs.llvm}/bin/llvm-config" + export PATH="${llvmPkgs.llvm}/bin:$PATH" + export PKG_CONFIG_PATH="${llvmPkgs.llvm.dev}/lib/pkgconfig:$PKG_CONFIG_PATH" + export LLVM_DIR="${llvmPkgs.llvm.dev}/lib/cmake/llvm" + export LLVM_ROOT="${llvmPkgs.llvm}" + + # Development tools in PATH + export PATH=${pkgs.clang-tools}/bin:$PATH + export PATH=${pkgs.cppcheck}/bin:$PATH + + # Clang + glibc configuration - use system linker instead of LLD for compatibility + export CC="${llvmPkgs.clang}/bin/clang" + export CXX="${llvmPkgs.clang}/bin/clang++" + + # Use system linker and standard runtime + #export CFLAGS="" + #export CXXFLAGS="" + #export LDFLAGS="" + + # PostgreSQL environment + export PG_SOURCE_DIR="${config.pgSourceDir}" + export PG_BUILD_DIR="${config.pgBuildDir}-clang" + export PG_INSTALL_DIR="${config.pgInstallDir}-clang" + export PG_DATA_DIR="${config.pgDataDir}-clang" + export PG_BENCH_DIR="${config.pgBenchDir}" + export PG_FLAME_DIR="${config.pgFlameDir}" + export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d) + + # GDB configuration + export GDBINIT="${gdbConfig}" + + # Performance tools in PATH + export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH" + + # Create output directories + mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR" + + # Compiler verification + echo "Environment configured:" + echo " Compiler: $CC" + echo " LibC: glibc" + echo " LLVM: $(llvm-config --version 2>/dev/null || echo 'not available')" + echo " Ccache: enabled ($CCACHE_DIR)" + + # Load PostgreSQL development aliases + if [ -f ./pg-aliases.sh ]; then + source ./pg-aliases.sh + else + echo "Warning: pg-aliases.sh not found in current directory" + fi + + echo "" + echo "PostgreSQL Development Environment Ready (Clang + glibc)" + echo "Run 'pg-info' for available commands" + ''; + }; + + # GCC + musl variant (cross-compilation) + muslDevShell = pkgs.mkShell { + name = "postgresql-gcc-musl"; + buildInputs = (getPostgreSQLDeps true) ++ [ + pkgs.gcc + flameGraphScript + pgbenchScript + ]; + + shellHook = '' + # Same base configuration as main shell + export HISTFILE=.history + export HISTSIZE=1000000 + export HISTFILESIZE=1000000 + + unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH + + export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH" + + # Cross-compilation to musl + export CC="${pkgs.gcc}/bin/gcc" + export CXX="${pkgs.gcc}/bin/g++" + + # Point to musl libraries for linking + export PKG_CONFIG_PATH="${pkgs.pkgsMusl.openssl.dev}/lib/pkgconfig:${pkgs.pkgsMusl.zlib.dev}/lib/pkgconfig:${pkgs.pkgsMusl.icu.dev}/lib/pkgconfig" + export CFLAGS="-ggdb -Og -fno-omit-frame-pointer -DUSE_VALGRIND -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export CXXFLAGS="-ggdb -Og -fno-omit-frame-pointer -DUSE_VALGRIND -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export LDFLAGS="-L${pkgs.pkgsMusl.stdenv.cc.libc}/lib -static-libgcc" + + # PostgreSQL environment + export PG_SOURCE_DIR="${config.pgSourceDir}" + export PG_BUILD_DIR="${config.pgBuildDir}-musl" + export PG_INSTALL_DIR="${config.pgInstallDir}-musl" + export PG_DATA_DIR="${config.pgDataDir}-musl" + export PG_BENCH_DIR="${config.pgBenchDir}" + export PG_FLAME_DIR="${config.pgFlameDir}" + export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d) + + export GDBINIT="${gdbConfig}" + export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH" + + mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR" + + echo "GCC + musl environment configured" + echo " Compiler: $CC" + echo " LibC: musl (cross-compilation)" + + if [ -f ./pg-aliases.sh ]; then + source ./pg-aliases.sh + fi + + echo "PostgreSQL Development Environment Ready (GCC + musl)" + ''; + }; + + # Clang + musl variant (cross-compilation) + clangMuslDevShell = pkgs.mkShell { + name = "postgresql-clang-musl"; + buildInputs = (getPostgreSQLDeps true) ++ [ + llvmPkgs.clang + llvmPkgs.lld + flameGraphScript + pgbenchScript + ]; + + shellHook = '' + export HISTFILE=.history + export HISTSIZE=1000000 + export HISTFILESIZE=1000000 + + unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH + + export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH" + + # Cross-compilation to musl with clang + export CC="${llvmPkgs.clang}/bin/clang" + export CXX="${llvmPkgs.clang}/bin/clang++" + + # Point to musl libraries for linking + export PKG_CONFIG_PATH="${pkgs.pkgsMusl.openssl.dev}/lib/pkgconfig:${pkgs.pkgsMusl.zlib.dev}/lib/pkgconfig:${pkgs.pkgsMusl.icu.dev}/lib/pkgconfig" + export CFLAGS="--target=x86_64-linux-musl -ggdb -Og -fno-omit-frame-pointer -DUSE_VALGRIND -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export CXXFLAGS="--target=x86_64-linux-musl -ggdb -Og -fno-omit-frame-pointer -DUSE_VALGRIND -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export LDFLAGS="--target=x86_64-linux-musl -L${pkgs.pkgsMusl.stdenv.cc.libc}/lib -fuse-ld=lld" + + # PostgreSQL environment + export PG_SOURCE_DIR="${config.pgSourceDir}" + export PG_BUILD_DIR="${config.pgBuildDir}-clang-musl" + export PG_INSTALL_DIR="${config.pgInstallDir}-clang-musl" + export PG_DATA_DIR="${config.pgDataDir}-clang-musl" + export PG_BENCH_DIR="${config.pgBenchDir}" + export PG_FLAME_DIR="${config.pgFlameDir}" + export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d) + + export GDBINIT="${gdbConfig}" + export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH" + + mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR" + + echo "Clang + musl environment configured" + echo " Compiler: $CC" + echo " LibC: musl (cross-compilation)" + + if [ -f ./pg-aliases.sh ]; then + source ./pg-aliases.sh + fi + + echo "PostgreSQL Development Environment Ready (Clang + musl)" + ''; + }; + +in { + inherit devShell clangDevShell muslDevShell clangMuslDevShell gdbConfig flameGraphScript pgbenchScript; +} diff --git a/src/backend/nodes/bitmapset.c b/src/backend/nodes/bitmapset.c index bf512cf806ff7..5c8a5b74f129c 100644 --- a/src/backend/nodes/bitmapset.c +++ b/src/backend/nodes/bitmapset.c @@ -1343,7 +1343,7 @@ bms_next_member(const Bitmapset *a, int prevbit) * * Returns largest member less than "prevbit", or -2 if there is none. * "prevbit" must NOT be more than one above the highest possible bit that can - * be set at the Bitmapset at its current size. + * be set in the Bitmapset at its current size. * * To ease finding the highest set bit for the initial loop, the special * prevbit value of -1 can be passed to have the function find the highest @@ -1379,6 +1379,9 @@ bms_prev_member(const Bitmapset *a, int prevbit) if (a == NULL || prevbit == 0) return -2; + /* Validate callers didn't give us something out of range */ + Assert(prevbit <= a->nwords * BITS_PER_BITMAPWORD); + /* transform -1 to the highest possible bit we could have set */ if (prevbit == -1) prevbit = a->nwords * BITS_PER_BITMAPWORD - 1; diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 903a8ac151aa1..94071ec0e160a 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -16,6 +16,7 @@ SUBDIRS = \ spgist_name_ops \ test_aio \ test_binaryheap \ + test_bitmapset \ test_bloomfilter \ test_copy_callbacks \ test_custom_rmgrs \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 93be0f57289a8..d8f5c9c74942f 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -15,6 +15,7 @@ subdir('spgist_name_ops') subdir('ssl_passphrase_callback') subdir('test_aio') subdir('test_binaryheap') +subdir('test_bitmapset') subdir('test_bloomfilter') subdir('test_copy_callbacks') subdir('test_custom_rmgrs') diff --git a/src/test/modules/test_bitmapset/.gitignore b/src/test/modules/test_bitmapset/.gitignore new file mode 100644 index 0000000000000..5dcb3ff972350 --- /dev/null +++ b/src/test/modules/test_bitmapset/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_bitmapset/Makefile b/src/test/modules/test_bitmapset/Makefile new file mode 100644 index 0000000000000..67199d40d735e --- /dev/null +++ b/src/test/modules/test_bitmapset/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_bitmapset/Makefile + +MODULE_big = test_bitmapset +OBJS = \ + $(WIN32RES) \ + test_bitmapset.o +PGFILEDESC = "test_bitmapset - test code for src/include/nodes/bitmapset.h" + +EXTENSION = test_bitmapset +DATA = test_bitmapset--1.0.sql + +REGRESS = test_bitmapset + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_bitmapset +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_bitmapset/expected/test_bitmapset.out b/src/test/modules/test_bitmapset/expected/test_bitmapset.out new file mode 100644 index 0000000000000..32093d81f6728 --- /dev/null +++ b/src/test/modules/test_bitmapset/expected/test_bitmapset.out @@ -0,0 +1,19 @@ +CREATE EXTENSION test_bitmapset; +-- +-- All the logic is in the test_bitmapset() function. It will throw +-- an error if something fails. +-- +SELECT test_bitmapset(); +NOTICE: starting bitmapset tests +NOTICE: testing empty bitmapset operations +NOTICE: testing single element bitmapset operations +NOTICE: testing multiple elements and set operations +NOTICE: testing edge cases and boundary conditions +NOTICE: testing iteration functions +NOTICE: testing comprehensive set operations +NOTICE: all bitmapset tests passed + test_bitmapset +---------------- + +(1 row) + diff --git a/src/test/modules/test_bitmapset/meson.build b/src/test/modules/test_bitmapset/meson.build new file mode 100644 index 0000000000000..848f7a44eb933 --- /dev/null +++ b/src/test/modules/test_bitmapset/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024-2025, PostgreSQL Global Development Group + +test_bitmapset_sources = files( + 'test_bitmapset.c', +) + +if host_system == 'windows' + test_bitmapset_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_bitmapset', + '--FILEDESC', 'test_bitmapset - test code for src/include/nodes/bitmapset.h',]) +endif + +test_bitmapset = shared_module('test_bitmapset', + test_bitmapset_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_bitmapset + +test_install_data += files( + 'test_bitmapset.control', + 'test_bitmapset--1.0.sql', +) + +tests += { + 'name': 'test_bitmapset', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_bitmapset', + ], + }, +} diff --git a/src/test/modules/test_bitmapset/sql/test_bitmapset.sql b/src/test/modules/test_bitmapset/sql/test_bitmapset.sql new file mode 100644 index 0000000000000..ceb524ee51b58 --- /dev/null +++ b/src/test/modules/test_bitmapset/sql/test_bitmapset.sql @@ -0,0 +1,7 @@ +CREATE EXTENSION test_bitmapset; + +-- +-- All the logic is in the test_bitmapset() function. It will throw +-- an error if something fails. +-- +SELECT test_bitmapset(); diff --git a/src/test/modules/test_bitmapset/test_bitmapset--1.0.sql b/src/test/modules/test_bitmapset/test_bitmapset--1.0.sql new file mode 100644 index 0000000000000..ba8af99704497 --- /dev/null +++ b/src/test/modules/test_bitmapset/test_bitmapset--1.0.sql @@ -0,0 +1,8 @@ +/* src/test/modules/test_bitmapset/test_bitmapset--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_bitmapset" to load this file. \quit + +CREATE FUNCTION test_bitmapset() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_bitmapset/test_bitmapset.c b/src/test/modules/test_bitmapset/test_bitmapset.c new file mode 100644 index 0000000000000..39e2878189397 --- /dev/null +++ b/src/test/modules/test_bitmapset/test_bitmapset.c @@ -0,0 +1,419 @@ +/* + * test_bitmapset.c + * Test module for bitmapset data structure + * + * This module tests the bitmapset implementation in PostgreSQL, + * covering all public API functions, edge cases, and memory usage. + * + * src/test/modules/test_bitmapset/test_bitmapset.c + */ + +#include "postgres.h" + +#include "fmgr.h" +#include "nodes/bitmapset.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(test_bitmapset); + +/* Test assertion macros following test_bitmapset pattern */ +#define EXPECT_TRUE(expr) \ + do { \ + if (!(expr)) \ + elog(ERROR, \ + "%s was unexpectedly false in file \"%s\" line %u", \ + #expr, __FILE__, __LINE__); \ + } while (0) + +#define EXPECT_FALSE(expr) \ + do { \ + if (expr) \ + elog(ERROR, \ + "%s was unexpectedly true in file \"%s\" line %u", \ + #expr, __FILE__, __LINE__); \ + } while (0) + +#define EXPECT_EQ_INT(result_expr, expected_expr) \ + do { \ + int _result = (result_expr); \ + int _expected = (expected_expr); \ + if (_result != _expected) \ + elog(ERROR, \ + "%s yielded %d, expected %d (%s) in file \"%s\" line %u", \ + #result_expr, _result, _expected, #expected_expr, __FILE__, __LINE__); \ + } while (0) + +#define EXPECT_NULL(expr) \ + do { \ + if ((expr) != NULL) \ + elog(ERROR, \ + "%s was unexpectedly non-NULL in file \"%s\" line %u", \ + #expr, __FILE__, __LINE__); \ + } while (0) + +#define EXPECT_NOT_NULL(expr) \ + do { \ + if ((expr) == NULL) \ + elog(ERROR, \ + "%s was unexpectedly NULL in file \"%s\" line %u", \ + #expr, __FILE__, __LINE__); \ + } while (0) + +/* Test empty bitmapset operations */ +static void +test_empty_bitmapset(void) +{ + Bitmapset *result, + *bms = NULL; + + elog(NOTICE, "testing empty bitmapset operations"); + + /* Test operations on NULL bitmapset */ + EXPECT_TRUE(bms_is_empty(bms)); + EXPECT_EQ_INT(bms_num_members(bms), 0); + EXPECT_FALSE(bms_is_member(0, bms)); + EXPECT_FALSE(bms_is_member(1, bms)); + EXPECT_FALSE(bms_is_member(100, bms)); + EXPECT_EQ_INT(bms_next_member(bms, -1), -2); + EXPECT_EQ_INT(bms_prev_member(bms, -1), -2); + + /* Test copy of empty set */ + result = bms_copy(bms); + EXPECT_NULL(result); + + /* Test union with empty set */ + result = bms_union(bms, NULL); + EXPECT_NULL(result); + + /* Test intersection with empty set */ + result = bms_intersect(bms, NULL); + EXPECT_NULL(result); + + /* Test difference with empty set */ + result = bms_difference(bms, NULL); + EXPECT_NULL(result); + + /* Test equal comparison */ + EXPECT_TRUE(bms_equal(bms, NULL)); + EXPECT_TRUE(bms_equal(NULL, bms)); + + /* Test subset operations */ + EXPECT_TRUE(bms_is_subset(bms, NULL)); + EXPECT_TRUE(bms_is_subset(NULL, bms)); + + /* Test overlap */ + EXPECT_FALSE(bms_overlap(bms, NULL)); + EXPECT_FALSE(bms_overlap(NULL, bms)); +} + +/* Test single element bitmapset operations */ +static void +test_single_element(void) +{ + Bitmapset *result, + *bms = NULL; + int test_member = 42; + + elog(NOTICE, "testing single element bitmapset operations"); + + /* Add single element */ + bms = bms_add_member(bms, test_member); + EXPECT_NOT_NULL(bms); + EXPECT_FALSE(bms_is_empty(bms)); + EXPECT_EQ_INT(bms_num_members(bms), 1); + EXPECT_TRUE(bms_is_member(test_member, bms)); + EXPECT_FALSE(bms_is_member(test_member - 1, bms)); + EXPECT_FALSE(bms_is_member(test_member + 1, bms)); + + /* Test iteration */ + EXPECT_EQ_INT(bms_next_member(bms, -1), test_member); + EXPECT_EQ_INT(bms_next_member(bms, test_member), -2); + EXPECT_EQ_INT(bms_prev_member(bms, test_member + 1), test_member); + EXPECT_EQ_INT(bms_prev_member(bms, test_member), -2); + + /* Test copy */ + result = bms_copy(bms); + EXPECT_NOT_NULL(result); + EXPECT_TRUE(bms_equal(bms, result)); + EXPECT_TRUE(bms_is_member(test_member, result)); + + /* Test remove member */ + result = bms_del_member(result, test_member); + EXPECT_NULL(result); + + /* Test remove non-existent member */ + result = bms_copy(bms); + EXPECT_NOT_NULL(result); + result = bms_del_member(result, test_member + 1); + EXPECT_NOT_NULL(result); + EXPECT_TRUE(bms_equal(bms, result)); + + bms_free(bms); + bms_free(result); +} + +/* Test multiple elements and set operations */ +static void +test_multiple_elements(void) +{ + Bitmapset *bms1 = NULL; + Bitmapset *bms2 = NULL; + Bitmapset *result = NULL; + int elements1[] = {1, 5, 10, 15, 20, 100}; + int elements2[] = {3, 5, 12, 15, 25, 200}; + int num_elements1 = sizeof(elements1) / sizeof(elements1[0]); + int num_elements2 = sizeof(elements2) / sizeof(elements2[0]); + + elog(NOTICE, "testing multiple elements and set operations"); + + /* Build first set */ + for (int i = 0; i < num_elements1; i++) + bms1 = bms_add_member(bms1, elements1[i]); + + EXPECT_EQ_INT(bms_num_members(bms1), num_elements1); + + /* Build second set */ + for (int i = 0; i < num_elements2; i++) + bms2 = bms_add_member(bms2, elements2[i]); + + EXPECT_EQ_INT(bms_num_members(bms2), num_elements2); + + /* Test membership */ + for (int i = 0; i < num_elements1; i++) + EXPECT_TRUE(bms_is_member(elements1[i], bms1)); + + for (int i = 0; i < num_elements2; i++) + EXPECT_TRUE(bms_is_member(elements2[i], bms2)); + + /* Test union */ + result = bms_union(bms1, bms2); + EXPECT_NOT_NULL(result); + for (int i = 0; i < num_elements1; i++) + EXPECT_TRUE(bms_is_member(elements1[i], result)); + for (int i = 0; i < num_elements2; i++) + EXPECT_TRUE(bms_is_member(elements2[i], result)); + EXPECT_EQ_INT(bms_num_members(result), 10); /* 1,3,5,10,12,15,20,25,100,200 */ + bms_free(result); + + /* Test intersection */ + result = bms_intersect(bms1, bms2); + EXPECT_NOT_NULL(result); + EXPECT_TRUE(bms_is_member(5, result)); + EXPECT_TRUE(bms_is_member(15, result)); + EXPECT_EQ_INT(bms_num_members(result), 2); /* 5, 15 */ + bms_free(result); + + /* Test difference */ + result = bms_difference(bms1, bms2); + EXPECT_NOT_NULL(result); + EXPECT_TRUE(bms_is_member(1, result)); + EXPECT_TRUE(bms_is_member(10, result)); + EXPECT_TRUE(bms_is_member(20, result)); + EXPECT_TRUE(bms_is_member(100, result)); + EXPECT_FALSE(bms_is_member(5, result)); + EXPECT_FALSE(bms_is_member(15, result)); + EXPECT_EQ_INT(bms_num_members(result), 4); /* 1, 10, 20, 100 */ + bms_free(result); + + /* Test overlap */ + EXPECT_TRUE(bms_overlap(bms1, bms2)); + + /* Test subset operations */ + result = NULL; + result = bms_add_member(result, 5); + result = bms_add_member(result, 15); + EXPECT_TRUE(bms_is_subset(result, bms1)); + EXPECT_TRUE(bms_is_subset(result, bms2)); + EXPECT_FALSE(bms_is_subset(bms1, result)); + bms_free(result); + + bms_free(bms1); + bms_free(bms2); +} + +/* Test edge cases and boundary conditions */ +static void +test_edge_cases(void) +{ + int large_element = 10000; + int count; + int member; + Bitmapset *result; + Bitmapset *bms = NULL; + + elog(NOTICE, "testing edge cases and boundary conditions"); + + /* Test element 0 */ + bms = bms_add_member(bms, 0); + EXPECT_TRUE(bms_is_member(0, bms)); + EXPECT_EQ_INT(bms_num_members(bms), 1); + EXPECT_EQ_INT(bms_next_member(bms, -1), 0); + bms_free(bms); + bms = NULL; + + /* Test large element numbers */ + bms = bms_add_member(bms, large_element); + EXPECT_TRUE(bms_is_member(large_element, bms)); + EXPECT_EQ_INT(bms_num_members(bms), 1); + bms_free(bms); + bms = NULL; + + /* Test adding same element multiple times */ + bms = bms_add_member(bms, 42); + bms = bms_add_member(bms, 42); + EXPECT_EQ_INT(bms_num_members(bms), 1); + EXPECT_TRUE(bms_is_member(42, bms)); + bms_free(bms); + bms = NULL; + + /* Test removing from single-element set */ + bms = bms_add_member(bms, 99); + result = bms_del_member(bms, 99); + EXPECT_NULL(result); + bms_free(bms); + bms = NULL; + + /* Test dense range */ + for (int i = 0; i < 64; i++) + bms = bms_add_member(bms, i); + EXPECT_EQ_INT(bms_num_members(bms), 64); + + /* Test iteration over dense range */ + count = 0; + member = -1; + while ((member = bms_next_member(bms, member)) >= 0) + { + EXPECT_EQ_INT(member, count); + count++; + } + EXPECT_EQ_INT(count, 64); + + bms_free(bms); +} + +/* Test iterator functions thoroughly */ +static void +test_iteration(void) +{ + Bitmapset *bms = NULL; + int member; + int index; + int elements[] = {2, 7, 15, 31, 63, 127, 255, 511, 1023}; + int num_elements = sizeof(elements) / sizeof(elements[0]); + + elog(NOTICE, "testing iteration functions"); + + /* Build test set */ + for (int i = 0; i < num_elements; i++) + bms = bms_add_member(bms, elements[i]); + + /* Test forward iteration */ + member = -1; + index = 0; + while ((member = bms_next_member(bms, member)) >= 0) + { + EXPECT_EQ_INT(member, elements[index]); + index++; + } + EXPECT_EQ_INT(index, num_elements); + + /* Test backward iteration */ + member = bms->nwords * BITS_PER_BITMAPWORD; + index = num_elements - 1; + while ((member = bms_prev_member(bms, member)) >= 0) + { + EXPECT_EQ_INT(member, elements[index]); + index--; + } + EXPECT_EQ_INT(index, -1); + + /* Test iteration bounds */ + EXPECT_EQ_INT(bms_next_member(bms, 1023), -2); + EXPECT_EQ_INT(bms_prev_member(bms, 2), -2); + + bms_free(bms); +} + +/* Test set operations with various combinations */ +static void +test_set_operations(void) +{ + Bitmapset *empty = NULL; + Bitmapset *single = NULL; + Bitmapset *multi = NULL; + Bitmapset *result = NULL; + Bitmapset *single_copy = NULL; + + elog(NOTICE, "testing comprehensive set operations"); + + single = bms_add_member(single, 10); + multi = bms_add_member(multi, 5); + multi = bms_add_member(multi, 10); + multi = bms_add_member(multi, 15); + + /* Union operations */ + result = bms_union(empty, single); + EXPECT_TRUE(bms_equal(result, single)); + bms_free(result); + + result = bms_union(single, empty); + EXPECT_TRUE(bms_equal(result, single)); + bms_free(result); + + result = bms_union(single, multi); + EXPECT_EQ_INT(bms_num_members(result), 3); + EXPECT_TRUE(bms_is_member(5, result)); + EXPECT_TRUE(bms_is_member(10, result)); + EXPECT_TRUE(bms_is_member(15, result)); + bms_free(result); + + /* Intersection operations */ + result = bms_intersect(empty, single); + EXPECT_NULL(result); + + result = bms_intersect(single, multi); + EXPECT_EQ_INT(bms_num_members(result), 1); + EXPECT_TRUE(bms_is_member(10, result)); + bms_free(result); + + /* Difference operations */ + result = bms_difference(single, multi); + EXPECT_NULL(result); + + result = bms_difference(multi, single); + EXPECT_EQ_INT(bms_num_members(result), 2); + EXPECT_TRUE(bms_is_member(5, result)); + EXPECT_TRUE(bms_is_member(15, result)); + EXPECT_FALSE(bms_is_member(10, result)); + bms_free(result); + + /* Equality tests */ + EXPECT_FALSE(bms_equal(single, multi)); + EXPECT_TRUE(bms_equal(empty, NULL)); + + single_copy = bms_copy(single); + EXPECT_TRUE(bms_equal(single, single_copy)); + bms_free(single_copy); + + bms_free(single); + bms_free(multi); +} + +/* Main test function */ +Datum +test_bitmapset(PG_FUNCTION_ARGS) +{ + elog(NOTICE, "starting bitmapset tests"); + + test_empty_bitmapset(); + test_single_element(); + test_multiple_elements(); + test_edge_cases(); + test_iteration(); + test_set_operations(); + + elog(NOTICE, "all bitmapset tests passed"); + + PG_RETURN_VOID(); +} diff --git a/src/test/modules/test_bitmapset/test_bitmapset.control b/src/test/modules/test_bitmapset/test_bitmapset.control new file mode 100644 index 0000000000000..8d02ec8bf0a95 --- /dev/null +++ b/src/test/modules/test_bitmapset/test_bitmapset.control @@ -0,0 +1,4 @@ +comment = 'Test code for Bitmapset' +default_version = '1.0' +module_pathname = '$libdir/test_bitmapset' +relocatable = true diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 5d85dcc62f0a5..12333b5b9a5f7 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -1232,7 +1232,7 @@ spawn_process(const char *cmdline) char *cmdline2; cmdline2 = psprintf("exec %s", cmdline); - execl(shellprog, shellprog, "-c", cmdline2, (char *) NULL); + execlp(shellprog, shellprog, "-c", cmdline2, (char *) NULL); /* Not using the normal bail() here as we want _exit */ bail_noatexit("could not exec \"%s\": %m", shellprog); }