Skip to content

Commit 2284b3b

Browse files
gburdgburd
committed
DO NOT COMMIT - TESTING CODE
1 parent 6d5e822 commit 2284b3b

File tree

17 files changed

+2589
-11
lines changed

17 files changed

+2589
-11
lines changed

‎out.log‎

Lines changed: 583 additions & 0 deletions
Large diffs are not rendered by default.

‎post_setup.sql‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
-- Enable tracking (in case it was reset)
2+
SET enable_heap_prune_tracking = on;
3+
4+
-- Disable autovacuum on pgbench tables to preserve dead tuples for pruning tests
5+
ALTER TABLE pgbench_accounts SET (autovacuum_enabled = false);
6+
ALTER TABLE pgbench_branches SET (autovacuum_enabled = false);
7+
ALTER TABLE pgbench_tellers SET (autovacuum_enabled = false);
8+
ALTER TABLE pgbench_history SET (autovacuum_enabled = false);
9+
10+
-- Add columns that can be updated without affecting indexes (for HOT updates)
11+
ALTER TABLE pgbench_accounts ADD COLUMN IF NOT EXISTS last_updated timestamp DEFAULT now();
12+
ALTER TABLE pgbench_accounts ADD COLUMN IF NOT EXISTS update_count int DEFAULT 0;
13+
ALTER TABLE pgbench_accounts ADD COLUMN IF NOT EXISTS notes text DEFAULT 'initial';
14+
15+
-- Make the notes column much larger to create page pressure
16+
ALTER TABLE pgbench_accounts ALTER COLUMN notes SET DEFAULT repeat('initial_data', 100);
17+
18+
-- Create a partial index to allow more HOT updates
19+
CREATE INDEX IF NOT EXISTS idx_accounts_high_balance ON pgbench_accounts(abalance) WHERE abalance > 1000;
20+
21+
-- Reset statistics after setup
22+
SELECT pg_stat_reset();
23+
24+
-- Show initial state
25+
SELECT 'POST_SETUP_BASELINE' as phase, * FROM capture_prune_stats();

‎prune_test.sql‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
\set aid random(1, 100000 * :scale)
2+
\set note_suffix random(1, 1000)
3+
\set large_data_size random(50, 200)
4+
5+
-- Single transaction with multiple updates to same row to create dead tuples and page pressure
6+
BEGIN;
7+
8+
-- Update the same row multiple times with increasingly large data to create dead tuples
9+
UPDATE pgbench_accounts
10+
SET last_updated = now(),
11+
update_count = update_count + 1,
12+
notes = 'v1_' || :note_suffix || '_' || repeat('data_chunk_', :large_data_size)
13+
WHERE aid = :aid;
14+
15+
UPDATE pgbench_accounts
16+
SET last_updated = now(),
17+
update_count = update_count + 2,
18+
notes = 'v2_' || :note_suffix || '_' || repeat('more_data_chunk_', :large_data_size + 10)
19+
WHERE aid = :aid;
20+
21+
UPDATE pgbench_accounts
22+
SET last_updated = now(),
23+
update_count = update_count + 3,
24+
notes = 'v3_' || :note_suffix || '_' || repeat('final_data_chunk_', :large_data_size + 20)
25+
WHERE aid = :aid;
26+
27+
-- Also update a few nearby rows to create more page pressure
28+
UPDATE pgbench_accounts
29+
SET last_updated = now(),
30+
update_count = update_count + 1,
31+
notes = 'neighbor_' || :note_suffix || '_' || repeat('neighbor_data_', :large_data_size)
32+
WHERE aid BETWEEN :aid + 1 AND :aid + 3;
33+
34+
COMMIT;

‎results.log‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
phase | context | calls_total | pages_pruned | tuples_pruned | space_freed | time_spent_us | exit_success | exit_invalid_xact_xid | exit_no_removable_xids | exit_page_not_prunable | exit_lock_failed | exit_other | prune_success_rate_pct | avg_time_per_call_us
2+
----------+--------------------+-------------+--------------+---------------+-------------+---------------+--------------+-----------------------+------------------------+------------------------+------------------+------------+------------------------+----------------------
3+
BASELINE | SCAN_OPPORTUNISTIC | 153 | 0 | 0 | 0 | 0 | 0 | 150 | 0 | 3 | 0 | 0 | 0.00 | 0.000
4+
BASELINE | UPDATE_FULL_PAGE | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
5+
BASELINE | INSERT_SPACE_CHECK | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
6+
BASELINE | SCAN_END | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
7+
BASELINE | MULTI_INSERT | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
8+
(5 rows)
9+
10+
phase | context | calls_total | success_rate_pct | main_failure_reason | failure_count | failure_pct
11+
------------------------+--------------------+-------------+------------------+---------------------+---------------+-------------
12+
BASELINE_EXIT_ANALYSIS | SCAN_OPPORTUNISTIC | 160 | 0.00 | INVALID_XACT_XID | 157 | 98.13
13+
(1 row)
14+
15+
phase | context | calls_total | pages_pruned | tuples_pruned | space_freed | time_spent_us | exit_success | exit_invalid_xact_xid | exit_no_removable_xids | exit_page_not_prunable | exit_lock_failed | exit_other | prune_success_rate_pct | avg_time_per_call_us
16+
----------+--------------------+-------------+--------------+---------------+-------------+---------------+--------------+-----------------------+------------------------+------------------------+------------------+------------+------------------------+----------------------
17+
MID_TEST | SCAN_OPPORTUNISTIC | 154 | 0 | 0 | 0 | 1 | 0 | 151 | 0 | 3 | 0 | 0 | 0.00 | 0.006
18+
MID_TEST | UPDATE_FULL_PAGE | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
19+
MID_TEST | INSERT_SPACE_CHECK | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
20+
MID_TEST | SCAN_END | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
21+
MID_TEST | MULTI_INSERT | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
22+
(5 rows)
23+
24+
phase | context | calls_total | success_rate_pct | main_failure_reason | failure_count | failure_pct
25+
------------------------+--------------------+-------------+------------------+---------------------+---------------+-------------
26+
MID_TEST_EXIT_ANALYSIS | SCAN_OPPORTUNISTIC | 160 | 0.00 | INVALID_XACT_XID | 157 | 98.13
27+
(1 row)
28+
29+
phase | context | calls_total | pages_pruned | tuples_pruned | space_freed | time_spent_us | exit_success | exit_invalid_xact_xid | exit_no_removable_xids | exit_page_not_prunable | exit_lock_failed | exit_other | prune_success_rate_pct | avg_time_per_call_us
30+
-----------+--------------------+-------------+--------------+---------------+-------------+---------------+--------------+-----------------------+------------------------+------------------------+------------------+------------+------------------------+----------------------
31+
POST_TEST | SCAN_OPPORTUNISTIC | 153 | 0 | 0 | 0 | 1 | 0 | 150 | 0 | 3 | 0 | 0 | 0.00 | 0.007
32+
POST_TEST | UPDATE_FULL_PAGE | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
33+
POST_TEST | INSERT_SPACE_CHECK | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
34+
POST_TEST | SCAN_END | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
35+
POST_TEST | MULTI_INSERT | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
36+
(5 rows)
37+
38+
phase | context | calls_total | success_rate_pct | main_failure_reason | failure_count | failure_pct
39+
-------------------------+--------------------+-------------+------------------+---------------------+---------------+-------------
40+
POST_TEST_EXIT_ANALYSIS | SCAN_OPPORTUNISTIC | 160 | 0.00 | INVALID_XACT_XID | 157 | 98.13
41+
(1 row)
42+
43+
report_type | context | calls_total | success_rate_pct | main_failure_reason | failure_count | failure_pct
44+
---------------------+--------------------+-------------+------------------+---------------------+---------------+-------------
45+
EXIT_REASON_SUMMARY | SCAN_OPPORTUNISTIC | 160 | 0.00 | INVALID_XACT_XID | 157 | 98.13
46+
(1 row)
47+
48+
report_type | context | main_failure_reason | recommendation | failure_pct
49+
-----------------+--------------------+---------------------+---------------------------------------------------------------------------+-------------
50+
RECOMMENDATIONS | SCAN_OPPORTUNISTIC | INVALID_XACT_XID | Check prune_xid setting and page header - may need transaction visibility | 98.58
51+
(1 row)
52+
53+
schemaname | relname | n_tup_ins | n_tup_upd | n_tup_hot_upd | hot_update_pct | table_size
54+
------------+------------------+-----------+-----------+---------------+----------------+------------
55+
public | pgbench_accounts | 0 | 7786123 | 6686906 | 85.88 | 1932 MB
56+
(1 row)
57+
58+
relpages | reltuples | n_dead_tup | last_vacuum | last_autovacuum | heap_size
59+
----------+-----------+------------+-------------+-----------------+-----------
60+
81968 | 5e+06 | 1355592 | | | 1820 MB
61+
(1 row)
62+
63+
context | calls_total | pages_pruned | tuples_pruned | space_freed | exit_success | exit_invalid_xact_xid | exit_no_removable_xids | exit_page_not_prunable | exit_lock_failed | exit_other | prune_success_rate_pct | avg_time_per_call_us
64+
--------------------+-------------+--------------+---------------+-------------+--------------+-----------------------+------------------------+------------------------+------------------+------------+------------------------+----------------------
65+
SCAN_OPPORTUNISTIC | 153 | 0 | 0 | 0 | 0 | 150 | 0 | 3 | 0 | 0 | 0.00 | 0.144
66+
(1 row)
67+

‎run_test.sh‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
#set -x
5+
6+
PSQL="$PG_INSTALL_DIR/bin/psql -X -h $PG_DATA_DIR postgres"
7+
PGBENCH="$PG_INSTALL_DIR/bin/pgbench -h $PG_DATA_DIR"
8+
9+
PGBENCH_SCALE=50
10+
PGBENCH_CLIENTS=8
11+
PGBENCH_JOBS=4
12+
PGBENCH_TIME=120
13+
14+
echo "=== Setting up test environment ==="
15+
$PSQL -f setup.sql
16+
17+
echo "=== Setup the benchmark ==="
18+
$PGBENCH -i -s $PGBENCH_SCALE postgres
19+
20+
echo "=== Post-setup configuration ==="
21+
$PSQL -f post_setup.sql
22+
23+
echo "=== Pre-loading data to create page pressure ==="
24+
$PSQL -c "
25+
-- Create more update activity to fill pages with large tuples
26+
UPDATE pgbench_accounts
27+
SET notes = 'preload_' || (random() * 10000)::int || '_' || repeat('large_data_chunk', 50),
28+
update_count = update_count + 1,
29+
last_updated = now()
30+
WHERE aid IN (
31+
SELECT (random() * (100000 * $PGBENCH_SCALE))::int + 1
32+
FROM generate_series(1, 20000)
33+
);
34+
35+
-- Create some dead tuples by updating again
36+
UPDATE pgbench_accounts
37+
SET notes = 'dead_preload_' || (random() * 10000)::int || '_' || repeat('dead_data_chunk', 60),
38+
update_count = update_count + 2
39+
WHERE aid IN (
40+
SELECT (random() * (100000 * $PGBENCH_SCALE))::int + 1
41+
FROM generate_series(1, 10000)
42+
);
43+
"
44+
45+
echo "=== Capturing baseline statistics ==="
46+
$PSQL -c "SELECT 'BASELINE' as phase, * FROM capture_prune_stats();" >results.log
47+
$PSQL -c "SELECT 'BASELINE_EXIT_ANALYSIS' as phase, * FROM analyze_exit_reasons();" >>results.log
48+
49+
echo "=== Running pgbench HOT update workload ==="
50+
$PGBENCH -c $PGBENCH_CLIENTS -j $PGBENCH_JOBS -T $PGBENCH_TIME -f prune_test.sql -P 10 postgres
51+
52+
echo "=== Capturing mid-test statistics ==="
53+
$PSQL -c "SELECT 'MID_TEST' as phase, * FROM capture_prune_stats();" >>results.log
54+
$PSQL -c "SELECT 'MID_TEST_EXIT_ANALYSIS' as phase, * FROM analyze_exit_reasons();" >>results.log
55+
56+
echo "=== Running standard TPC-B workload for comparison ==="
57+
$PGBENCH -c $PGBENCH_CLIENTS -j $PGBENCH_JOBS -T 30 -P 10 postgres
58+
59+
echo "=== Capturing post-test statistics ==="
60+
$PSQL -c "SELECT 'POST_TEST' as phase, * FROM capture_prune_stats();" >>results.log
61+
$PSQL -c "SELECT 'POST_TEST_EXIT_ANALYSIS' as phase, * FROM analyze_exit_reasons();" >>results.log
62+
63+
echo "=== Exit Reason Analysis and Recommendations ==="
64+
$PSQL -c "
65+
SELECT
66+
'EXIT_REASON_SUMMARY' as report_type,
67+
context,
68+
calls_total,
69+
success_rate_pct,
70+
main_failure_reason,
71+
failure_count,
72+
failure_pct
73+
FROM analyze_exit_reasons()
74+
ORDER BY calls_total DESC;
75+
" >>results.log
76+
77+
$PSQL -c "
78+
SELECT
79+
'RECOMMENDATIONS' as report_type,
80+
context,
81+
main_failure_reason,
82+
recommendation,
83+
failure_pct
84+
FROM get_pruning_recommendations()
85+
ORDER BY failure_pct DESC;
86+
" >>results.log
87+
88+
echo "=== HOT Update Effectiveness ==="
89+
$PSQL -c "
90+
SELECT
91+
schemaname, relname,
92+
n_tup_ins, n_tup_upd, n_tup_hot_upd,
93+
CASE WHEN n_tup_upd > 0
94+
THEN round(100.0 * n_tup_hot_upd / n_tup_upd, 2)
95+
ELSE 0
96+
END as hot_update_pct,
97+
pg_size_pretty(pg_total_relation_size(schemaname||'.'||relname)) as table_size
98+
FROM pg_stat_user_tables
99+
WHERE relname = 'pgbench_accounts';
100+
" >>results.log
101+
102+
echo "=== Page-level analysis ==="
103+
$PSQL -c "
104+
SELECT
105+
relpages,
106+
reltuples,
107+
n_dead_tup,
108+
last_vacuum,
109+
last_autovacuum,
110+
pg_size_pretty(pg_relation_size('pgbench_accounts')) as heap_size
111+
FROM pg_stat_user_tables
112+
JOIN pg_class ON pg_class.relname = pg_stat_user_tables.relname
113+
WHERE pg_stat_user_tables.relname = 'pgbench_accounts';
114+
" >>results.log
115+
116+
echo "=== Detailed Statistics Breakdown ==="
117+
$PSQL -c "
118+
SELECT
119+
context,
120+
calls_total,
121+
pages_pruned,
122+
tuples_pruned,
123+
space_freed,
124+
exit_success,
125+
exit_invalid_xact_xid,
126+
exit_no_removable_xids,
127+
exit_page_not_prunable,
128+
exit_lock_failed,
129+
exit_other,
130+
prune_success_rate_pct,
131+
avg_time_per_call_us
132+
FROM capture_prune_stats()
133+
WHERE calls_total > 0
134+
ORDER BY calls_total DESC;
135+
" >>results.log
136+
137+
echo "=== Test complete. Results in results.log ==="
138+
cat results.log

0 commit comments

Comments
 (0)