Skip to content

Store account_id fields as BYTEA #488

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aditya1702 wants to merge 32 commits into
base: db-indexes
Choose a base branch
from
Open

Store account_id fields as BYTEA #488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
120b252
Add StellarAddress type and update migrations to use BYTEA
aditya1702 Feb 3, 2026
fe40b54
Update accounts.go for BYTEA stellar_address column
aditya1702 Feb 3, 2026
dee8de5
Update transactions.go for BYTEA account_id column
aditya1702 Feb 3, 2026
12b192b
Update query_utils.go for BYTEA account_id conversion
aditya1702 Feb 3, 2026
a5df424
Add tests for StellarAddress Scan/Value methods
aditya1702 Feb 3, 2026
f8aca24
Fix test failures and IsAccountFeeBumpEligible query
aditya1702 Feb 3, 2026
da4c44e
Fix tests
aditya1702 Feb 4, 2026
f23fd75
StellarAddress -> AddressBytea
aditya1702 Feb 4, 2026
096a08e
Change operations_accounts.account_id from TEXT to BYTEA
aditya1702 Feb 4, 2026
705d2a4
Simplify AddressBytea.Scan() to only handle BYTEA
aditya1702 Feb 4, 2026
dacec16
Remove AccountIDBytea from query_utils.go
aditya1702 Feb 4, 2026
8487254
Remove AccountIDBytea field from operations.go and transactions.go
aditya1702 Feb 4, 2026
fa60b23
Update BatchCopy to write account_id as BYTEA
aditya1702 Feb 4, 2026
ac3b07e
Update backfill_helpers.go to use BYTEA for account_id
aditya1702 Feb 4, 2026
331461b
Update test files for BYTEA operations_accounts.account_id
aditya1702 Feb 4, 2026
01b06d3
Update BatchInsert to write account_id as BYTEA
aditya1702 Feb 4, 2026
9021603
Fix operations tests for BYTEA account_id
aditya1702 Feb 4, 2026
db17bf3
Fix make check issues
aditya1702 Feb 4, 2026
ba42d49
Change state_changes account_id columns from TEXT to BYTEA
aditya1702 Feb 4, 2026
9c1e726
Add pgtypeBytesFromNullStringAddress helper for BYTEA address conversion
aditya1702 Feb 4, 2026
5a10c3b
Change StateChange.AccountID type from string to AddressBytea
aditya1702 Feb 4, 2026
e209dc5
Update BatchInsert to write account_id columns as BYTEA
aditya1702 Feb 4, 2026
c6c22e9
Update BatchCopy to write account_id columns as BYTEA
aditya1702 Feb 4, 2026
3bbc0a7
Update BatchGetByAccountAddress to query BYTEA account_id
aditya1702 Feb 4, 2026
e4199c3
Fix all tests
aditya1702 Feb 4, 2026
4677df9
Update backfill_helpers.go
aditya1702 Feb 5, 2026
af12032
Update backfill_helpers.go
aditya1702 Feb 5, 2026
39e24ec
Use NullAddressBytea method for nullable fields of state changes
aditya1702 Feb 5, 2026
6fb0b76
fix tests
aditya1702 Feb 5, 2026
7c5089e
fix more tests
aditya1702 Feb 5, 2026
4ad1ede
fix more tests again
aditya1702 Feb 5, 2026
4d825e1
Update account_service_test.go
aditya1702 Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 50 additions & 17 deletions internal/data/accounts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func (m *AccountModel) Get(ctx context.Context, address string) (*types.Account,
const query = `SELECT * FROM accounts WHERE stellar_address = $1`
var account types.Account
start := time.Now()
err := m.DB.GetContext(ctx, &account, query, address)
err := m.DB.GetContext(ctx, &account, query, types.AddressBytea(address))
duration := time.Since(start).Seconds()
m.MetricsService.ObserveDBQueryDuration("Get", "accounts", duration)
if err != nil {
Expand All @@ -50,22 +50,27 @@ func (m *AccountModel) Get(ctx context.Context, address string) (*types.Account,
func (m *AccountModel) GetAll(ctx context.Context) ([]string, error) {
const query = `SELECT stellar_address FROM accounts`
start := time.Now()
accounts := []string{}
err := m.DB.SelectContext(ctx, &accounts, query)
var addresses []types.AddressBytea
err := m.DB.SelectContext(ctx, &addresses, query)
duration := time.Since(start).Seconds()
m.MetricsService.ObserveDBQueryDuration("GetAll", "accounts", duration)
if err != nil {
m.MetricsService.IncDBQueryError("GetAll", "accounts", utils.GetDBErrorType(err))
return nil, fmt.Errorf("getting all accounts: %w", err)
}
m.MetricsService.IncDBQuery("GetAll", "accounts")
return accounts, nil
// Convert []AddressBytea to []string
result := make([]string, len(addresses))
for i, addr := range addresses {
result[i] = string(addr)
}
return result, nil
}

func (m *AccountModel) Insert(ctx context.Context, address string) error {
const query = `INSERT INTO accounts (stellar_address) VALUES ($1)`
start := time.Now()
_, err := m.DB.ExecContext(ctx, query, address)
_, err := m.DB.ExecContext(ctx, query, types.AddressBytea(address))
duration := time.Since(start).Seconds()
m.MetricsService.ObserveDBQueryDuration("Insert", "accounts", duration)
if err != nil {
Expand All @@ -82,7 +87,7 @@ func (m *AccountModel) Insert(ctx context.Context, address string) error {
func (m *AccountModel) Delete(ctx context.Context, address string) error {
const query = `DELETE FROM accounts WHERE stellar_address = $1`
start := time.Now()
result, err := m.DB.ExecContext(ctx, query, address)
result, err := m.DB.ExecContext(ctx, query, types.AddressBytea(address))
duration := time.Since(start).Seconds()
m.MetricsService.ObserveDBQueryDuration("Delete", "accounts", duration)
if err != nil {
Expand All @@ -104,25 +109,53 @@ func (m *AccountModel) Delete(ctx context.Context, address string) error {
return nil
}

// BatchGetByIDs returns the subset of provided account IDs that exist in the accounts table.
// BatchGetByIDs returns the subset of provided account IDs that exist in the accounts table.
Copy link

Copilot AI Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Duplicate comment found. Line 112 duplicates line 113.

Suggested change
// BatchGetByIDs returns the subset of provided account IDs that exist in the accounts table.

Copilot uses AI. Check for mistakes.
func (m *AccountModel) BatchGetByIDs(ctx context.Context, dbTx pgx.Tx, accountIDs []string) ([]string, error) {
if len(accountIDs) == 0 {
return []string{}, nil
}

// Convert string addresses to [][]byte for BYTEA array comparison
byteAddresses := make([][]byte, len(accountIDs))
for i, addr := range accountIDs {
addrBytes, err := types.AddressBytea(addr).Value()
if err != nil {
return nil, fmt.Errorf("converting address %s to bytes: %w", addr, err)
}
if addrBytes == nil {
return nil, fmt.Errorf("address %s converted to nil", addr)
}
byteAddresses[i] = addrBytes.([]byte)
}

const query = `SELECT stellar_address FROM accounts WHERE stellar_address = ANY($1)`
start := time.Now()
var existingAccounts []string
rows, err := dbTx.Query(ctx, query, accountIDs)
rows, err := dbTx.Query(ctx, query, byteAddresses)
if err != nil {
m.MetricsService.IncDBQueryError("BatchGetByIDs", "accounts", utils.GetDBErrorType(err))
return nil, fmt.Errorf("querying accounts by IDs: %w", err)
}
existingAccounts, err = pgx.CollectRows(rows, pgx.RowTo[string])
if err != nil {
defer rows.Close()

var existingAccounts []string
for rows.Next() {
var addrBytes []byte
if err := rows.Scan(&addrBytes); err != nil {
m.MetricsService.IncDBQueryError("BatchGetByIDs", "accounts", utils.GetDBErrorType(err))
return nil, fmt.Errorf("scanning address: %w", err)
}
var addr types.AddressBytea
if err := addr.Scan(addrBytes); err != nil {
return nil, fmt.Errorf("converting address bytes: %w", err)
}
existingAccounts = append(existingAccounts, string(addr))
}
if err := rows.Err(); err != nil {
m.MetricsService.IncDBQueryError("BatchGetByIDs", "accounts", utils.GetDBErrorType(err))
return nil, fmt.Errorf("collecting rows: %w", err)
return nil, fmt.Errorf("iterating rows: %w", err)
}

duration := time.Since(start).Seconds()
m.MetricsService.ObserveDBQueryDuration("BatchGetByIDs", "accounts", duration)
m.MetricsService.ObserveDBBatchSize("BatchGetByIDs", "accounts", len(accountIDs))
Expand All @@ -133,17 +166,17 @@ func (m *AccountModel) BatchGetByIDs(ctx context.Context, dbTx pgx.Tx, accountID
// IsAccountFeeBumpEligible checks whether an account is eligible to have its transaction fee-bumped. Channel Accounts should be
// eligible because some of the transactions will have the channel accounts as the source account (i. e. create account sponsorship).
func (m *AccountModel) IsAccountFeeBumpEligible(ctx context.Context, address string) (bool, error) {
// accounts.stellar_address is BYTEA, channel_accounts.public_key is VARCHAR
// Use separate EXISTS checks to avoid type mismatch in UNION
const query = `
SELECT
EXISTS(
SELECT stellar_address FROM accounts WHERE stellar_address = $1
UNION
SELECT public_key FROM channel_accounts WHERE public_key = $1
)
EXISTS(SELECT 1 FROM accounts WHERE stellar_address = $1)
OR
EXISTS(SELECT 1 FROM channel_accounts WHERE public_key = $2)
`
var exists bool
start := time.Now()
err := m.DB.GetContext(ctx, &exists, query, address)
err := m.DB.GetContext(ctx, &exists, query, types.AddressBytea(address), address)
duration := time.Since(start).Seconds()
m.MetricsService.ObserveDBQueryDuration("IsAccountFeeBumpEligible", "accounts", duration)
if err != nil {
Expand Down
72 changes: 42 additions & 30 deletions internal/data/accounts_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/stellar/wallet-backend/internal/db"
"github.com/stellar/wallet-backend/internal/db/dbtest"
"github.com/stellar/wallet-backend/internal/indexer/types"
"github.com/stellar/wallet-backend/internal/metrics"
)

Expand All @@ -33,6 +34,12 @@ func TestAccountModel_BatchGetByIDs(t *testing.T) {

ctx := context.Background()

// Generate test addresses
account1 := keypair.MustRandom().Address()
account2 := keypair.MustRandom().Address()
nonexistent1 := keypair.MustRandom().Address()
nonexistent2 := keypair.MustRandom().Address()

t.Run("empty input returns empty result", func(t *testing.T) {
var result []string
err := db.RunInPgxTransaction(ctx, dbConnectionPool, func(tx pgx.Tx) error {
Expand All @@ -44,8 +51,9 @@ func TestAccountModel_BatchGetByIDs(t *testing.T) {
})

t.Run("returns existing accounts only", func(t *testing.T) {
// Insert some test accounts
_, err := dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", "account1", "account2")
// Insert some test accounts using StellarAddress for BYTEA conversion
_, err := dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)",
types.AddressBytea(account1), types.AddressBytea(account2))
require.NoError(t, err)

// Test with mix of existing and non-existing accounts
Expand All @@ -55,17 +63,17 @@ func TestAccountModel_BatchGetByIDs(t *testing.T) {

var result []string
err = db.RunInPgxTransaction(ctx, dbConnectionPool, func(tx pgx.Tx) error {
result, err = accountModel.BatchGetByIDs(ctx, tx, []string{"account1", "nonexistent", "account2", "another_nonexistent"})
result, err = accountModel.BatchGetByIDs(ctx, tx, []string{account1, nonexistent1, account2, nonexistent2})
return err
})
require.NoError(t, err)

// Should only return the existing accounts
assert.Len(t, result, 2)
assert.Contains(t, result, "account1")
assert.Contains(t, result, "account2")
assert.NotContains(t, result, "nonexistent")
assert.NotContains(t, result, "another_nonexistent")
assert.Contains(t, result, account1)
assert.Contains(t, result, account2)
assert.NotContains(t, result, nonexistent1)
assert.NotContains(t, result, nonexistent2)
})

t.Run("returns empty when no accounts exist", func(t *testing.T) {
Expand All @@ -79,7 +87,7 @@ func TestAccountModel_BatchGetByIDs(t *testing.T) {

var result []string
err = db.RunInPgxTransaction(ctx, dbConnectionPool, func(tx pgx.Tx) error {
result, err = accountModel.BatchGetByIDs(ctx, tx, []string{"nonexistent1", "nonexistent2"})
result, err = accountModel.BatchGetByIDs(ctx, tx, []string{nonexistent1, nonexistent2})
return err
})
require.NoError(t, err)
Expand Down Expand Up @@ -110,12 +118,11 @@ func TestAccountModel_Insert(t *testing.T) {
err = m.Insert(ctx, address)
require.NoError(t, err)

var dbAddress sql.NullString
err = m.DB.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts WHERE stellar_address = $1", address)
var dbAddress types.AddressBytea
err = m.DB.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts WHERE stellar_address = $1", types.AddressBytea(address))
require.NoError(t, err)

assert.True(t, dbAddress.Valid)
assert.Equal(t, address, dbAddress.String)
assert.Equal(t, address, string(dbAddress))
})

t.Run("duplicate insert fails", func(t *testing.T) {
Expand Down Expand Up @@ -164,7 +171,7 @@ func TestAccountModel_Delete(t *testing.T) {

ctx := context.Background()
address := keypair.MustRandom().Address()
result, insertErr := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address)
result, insertErr := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address))
require.NoError(t, insertErr)
rowAffected, err := result.RowsAffected()
require.NoError(t, err)
Expand All @@ -173,7 +180,7 @@ func TestAccountModel_Delete(t *testing.T) {
err = m.Delete(ctx, address)
require.NoError(t, err)

var dbAddress sql.NullString
var dbAddress types.AddressBytea
err = m.DB.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts LIMIT 1")
assert.ErrorIs(t, err, sql.ErrNoRows)
})
Expand Down Expand Up @@ -218,7 +225,7 @@ func TestAccountModelGet(t *testing.T) {
address := keypair.MustRandom().Address()

// Insert test account
result, err := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address)
result, err := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address))
require.NoError(t, err)
rowAffected, err := result.RowsAffected()
require.NoError(t, err)
Expand All @@ -227,7 +234,7 @@ func TestAccountModelGet(t *testing.T) {
// Test Get function
account, err := m.Get(ctx, address)
require.NoError(t, err)
assert.Equal(t, address, account.StellarAddress)
assert.Equal(t, address, string(account.StellarAddress))
}

func TestAccountModelBatchGetByToIDs(t *testing.T) {
Expand Down Expand Up @@ -255,15 +262,17 @@ func TestAccountModelBatchGetByToIDs(t *testing.T) {
toID2 := int64(2)

// Insert test accounts
_, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2)
_, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)",
types.AddressBytea(address1), types.AddressBytea(address2))
require.NoError(t, err)

// Insert test transactions first
_, err = m.DB.ExecContext(ctx, "INSERT INTO transactions (hash, to_id, envelope_xdr, fee_charged, result_code, meta_xdr, ledger_number, ledger_created_at) VALUES ('tx1', $1, 'env1', 100, 'TransactionResultCodeTxSuccess', 'meta1', 1, NOW()), ('tx2', $2, 'env2', 200, 'TransactionResultCodeTxSuccess', 'meta2', 2, NOW())", toID1, toID2)
require.NoError(t, err)

// Insert test transactions_accounts links
_, err = m.DB.ExecContext(ctx, "INSERT INTO transactions_accounts (tx_to_id, account_id) VALUES ($1, $2), ($3, $4)", toID1, address1, toID2, address2)
_, err = m.DB.ExecContext(ctx, "INSERT INTO transactions_accounts (tx_to_id, account_id) VALUES ($1, $2), ($3, $4)",
toID1, types.AddressBytea(address1), toID2, types.AddressBytea(address2))
require.NoError(t, err)

// Test BatchGetByToIDs function
Expand All @@ -274,7 +283,7 @@ func TestAccountModelBatchGetByToIDs(t *testing.T) {
// Verify accounts are returned with correct to_id
addressSet := make(map[string]int64)
for _, acc := range accounts {
addressSet[acc.StellarAddress] = acc.ToID
addressSet[string(acc.StellarAddress)] = acc.ToID
}
assert.Equal(t, toID1, addressSet[address1])
assert.Equal(t, toID2, addressSet[address2])
Expand Down Expand Up @@ -304,8 +313,9 @@ func TestAccountModelBatchGetByOperationIDs(t *testing.T) {
operationID1 := int64(123)
operationID2 := int64(456)

// Insert test accounts
_, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2)
// Insert test accounts (stellar_address is BYTEA)
_, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)",
types.AddressBytea(address1), types.AddressBytea(address2))
require.NoError(t, err)

// Insert test transactions first
Expand All @@ -316,8 +326,9 @@ func TestAccountModelBatchGetByOperationIDs(t *testing.T) {
_, err = m.DB.ExecContext(ctx, "INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES ($1, 'PAYMENT', 'xdr1', 'op_success', true, 1, NOW()), ($2, 'PAYMENT', 'xdr2', 'op_success', true, 2, NOW())", operationID1, operationID2)
require.NoError(t, err)

// Insert test operations_accounts links
_, err = m.DB.ExecContext(ctx, "INSERT INTO operations_accounts (operation_id, account_id) VALUES ($1, $2), ($3, $4)", operationID1, address1, operationID2, address2)
// Insert test operations_accounts links (account_id is BYTEA)
_, err = m.DB.ExecContext(ctx, "INSERT INTO operations_accounts (operation_id, account_id) VALUES ($1, $2), ($3, $4)",
operationID1, types.AddressBytea(address1), operationID2, types.AddressBytea(address2))
require.NoError(t, err)

// Test BatchGetByOperationID function
Expand All @@ -328,7 +339,7 @@ func TestAccountModelBatchGetByOperationIDs(t *testing.T) {
// Verify accounts are returned with correct operation_id
addressSet := make(map[string]int64)
for _, acc := range accounts {
addressSet[acc.StellarAddress] = acc.OperationID
addressSet[string(acc.StellarAddress)] = acc.OperationID
}
assert.Equal(t, operationID1, addressSet[address1])
assert.Equal(t, operationID2, addressSet[address2])
Expand Down Expand Up @@ -358,7 +369,7 @@ func TestAccountModel_IsAccountFeeBumpEligible(t *testing.T) {
require.NoError(t, err)
assert.False(t, isFeeBumpEligible)

result, err := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", address)
result, err := m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", types.AddressBytea(address))
require.NoError(t, err)
rowAffected, err := result.RowsAffected()
require.NoError(t, err)
Expand Down Expand Up @@ -395,8 +406,9 @@ func TestAccountModelBatchGetByStateChangeIDs(t *testing.T) {
stateChangeOrder1 := int64(1)
stateChangeOrder2 := int64(1)

// Insert test accounts
_, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)", address1, address2)
// Insert test accounts (stellar_address is BYTEA)
_, err = m.DB.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1), ($2)",
types.AddressBytea(address1), types.AddressBytea(address2))
require.NoError(t, err)

// Insert test transactions first
Expand All @@ -407,15 +419,15 @@ func TestAccountModelBatchGetByStateChangeIDs(t *testing.T) {
_, err = m.DB.ExecContext(ctx, "INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) VALUES (4097, 'PAYMENT', 'xdr1', 'op_success', true, 1, NOW()), (8193, 'PAYMENT', 'xdr2', 'op_success', true, 2, NOW())")
require.NoError(t, err)

// Insert test state changes that reference the accounts
// Insert test state changes that reference the accounts (state_changes.account_id is TEXT)
_, err = m.DB.ExecContext(ctx, `
INSERT INTO state_changes (
to_id, state_change_order, state_change_category, ledger_created_at,
ledger_number, account_id, operation_id
) VALUES
($1, $2, 'BALANCE', NOW(), 1, $3, 4097),
($4, $5, 'BALANCE', NOW(), 2, $6, 8193)
`, toID1, stateChangeOrder1, address1, toID2, stateChangeOrder2, address2)
`, toID1, stateChangeOrder1, types.AddressBytea(address1), toID2, stateChangeOrder2, types.AddressBytea(address2))
require.NoError(t, err)

// Test BatchGetByStateChangeIDs function
Expand All @@ -429,7 +441,7 @@ func TestAccountModelBatchGetByStateChangeIDs(t *testing.T) {
// Verify accounts are returned with correct state_change_id (format: to_id-operation_id-state_change_order)
addressSet := make(map[string]string)
for _, acc := range accounts {
addressSet[acc.StellarAddress] = acc.StateChangeID
addressSet[string(acc.StellarAddress)] = acc.StateChangeID
}
assert.Equal(t, "4096-4097-1", addressSet[address1])
assert.Equal(t, "8192-8193-1", addressSet[address2])
Expand Down
Loading
Loading