From a42c92695c325ee834e6f76194a20410e4da1219 Mon Sep 17 00:00:00 2001 From: Ilnur Date: Fri, 14 Feb 2025 00:49:37 +0400 Subject: [PATCH 01/12] update builder image --- Dockerfile | 6 +++++- local.Dockerfile | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3c4d339b0..69facf9b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ -FROM --platform=$BUILDPLATFORM golang:1.21-alpine3.17 AS build-env +FROM --platform=$BUILDPLATFORM golang:1.22-alpine3.20 AS build-env RUN apk add --update --no-cache curl make git libc-dev bash gcc linux-headers eudev-dev ARG TARGETARCH ARG BUILDARCH +ARG GITHUB_USER=$GITHUB_USER +ARG GITHUB_PASS=$GITHUB_PASS RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \ wget -c https://musl.cc/aarch64-linux-musl-cross.tgz -O - | tar -xzvv --strip-components 1 -C /usr; \ @@ -13,6 +15,8 @@ RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \ ADD . . +RUN echo "machine github.com login $GITHUB_USER password $GITHUB_PASS" > ~/.netrc + RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \ export CC=aarch64-linux-musl-gcc CXX=aarch64-linux-musl-g++;\ elif [ "${TARGETARCH}" = "amd64" ] && [ "${BUILDARCH}" != "amd64" ]; then \ diff --git a/local.Dockerfile b/local.Dockerfile index 6cd45dd2c..473840c3b 100644 --- a/local.Dockerfile +++ b/local.Dockerfile @@ -1,9 +1,13 @@ -FROM golang:1-alpine3.17 AS build-env +FROM golang:1.22-alpine3.20 AS build-env + +ARG GITHUB_USER=$GITHUB_USER +ARG GITHUB_PASS=$GITHUB_PASS RUN apk add --update --no-cache curl make git libc-dev bash gcc linux-headers eudev-dev ADD . . +RUN echo "machine github.com login $GITHUB_USER password $GITHUB_PASS" > ~/.netrc RUN CGO_ENABLED=1 LDFLAGS='-linkmode external -extldflags "-static"' make install # Use minimal busybox from infra-toolkit image for final scratch image @@ -41,7 +45,7 @@ RUN ln sh pwd && \ rm ln rm # Install chain binaries -COPY --from=build-env /bin/rly /bin +COPY --from=build-env /go/bin/rly /bin # Install trusted CA certificates COPY --from=busybox-min /etc/ssl/cert.pem /etc/ssl/cert.pem From 28387ee81ccd5786b378b9ba27bbac4b05376172 Mon Sep 17 00:00:00 2001 From: Ilnur Date: Fri, 14 Feb 2025 00:50:35 +0400 Subject: [PATCH 02/12] modify query cycle in avalanche chain processor --- .../avalanche/avalanche_chain_processor.go | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/relayer/chains/avalanche/avalanche_chain_processor.go b/relayer/chains/avalanche/avalanche_chain_processor.go index 93bcc48df..59a243b23 100644 --- a/relayer/chains/avalanche/avalanche_chain_processor.go +++ b/relayer/chains/avalanche/avalanche_chain_processor.go @@ -75,6 +75,8 @@ type AvalancheChainProcessor struct { // map of channel ID to connection ID channelConnections map[string]string + + blockTries map[int64]int } func NewAvalancheChainProcessor(log *zap.Logger, provider *AvalancheProvider) *AvalancheChainProcessor { @@ -86,6 +88,7 @@ func NewAvalancheChainProcessor(log *zap.Logger, provider *AvalancheProvider) *A channelStateCache: make(processor.ChannelStateCache), connectionClients: make(map[string]string), channelConnections: make(map[string]string), + blockTries: make(map[int64]int), } } @@ -294,6 +297,22 @@ func (acp *AvalancheChainProcessor) queryCycle(ctx context.Context, persistence chainID := acp.chainProvider.ChainId() + if persistence.latestQueriedBlock != 0 { + if ibcHeader, err := acp.chainProvider.QueryIBCHeader(context.Background(), persistence.latestQueriedBlock); err == nil { + if avaHeader, ok := any(ibcHeader).(AvalancheIBCHeader); ok { + latestHeader = avaHeader + ibcHeaderCache[uint64(persistence.latestQueriedBlock)] = avaHeader + } + } + + if blockRes, err := acp.chainProvider.ethClient.BlockByNumber(context.Background(), big.NewInt(persistence.latestQueriedBlock)); err == nil { + acp.latestBlock = provider.LatestBlock{ + Height: uint64(persistence.latestQueriedBlock), + Time: time.Unix(int64(blockRes.Time()), 0), + } + } + } + for i := persistence.latestQueriedBlock + 1; i <= persistence.latestHeight; i++ { var eg errgroup.Group var blockRes *types.Block @@ -398,18 +417,32 @@ func (acp *AvalancheChainProcessor) queryCycle(ctx context.Context, persistence newLatestQueriedBlock = i } - if newLatestQueriedBlock == persistence.latestQueriedBlock /*&& !firstTimeInSync */ { - return nil + stuck := false + tries, ok := acp.blockTries[persistence.latestQueriedBlock] + if ok { + if tries+1 >= blockMaxRetries { + stuck = true + tries = 0 + } + } + acp.blockTries[persistence.latestQueriedBlock] = tries + 1 + + if newLatestQueriedBlock == persistence.latestQueriedBlock && !acp.inSync { + if !stuck { + return nil + } } if !ppChanged { - if firstTimeInSync { + if firstTimeInSync || !acp.inSync { for _, pp := range acp.pathProcessors { pp.ProcessBacklogIfReady() } } - return nil + if !stuck { + return nil + } } for _, pp := range acp.pathProcessors { From 68cf7d92597079a00f050481e71c56802799253d Mon Sep 17 00:00:00 2001 From: Ilnur Date: Fri, 14 Feb 2025 00:51:34 +0400 Subject: [PATCH 03/12] modify shouldSendConnectionMessage in pathEndRuntime --- relayer/processor/path_end_runtime.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/relayer/processor/path_end_runtime.go b/relayer/processor/path_end_runtime.go index 14a0357e7..a4e8c061a 100644 --- a/relayer/processor/path_end_runtime.go +++ b/relayer/processor/path_end_runtime.go @@ -60,6 +60,8 @@ type pathEndRuntime struct { finishedProcessing chan messageToTrack retryCount uint64 + + blockRetries map[uint64]int } func newPathEndRuntime(log *zap.Logger, pathEnd PathEnd, metrics *PrometheusMetrics) *pathEndRuntime { @@ -83,6 +85,7 @@ func newPathEndRuntime(log *zap.Logger, pathEnd PathEnd, metrics *PrometheusMetr clientICQProcessing: newClientICQProcessingCache(), connSubscribers: make(map[string][]func(provider.ConnectionInfo)), metrics: metrics, + blockRetries: make(map[uint64]int), } } @@ -734,11 +737,20 @@ func (pathEnd *pathEndRuntime) shouldSendChannelMessage(message channelIBCMessag } if message.info.Height >= counterparty.latestBlock.Height { + tries := pathEnd.blockRetries[message.info.Height] + stuck := false + if tries > 5 { + stuck = true + tries = 0 + } + pathEnd.blockRetries[message.info.Height] = tries + 1 pathEnd.log.Debug("Waiting to relay channel message until counterparty height has incremented", zap.Inline(channelKey), zap.String("event_type", eventType), ) - return false + if !stuck { + return false + } } msgProcessCache, ok := pathEnd.channelProcessing[eventType] if !ok { From e03162e90a1ef949c35f5098efcb01bd24e63ab3 Mon Sep 17 00:00:00 2001 From: Ilnur Date: Tue, 18 Feb 2025 12:00:40 +0300 Subject: [PATCH 04/12] fix MsgTransfer --- relayer/chains/avalanche/tx.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/relayer/chains/avalanche/tx.go b/relayer/chains/avalanche/tx.go index 2f3df31e4..01641d74e 100644 --- a/relayer/chains/avalanche/tx.go +++ b/relayer/chains/avalanche/tx.go @@ -866,16 +866,11 @@ func (a AvalancheProvider) MsgTransfer(dstAddr string, amount sdk.Coin, info pro return nil, err } - packetData, _ := json.Marshal(FungibleTokenPacketData{ - Denom: amount.Denom, - Amount: amount.Amount.String(), - Sender: a.txAuth.From.Hex(), - Receiver: dstAddr, - }) - msg, err := abi.Pack( "transfer", - big.NewInt(0), + amount.Denom, + amount.Amount.BigInt(), + []byte(dstAddr), info.SourcePort, info.SourceChannel, ics20banktransferapp.Height{ @@ -883,7 +878,7 @@ func (a AvalancheProvider) MsgTransfer(dstAddr string, amount sdk.Coin, info pro RevisionNumber: big.NewInt(int64(info.TimeoutHeight.RevisionNumber)), }, big.NewInt(int64(info.TimeoutTimestamp)), - packetData, + [32]byte{}, ) if err != nil { return nil, err From 610c962060d2dc66e0edc0252d031677e8558aa9 Mon Sep 17 00:00:00 2001 From: Ilnur Date: Tue, 18 Feb 2025 12:08:44 +0300 Subject: [PATCH 05/12] fix shouldSendPacketMessage in case when blocks didn't increment --- relayer/processor/path_end_runtime.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/relayer/processor/path_end_runtime.go b/relayer/processor/path_end_runtime.go index a4e8c061a..3647f7834 100644 --- a/relayer/processor/path_end_runtime.go +++ b/relayer/processor/path_end_runtime.go @@ -548,6 +548,13 @@ func (pathEnd *pathEndRuntime) shouldSendPacketMessage(message packetIBCMessage, } if message.info.Height >= pathEndForHeight.latestBlock.Height { + tries := pathEnd.blockRetries[message.info.Height] + stuck := false + if tries > 5 { + stuck = true + tries = 0 + } + pathEnd.blockRetries[message.info.Height] = tries + 1 pathEnd.log.Debug("Waiting to relay packet message until counterparty height has incremented", zap.String("event_type", eventType), zap.Uint64("sequence", sequence), @@ -555,7 +562,9 @@ func (pathEnd *pathEndRuntime) shouldSendPacketMessage(message packetIBCMessage, zap.Uint64("counterparty_height", counterparty.latestBlock.Height), zap.Inline(k), ) - return false + if !stuck { + return false + } } if !pathEnd.channelStateCache[k].Open { // channel is not open, do not send From 12a6a143686047fad4d6a197d9a6862cc246d674 Mon Sep 17 00:00:00 2001 From: Vasyl Naumenko Date: Fri, 28 Feb 2025 17:13:25 +0200 Subject: [PATCH 06/12] added more error details --- relayer/chains/avalanche/tx.go | 8 +++++++- relayer/processor/path_processor.go | 6 ++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/relayer/chains/avalanche/tx.go b/relayer/chains/avalanche/tx.go index 01641d74e..e46b6eff5 100644 --- a/relayer/chains/avalanche/tx.go +++ b/relayer/chains/avalanche/tx.go @@ -56,7 +56,7 @@ func (a AvalancheProvider) SendMessage(ctx context.Context, msg provider.Relayer func (a AvalancheProvider) broadcastTx( ctx context.Context, // context for tx broadcast signedTx *evmtypes.Transaction, - asyncCtx context.Context, // context for async wait for block inclusion after successful tx broadcast + asyncCtx context.Context, // context for async wait for block inclusion after successful tx broadcast asyncCallbacks []func(*provider.RelayerTxResponse, error), // callback for success/fail of the wait for block inclusion ) error { err := a.ethClient.SendTransaction(ctx, signedTx) @@ -210,6 +210,12 @@ func (a AvalancheProvider) SendMessagesToMempool(ctx context.Context, msgs []pro zap.Error(err), ) if err != nil { + a.log.Info("Avalanche tx broadcast failed", + zap.Binary("signedTx.Data()", signedTx.Data()), + zap.Uint64("signedTx.Nonce()", signedTx.Nonce()), + zap.String("(signedTx.ChainId()", signedTx.ChainId().String()), + zap.Uint64("(signedTx.Size()", signedTx.Size()), + ) return err } } diff --git a/relayer/processor/path_processor.go b/relayer/processor/path_processor.go index b492034c9..11c82de89 100644 --- a/relayer/processor/path_processor.go +++ b/relayer/processor/path_processor.go @@ -7,8 +7,9 @@ import ( chantypes "github.com/cosmos/ibc-go/v8/modules/core/04-channel/types" ibcexported "github.com/cosmos/ibc-go/v8/modules/core/exported" - "github.com/cosmos/relayer/v2/relayer/provider" "go.uber.org/zap" + + "github.com/cosmos/relayer/v2/relayer/provider" ) const ( @@ -444,7 +445,8 @@ func (pp *PathProcessor) Run(ctx context.Context, cancel func()) { // process latest message cache state from both pathEnds if err := pp.processLatestMessages(ctx, cancel); err != nil { - pp.log.Error("Failed to process latest messages", zap.Error(err)) + // todo uncomment if needed + // pp.log.Error("Failed to process latest messages", zap.Error(err)) // in case of IBC message send errors, schedule retry after durationErrorRetry if retryTimer != nil { From 6f60f233644477008d1656dd06b75b26fc606e51 Mon Sep 17 00:00:00 2001 From: Vasyl Naumenko Date: Fri, 28 Feb 2025 17:21:39 +0200 Subject: [PATCH 07/12] added more error details --- relayer/chains/avalanche/tx.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/relayer/chains/avalanche/tx.go b/relayer/chains/avalanche/tx.go index e46b6eff5..a5086ab77 100644 --- a/relayer/chains/avalanche/tx.go +++ b/relayer/chains/avalanche/tx.go @@ -56,7 +56,7 @@ func (a AvalancheProvider) SendMessage(ctx context.Context, msg provider.Relayer func (a AvalancheProvider) broadcastTx( ctx context.Context, // context for tx broadcast signedTx *evmtypes.Transaction, - asyncCtx context.Context, // context for async wait for block inclusion after successful tx broadcast + asyncCtx context.Context, // context for async wait for block inclusion after successful tx broadcast asyncCallbacks []func(*provider.RelayerTxResponse, error), // callback for success/fail of the wait for block inclusion ) error { err := a.ethClient.SendTransaction(ctx, signedTx) @@ -210,7 +210,13 @@ func (a AvalancheProvider) SendMessagesToMempool(ctx context.Context, msgs []pro zap.Error(err), ) if err != nil { + msgBytes, err2 := msgs[i].MsgBytes() + if err2 != nil { + a.log.Error("Failed to get msg bytes", zap.Error(err)) + } a.log.Info("Avalanche tx broadcast failed", + zap.String("msgs[i].Type()", msgs[i].Type()), + zap.Binary("msgs[i].Type()", msgBytes), zap.Binary("signedTx.Data()", signedTx.Data()), zap.Uint64("signedTx.Nonce()", signedTx.Nonce()), zap.String("(signedTx.ChainId()", signedTx.ChainId().String()), From 7c1e607266b66919d79154f01683f3aaea1d30b9 Mon Sep 17 00:00:00 2001 From: Vasyl Naumenko Date: Fri, 28 Feb 2025 17:22:20 +0200 Subject: [PATCH 08/12] added more error details --- relayer/chains/avalanche/tx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relayer/chains/avalanche/tx.go b/relayer/chains/avalanche/tx.go index a5086ab77..03cdc9e34 100644 --- a/relayer/chains/avalanche/tx.go +++ b/relayer/chains/avalanche/tx.go @@ -216,7 +216,7 @@ func (a AvalancheProvider) SendMessagesToMempool(ctx context.Context, msgs []pro } a.log.Info("Avalanche tx broadcast failed", zap.String("msgs[i].Type()", msgs[i].Type()), - zap.Binary("msgs[i].Type()", msgBytes), + zap.Binary("msgs[i].Bytes()", msgBytes), zap.Binary("signedTx.Data()", signedTx.Data()), zap.Uint64("signedTx.Nonce()", signedTx.Nonce()), zap.String("(signedTx.ChainId()", signedTx.ChainId().String()), From 3e33ced9b71aa01d959419e42fb47e407026ca42 Mon Sep 17 00:00:00 2001 From: Vasyl Naumenko Date: Fri, 28 Feb 2025 18:27:54 +0200 Subject: [PATCH 09/12] added more error details --- relayer/chains/avalanche/tx.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/relayer/chains/avalanche/tx.go b/relayer/chains/avalanche/tx.go index 03cdc9e34..ed064e343 100644 --- a/relayer/chains/avalanche/tx.go +++ b/relayer/chains/avalanche/tx.go @@ -210,6 +210,7 @@ func (a AvalancheProvider) SendMessagesToMempool(ctx context.Context, msgs []pro zap.Error(err), ) if err != nil { + r, s, v := signedTx.RawSignatureValues() msgBytes, err2 := msgs[i].MsgBytes() if err2 != nil { a.log.Error("Failed to get msg bytes", zap.Error(err)) @@ -221,6 +222,12 @@ func (a AvalancheProvider) SendMessagesToMempool(ctx context.Context, msgs []pro zap.Uint64("signedTx.Nonce()", signedTx.Nonce()), zap.String("(signedTx.ChainId()", signedTx.ChainId().String()), zap.Uint64("(signedTx.Size()", signedTx.Size()), + zap.String("r.String()", r.String()), + zap.String("r.Bytes()", hexutil.Encode(r.Bytes())), + zap.String("s.String()", s.String()), + zap.String("s.Bytes()", hexutil.Encode(s.Bytes())), + zap.String("v.String()", v.String()), + zap.String("v.Bytes()", hexutil.Encode(v.Bytes())), ) return err } From c10516150fafa6e7e72d3eef60974466ceb6d688 Mon Sep 17 00:00:00 2001 From: Vasyl Naumenko Date: Sat, 1 Mar 2025 19:52:24 +0200 Subject: [PATCH 10/12] override unbounding period --- relayer/client.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/relayer/client.go b/relayer/client.go index 00acb8eab..8f41cd837 100644 --- a/relayer/client.go +++ b/relayer/client.go @@ -10,9 +10,10 @@ import ( clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types" ibcexported "github.com/cosmos/ibc-go/v8/modules/core/exported" tmclient "github.com/cosmos/ibc-go/v8/modules/light-clients/07-tendermint" - "github.com/cosmos/relayer/v2/relayer/provider" "go.uber.org/zap" "golang.org/x/sync/errgroup" + + "github.com/cosmos/relayer/v2/relayer/provider" ) // CreateClients creates clients for src on dst and dst on src if the client ids are unspecified. @@ -64,8 +65,7 @@ func (c *Chain) CreateClients(ctx context.Context, } // overriding the unbonding period should only be possible when creating single clients at a time (CreateClient) - var overrideUnbondingPeriod = time.Duration(0) - + var overrideUnbondingPeriod = 20000 * time.Hour // 20000 Hours = 833 Days var clientSrc, clientDst string eg, egCtx := errgroup.WithContext(ctx) eg.Go(func() error { @@ -196,10 +196,10 @@ func CreateClient( // Check if an identical light client already exists on the src chain which matches the // proposed new client state from dst. // TODO - //clientID, err = findMatchingClient(ctx, src, dst, clientState) - //if err != nil { + // clientID, err = findMatchingClient(ctx, src, dst, clientState) + // if err != nil { // return "", fmt.Errorf("failed to find a matching client for the new client state: %w", err) - //} + // } } if clientID != "" && !override { From 3539645aa2ab05830341d6f1942a1e6248b6d5b4 Mon Sep 17 00:00:00 2001 From: Vasyl Naumenko Date: Sun, 2 Mar 2025 14:30:08 +0200 Subject: [PATCH 11/12] unbounding period --- relayer/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relayer/client.go b/relayer/client.go index 8f41cd837..87baeea6d 100644 --- a/relayer/client.go +++ b/relayer/client.go @@ -65,7 +65,7 @@ func (c *Chain) CreateClients(ctx context.Context, } // overriding the unbonding period should only be possible when creating single clients at a time (CreateClient) - var overrideUnbondingPeriod = 20000 * time.Hour // 20000 Hours = 833 Days + var overrideUnbondingPeriod = time.Duration(0) // override should be smaller than unbonding period 504h var clientSrc, clientDst string eg, egCtx := errgroup.WithContext(ctx) eg.Go(func() error { From 238699c0dc691d9d15fe1d78d0cec89c6249742b Mon Sep 17 00:00:00 2001 From: Vasyl Naumenko Date: Thu, 6 Mar 2025 17:02:05 +0200 Subject: [PATCH 12/12] enable client state update --- relayer/processor/message_processor.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/relayer/processor/message_processor.go b/relayer/processor/message_processor.go index 49bc6b57b..78cb0de4d 100644 --- a/relayer/processor/message_processor.go +++ b/relayer/processor/message_processor.go @@ -13,6 +13,7 @@ import ( ibcexported "github.com/cosmos/ibc-go/v8/modules/core/exported" tmclient "github.com/cosmos/ibc-go/v8/modules/light-clients/07-tendermint" avaclient "github.com/cosmos/ibc-go/v8/modules/light-clients/14-avalanche" + "github.com/cosmos/relayer/v2/relayer/provider" "go.uber.org/zap" @@ -105,9 +106,7 @@ func (mp *messageProcessor) processMessages( // Localhost IBC does not permit client updates if !isLocalhostClient(src.clientState.ClientID, dst.clientState.ClientID) { var err error - needsClientUpdate = false - // TODO: uncomment next line - // needsClientUpdate, err = mp.shouldUpdateClientNow(ctx, src, dst) + needsClientUpdate, err = mp.shouldUpdateClientNow(ctx, src, dst) if err != nil { return err } @@ -142,7 +141,7 @@ func (mp *messageProcessor) shouldUpdateClientNow(ctx context.Context, src, dst h, err := src.chainProvider.QueryIBCHeader(ctx, int64(dst.clientState.ConsensusHeight.RevisionHeight)) if err != nil { // TODO - //return false, nil + // return false, nil return false, fmt.Errorf("failed to get header height: %w", err) } consensusHeightTime = time.Unix(0, int64(h.ConsensusState().GetTimestamp())) @@ -490,7 +489,7 @@ func (mp *messageProcessor) sendBatchMessages( } callbacks := []func(rtr *provider.RelayerTxResponse, err error){callback} - //During testing, this adds a callback so our test case can inspect the TX results + // During testing, this adds a callback so our test case can inspect the TX results if PathProcMessageCollector != nil { testCallback := func(rtr *provider.RelayerTxResponse, err error) { msgResult := &PathProcessorMessageResp{ @@ -577,7 +576,7 @@ func (mp *messageProcessor) sendSingleMessage( callbacks = append(callbacks, callback) - //During testing, this adds a callback so our test case can inspect the TX results + // During testing, this adds a callback so our test case can inspect the TX results if PathProcMessageCollector != nil { testCallback := func(rtr *provider.RelayerTxResponse, err error) { msgResult := &PathProcessorMessageResp{