upgrade golang version and modify logic in acp and path runtime

Dockerfile

@@ -1,9 +1,11 @@
-FROM --platform=$BUILDPLATFORM golang:1.21-alpine3.17 AS build-env
+FROM --platform=$BUILDPLATFORM golang:1.22-alpine3.20 AS build-env
 
 RUN apk add --update --no-cache curl make git libc-dev bash gcc linux-headers eudev-dev
 
 ARG TARGETARCH
 ARG BUILDARCH
+ARG GITHUB_USER=$GITHUB_USER
+ARG GITHUB_PASS=$GITHUB_PASS
 
 RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
     wget -c https://musl.cc/aarch64-linux-musl-cross.tgz -O - | tar -xzvv --strip-components 1 -C /usr; \
@@ -13,6 +15,8 @@ RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
 
 ADD . .
 
+RUN echo "machine github.com login $GITHUB_USER password $GITHUB_PASS" > ~/.netrc
+
 RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
     export CC=aarch64-linux-musl-gcc CXX=aarch64-linux-musl-g++;\
     elif [ "${TARGETARCH}" = "amd64" ] && [ "${BUILDARCH}" != "amd64" ]; then \

local.Dockerfile

@@ -1,9 +1,13 @@
-FROM golang:1-alpine3.17 AS build-env
+FROM golang:1.22-alpine3.20 AS build-env
+
+ARG GITHUB_USER=$GITHUB_USER
+ARG GITHUB_PASS=$GITHUB_PASS
 
 RUN apk add --update --no-cache curl make git libc-dev bash gcc linux-headers eudev-dev
 
 ADD . .
 
+RUN echo "machine github.com login $GITHUB_USER password $GITHUB_PASS" > ~/.netrc
 RUN CGO_ENABLED=1 LDFLAGS='-linkmode external -extldflags "-static"' make install
 
 # Use minimal busybox from infra-toolkit image for final scratch image
@@ -41,7 +45,7 @@ RUN ln sh pwd && \
     rm ln rm
 
 # Install chain binaries
-COPY --from=build-env /bin/rly /bin
+COPY --from=build-env /go/bin/rly /bin
 
 # Install trusted CA certificates
 COPY --from=busybox-min /etc/ssl/cert.pem /etc/ssl/cert.pem

relayer/chains/avalanche/avalanche_chain_processor.go

@@ -75,6 +75,8 @@ type AvalancheChainProcessor struct {
 
 	// map of channel ID to connection ID
 	channelConnections map[string]string
+
+	blockTries map[int64]int
 }
 
 func NewAvalancheChainProcessor(log *zap.Logger, provider *AvalancheProvider) *AvalancheChainProcessor {
@@ -86,6 +88,7 @@ func NewAvalancheChainProcessor(log *zap.Logger, provider *AvalancheProvider) *A
 		channelStateCache:    make(processor.ChannelStateCache),
 		connectionClients:    make(map[string]string),
 		channelConnections:   make(map[string]string),
+		blockTries:           make(map[int64]int),
 	}
 }
 
@@ -294,6 +297,22 @@ func (acp *AvalancheChainProcessor) queryCycle(ctx context.Context, persistence
 
 	chainID := acp.chainProvider.ChainId()
 
+	if persistence.latestQueriedBlock != 0 {
+		if ibcHeader, err := acp.chainProvider.QueryIBCHeader(context.Background(), persistence.latestQueriedBlock); err == nil {
+			if avaHeader, ok := any(ibcHeader).(AvalancheIBCHeader); ok {
+				latestHeader = avaHeader
+				ibcHeaderCache[uint64(persistence.latestQueriedBlock)] = avaHeader
+			}
+		}
+
+		if blockRes, err := acp.chainProvider.ethClient.BlockByNumber(context.Background(), big.NewInt(persistence.latestQueriedBlock)); err == nil {
+			acp.latestBlock = provider.LatestBlock{
+				Height: uint64(persistence.latestQueriedBlock),
+				Time:   time.Unix(int64(blockRes.Time()), 0),
+			}
+		}
+	}
+
 	for i := persistence.latestQueriedBlock + 1; i <= persistence.latestHeight; i++ {
 		var eg errgroup.Group
 		var blockRes *types.Block
@@ -398,18 +417,32 @@ func (acp *AvalancheChainProcessor) queryCycle(ctx context.Context, persistence
 		newLatestQueriedBlock = i
 	}
 
-	if newLatestQueriedBlock == persistence.latestQueriedBlock /*&& !firstTimeInSync */ {
-		return nil
+	stuck := false
+	tries, ok := acp.blockTries[persistence.latestQueriedBlock]
+	if ok {
+		if tries+1 >= blockMaxRetries {
+			stuck = true
+			tries = 0
+		}
+	}
+	acp.blockTries[persistence.latestQueriedBlock] = tries + 1
+
+	if newLatestQueriedBlock == persistence.latestQueriedBlock && !acp.inSync {
+		if !stuck {
+			return nil
+		}
 	}
 
 	if !ppChanged {
-		if firstTimeInSync {
+		if firstTimeInSync || !acp.inSync {
 			for _, pp := range acp.pathProcessors {
 				pp.ProcessBacklogIfReady()
 			}
 		}
 
-		return nil
+		if !stuck {
+			return nil
+		}
 	}
 
 	for _, pp := range acp.pathProcessors {

relayer/chains/avalanche/tx.go

@@ -210,6 +210,25 @@ func (a AvalancheProvider) SendMessagesToMempool(ctx context.Context, msgs []pro
 			zap.Error(err),
 		)
 		if err != nil {
+			r, s, v := signedTx.RawSignatureValues()
+			msgBytes, err2 := msgs[i].MsgBytes()
+			if err2 != nil {
+				a.log.Error("Failed to get msg bytes", zap.Error(err))
+			}
+			a.log.Info("Avalanche tx broadcast failed",
+				zap.String("msgs[i].Type()", msgs[i].Type()),
+				zap.Binary("msgs[i].Bytes()", msgBytes),
+				zap.Binary("signedTx.Data()", signedTx.Data()),
+				zap.Uint64("signedTx.Nonce()", signedTx.Nonce()),
+				zap.String("(signedTx.ChainId()", signedTx.ChainId().String()),
+				zap.Uint64("(signedTx.Size()", signedTx.Size()),
+				zap.String("r.String()", r.String()),
+				zap.String("r.Bytes()", hexutil.Encode(r.Bytes())),
+				zap.String("s.String()", s.String()),
+				zap.String("s.Bytes()", hexutil.Encode(s.Bytes())),
+				zap.String("v.String()", v.String()),
+				zap.String("v.Bytes()", hexutil.Encode(v.Bytes())),
+			)
 			return err
 		}
 	}
@@ -866,24 +885,19 @@ func (a AvalancheProvider) MsgTransfer(dstAddr string, amount sdk.Coin, info pro
 		return nil, err
 	}
 
-	packetData, _ := json.Marshal(FungibleTokenPacketData{
-		Denom:    amount.Denom,
-		Amount:   amount.Amount.String(),
-		Sender:   a.txAuth.From.Hex(),
-		Receiver: dstAddr,
-	})
-
 	msg, err := abi.Pack(
 		"transfer",
-		big.NewInt(0),
+		amount.Denom,
+		amount.Amount.BigInt(),
+		[]byte(dstAddr),
 		info.SourcePort,
 		info.SourceChannel,
 		ics20banktransferapp.Height{
 			RevisionHeight: big.NewInt(int64(info.TimeoutHeight.RevisionHeight)),
 			RevisionNumber: big.NewInt(int64(info.TimeoutHeight.RevisionNumber)),
 		},
 		big.NewInt(int64(info.TimeoutTimestamp)),
-		packetData,
+		[32]byte{},
 	)
 	if err != nil {
 		return nil, err

relayer/client.go

@@ -10,9 +10,10 @@ import (
 	clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types"
 	ibcexported "github.com/cosmos/ibc-go/v8/modules/core/exported"
 	tmclient "github.com/cosmos/ibc-go/v8/modules/light-clients/07-tendermint"
-	"github.com/cosmos/relayer/v2/relayer/provider"
 	"go.uber.org/zap"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cosmos/relayer/v2/relayer/provider"
 )
 
 // CreateClients creates clients for src on dst and dst on src if the client ids are unspecified.
@@ -64,8 +65,7 @@ func (c *Chain) CreateClients(ctx context.Context,
 	}
 
 	// overriding the unbonding period should only be possible when creating single clients at a time (CreateClient)
-	var overrideUnbondingPeriod = time.Duration(0)
-
+	var overrideUnbondingPeriod = time.Duration(0) // override should be smaller than unbonding period 504h
 	var clientSrc, clientDst string
 	eg, egCtx := errgroup.WithContext(ctx)
 	eg.Go(func() error {
@@ -196,10 +196,10 @@ func CreateClient(
 		// Check if an identical light client already exists on the src chain which matches the
 		// proposed new client state from dst.
 		// TODO
-		//clientID, err = findMatchingClient(ctx, src, dst, clientState)
-		//if err != nil {
+		// clientID, err = findMatchingClient(ctx, src, dst, clientState)
+		// if err != nil {
 		//	return "", fmt.Errorf("failed to find a matching client for the new client state: %w", err)
-		//}
+		// }
 	}
 
 	if clientID != "" && !override {

relayer/processor/message_processor.go

@@ -13,6 +13,7 @@ import (
 	ibcexported "github.com/cosmos/ibc-go/v8/modules/core/exported"
 	tmclient "github.com/cosmos/ibc-go/v8/modules/light-clients/07-tendermint"
 	avaclient "github.com/cosmos/ibc-go/v8/modules/light-clients/14-avalanche"
+
 	"github.com/cosmos/relayer/v2/relayer/provider"
 
 	"go.uber.org/zap"
@@ -105,9 +106,7 @@ func (mp *messageProcessor) processMessages(
 	// Localhost IBC does not permit client updates
 	if !isLocalhostClient(src.clientState.ClientID, dst.clientState.ClientID) {
 		var err error
-		needsClientUpdate = false
-		// TODO: uncomment next line
-		// needsClientUpdate, err = mp.shouldUpdateClientNow(ctx, src, dst)
+		needsClientUpdate, err = mp.shouldUpdateClientNow(ctx, src, dst)
 		if err != nil {
 			return err
 		}
@@ -142,7 +141,7 @@ func (mp *messageProcessor) shouldUpdateClientNow(ctx context.Context, src, dst
 		h, err := src.chainProvider.QueryIBCHeader(ctx, int64(dst.clientState.ConsensusHeight.RevisionHeight))
 		if err != nil {
 			// TODO
-			//return false, nil
+			// return false, nil
 			return false, fmt.Errorf("failed to get header height: %w", err)
 		}
 		consensusHeightTime = time.Unix(0, int64(h.ConsensusState().GetTimestamp()))
@@ -490,7 +489,7 @@ func (mp *messageProcessor) sendBatchMessages(
 	}
 	callbacks := []func(rtr *provider.RelayerTxResponse, err error){callback}
 
-	//During testing, this adds a callback so our test case can inspect the TX results
+	// During testing, this adds a callback so our test case can inspect the TX results
 	if PathProcMessageCollector != nil {
 		testCallback := func(rtr *provider.RelayerTxResponse, err error) {
 			msgResult := &PathProcessorMessageResp{
@@ -577,7 +576,7 @@ func (mp *messageProcessor) sendSingleMessage(
 
 	callbacks = append(callbacks, callback)
 
-	//During testing, this adds a callback so our test case can inspect the TX results
+	// During testing, this adds a callback so our test case can inspect the TX results
 	if PathProcMessageCollector != nil {
 		testCallback := func(rtr *provider.RelayerTxResponse, err error) {
 			msgResult := &PathProcessorMessageResp{

relayer/processor/path_end_runtime.go

@@ -60,6 +60,8 @@ type pathEndRuntime struct {
 
 	finishedProcessing chan messageToTrack
 	retryCount         uint64
+
+	blockRetries map[uint64]int
 }
 
 func newPathEndRuntime(log *zap.Logger, pathEnd PathEnd, metrics *PrometheusMetrics) *pathEndRuntime {
@@ -83,6 +85,7 @@ func newPathEndRuntime(log *zap.Logger, pathEnd PathEnd, metrics *PrometheusMetr
 		clientICQProcessing:  newClientICQProcessingCache(),
 		connSubscribers:      make(map[string][]func(provider.ConnectionInfo)),
 		metrics:              metrics,
+		blockRetries:         make(map[uint64]int),
 	}
 }
 
@@ -545,14 +548,23 @@ func (pathEnd *pathEndRuntime) shouldSendPacketMessage(message packetIBCMessage,
 	}
 
 	if message.info.Height >= pathEndForHeight.latestBlock.Height {
+		tries := pathEnd.blockRetries[message.info.Height]
+		stuck := false
+		if tries > 5 {
+			stuck = true
+			tries = 0
+		}
+		pathEnd.blockRetries[message.info.Height] = tries + 1
 		pathEnd.log.Debug("Waiting to relay packet message until counterparty height has incremented",
 			zap.String("event_type", eventType),
 			zap.Uint64("sequence", sequence),
 			zap.Uint64("message_height", message.info.Height),
 			zap.Uint64("counterparty_height", counterparty.latestBlock.Height),
 			zap.Inline(k),
 		)
-		return false
+		if !stuck {
+			return false
+		}
 	}
 	if !pathEnd.channelStateCache[k].Open {
 		// channel is not open, do not send
@@ -734,11 +746,20 @@ func (pathEnd *pathEndRuntime) shouldSendChannelMessage(message channelIBCMessag
 	}
 
 	if message.info.Height >= counterparty.latestBlock.Height {
+		tries := pathEnd.blockRetries[message.info.Height]
+		stuck := false
+		if tries > 5 {
+			stuck = true
+			tries = 0
+		}
+		pathEnd.blockRetries[message.info.Height] = tries + 1
 		pathEnd.log.Debug("Waiting to relay channel message until counterparty height has incremented",
 			zap.Inline(channelKey),
 			zap.String("event_type", eventType),
 		)
-		return false
+		if !stuck {
+			return false
+		}
 	}
 	msgProcessCache, ok := pathEnd.channelProcessing[eventType]
 	if !ok {

relayer/processor/path_processor.go

@@ -7,8 +7,9 @@ import (
 
 	chantypes "github.com/cosmos/ibc-go/v8/modules/core/04-channel/types"
 	ibcexported "github.com/cosmos/ibc-go/v8/modules/core/exported"
-	"github.com/cosmos/relayer/v2/relayer/provider"
 	"go.uber.org/zap"
+
+	"github.com/cosmos/relayer/v2/relayer/provider"
 )
 
 const (
@@ -444,7 +445,8 @@ func (pp *PathProcessor) Run(ctx context.Context, cancel func()) {
 
 		// process latest message cache state from both pathEnds
 		if err := pp.processLatestMessages(ctx, cancel); err != nil {
-			pp.log.Error("Failed to process latest messages", zap.Error(err))
+			// todo uncomment if needed
+			// pp.log.Error("Failed to process latest messages", zap.Error(err))
 
 			// in case of IBC message send errors, schedule retry after durationErrorRetry
 			if retryTimer != nil {