Skip to content

Commit 995fe61

Browse files
devblacdevblac
committed
feat(ops): add health, metrics, and structured logging
1 parent 9fae373 commit 995fe61

File tree

12 files changed

+468
-44
lines changed

12 files changed

+468
-44
lines changed

‎cmd/watch-tower/run.go‎

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,51 @@
11
package main
22

33
import (
4+
"context"
45
"fmt"
6+
"net/http"
7+
"os"
58
"time"
69

710
"github.com/devblac/watch-tower/internal/config"
811
"github.com/devblac/watch-tower/internal/engine"
12+
"github.com/devblac/watch-tower/internal/health"
13+
"github.com/devblac/watch-tower/internal/logging"
14+
"github.com/devblac/watch-tower/internal/metrics"
915
"github.com/devblac/watch-tower/internal/sink"
1016
"github.com/devblac/watch-tower/internal/source/algorand"
1117
"github.com/devblac/watch-tower/internal/source/evm"
1218
"github.com/devblac/watch-tower/internal/storage"
13-
"github.com/devblac/watch-tower/internal/logging"
1419
"github.com/spf13/cobra"
1520
)
1621

1722
var (
18-
flagOnce bool
19-
flagDryRun bool
20-
flagFrom uint64
21-
flagTo uint64
23+
flagOnce bool
24+
flagDryRun bool
25+
flagFrom uint64
26+
flagTo uint64
27+
flagHealth string
28+
flagMetrics string
2229
)
2330

2431
func init() {
2532
runCmd.Flags().BoolVar(&flagOnce, "once", false, "Process one tick and exit")
2633
runCmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Do not send to sinks")
2734
runCmd.Flags().Uint64Var(&flagFrom, "from", 0, "Start from height/round override")
2835
runCmd.Flags().Uint64Var(&flagTo, "to", 0, "Stop at height/round (inclusive)")
36+
runCmd.Flags().StringVar(&flagHealth, "health", "", "Health check HTTP address (e.g., :8080)")
37+
runCmd.Flags().StringVar(&flagMetrics, "metrics", "", "Metrics HTTP address (e.g., :9090)")
2938
}
3039

3140
var runCmd = &cobra.Command{
3241
Use: "run",
3342
Short: "Run watch-tower pipelines",
3443
RunE: func(cmd *cobra.Command, args []string) error {
35-
log := logging.New()
44+
logLevel := os.Getenv("LOG_LEVEL")
45+
if logLevel == "" {
46+
logLevel = "info"
47+
}
48+
log := logging.NewWithLevel(logLevel)
3649
ctx := cmd.Context()
3750

3851
cfg, err := config.Load(cfgPath)
@@ -46,8 +59,11 @@ var runCmd = &cobra.Command{
4659
}
4760
defer store.Close()
4861

62+
evmClients := map[string]evm.BlockClient{}
63+
algoClients := map[string]algorand.AlgodClient{}
4964
evmScanners := map[string]*evm.Scanner{}
5065
algoScanners := map[string]*algorand.Scanner{}
66+
5167
for _, src := range cfg.Sources {
5268
switch src.Type {
5369
case "evm":
@@ -58,6 +74,7 @@ var runCmd = &cobra.Command{
5874
if err != nil {
5975
return err
6076
}
77+
evmClients[src.ID] = cli
6178
abis, _ := evm.LoadABIs(src.ABIDirs)
6279
confirmations := cfg.Global.Confirmations["evm"]
6380
sc, err := evm.NewScanner(cli, store, src, confirmations, abis, cfg.Rules)
@@ -73,6 +90,7 @@ var runCmd = &cobra.Command{
7390
if err != nil {
7491
return err
7592
}
93+
algoClients[src.ID] = cli
7694
confirmations := cfg.Global.Confirmations["algorand"]
7795
sc, err := algorand.NewScanner(cli, store, src, confirmations, cfg.Rules)
7896
if err != nil {
@@ -108,15 +126,53 @@ var runCmd = &cobra.Command{
108126
}
109127
}
110128

129+
var mtr *metrics.Metrics
130+
if flagMetrics != "" {
131+
mtr = metrics.Init()
132+
log.Info("metrics enabled", "addr", flagMetrics)
133+
}
134+
135+
if flagHealth != "" {
136+
rpcChecker := health.NewRPCChecker(evmClients, algoClients)
137+
healthSrv := health.Serve(flagHealth, health.Checker{
138+
DBPing: store.Ping,
139+
RPCPing: rpcChecker.Ping,
140+
})
141+
log.Info("health check enabled", "addr", flagHealth)
142+
defer func() {
143+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
144+
defer cancel()
145+
_ = health.Shutdown(shutdownCtx, healthSrv)
146+
}()
147+
}
148+
149+
if flagMetrics != "" {
150+
go func() {
151+
mux := http.NewServeMux()
152+
mux.Handle("/metrics", metrics.Handler())
153+
srv := &http.Server{Addr: flagMetrics, Handler: mux}
154+
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
155+
log.Error("metrics server error", "error", err)
156+
}
157+
}()
158+
}
159+
111160
runner, err := engine.NewRunner(store, cfg, evmScanners, algoScanners, sinks, flagDryRun, flagFrom, flagTo)
112161
if err != nil {
113162
return err
114163
}
115164

116165
for {
117166
if err := runner.RunOnce(ctx); err != nil {
167+
if mtr != nil {
168+
mtr.Errors()
169+
}
170+
log.Error("run error", "error", err)
118171
return err
119172
}
173+
if mtr != nil {
174+
mtr.BlocksProcessed()
175+
}
120176
log.Info("tick complete", "dry_run", flagDryRun)
121177
if flagOnce {
122178
break

‎go.mod‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@ go 1.23.0
44

55
require (
66
github.com/algorand/go-algorand-sdk/v2 v2.9.0
7+
github.com/algorand/go-codec/codec v1.1.10
78
github.com/ethereum/go-ethereum v1.13.11
89
github.com/joho/godotenv v1.5.1
10+
github.com/prometheus/client_golang v1.23.2
911
github.com/spf13/cobra v1.10.2
1012
gopkg.in/yaml.v3 v3.0.1
1113
modernc.org/sqlite v1.23.1
@@ -14,9 +16,10 @@ require (
1416
require (
1517
github.com/Microsoft/go-winio v0.6.1 // indirect
1618
github.com/StackExchange/wmi v1.2.1 // indirect
17-
github.com/algorand/go-codec/codec v1.1.10 // indirect
19+
github.com/beorn7/perks v1.0.1 // indirect
1820
github.com/bits-and-blooms/bitset v1.10.0 // indirect
1921
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
22+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
2023
github.com/consensys/bavard v0.1.13 // indirect
2124
github.com/consensys/gnark-crypto v0.12.1 // indirect
2225
github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
@@ -33,18 +36,24 @@ require (
3336
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
3437
github.com/mattn/go-isatty v0.0.17 // indirect
3538
github.com/mmcloughlin/addchain v0.4.0 // indirect
39+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
40+
github.com/prometheus/client_model v0.6.2 // indirect
41+
github.com/prometheus/common v0.66.1 // indirect
42+
github.com/prometheus/procfs v0.16.1 // indirect
3643
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
3744
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
3845
github.com/spf13/pflag v1.0.9 // indirect
3946
github.com/supranational/blst v0.3.11 // indirect
4047
github.com/tklauser/go-sysconf v0.3.12 // indirect
4148
github.com/tklauser/numcpus v0.6.1 // indirect
49+
go.yaml.in/yaml/v2 v2.4.2 // indirect
4250
golang.org/x/crypto v0.35.0 // indirect
4351
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
4452
golang.org/x/mod v0.14.0 // indirect
45-
golang.org/x/sync v0.5.0 // indirect
46-
golang.org/x/sys v0.30.0 // indirect
53+
golang.org/x/sync v0.13.0 // indirect
54+
golang.org/x/sys v0.35.0 // indirect
4755
golang.org/x/tools v0.15.0 // indirect
56+
google.golang.org/protobuf v1.36.8 // indirect
4857
lukechampine.com/uint128 v1.2.0 // indirect
4958
modernc.org/cc/v3 v3.40.0 // indirect
5059
modernc.org/ccgo/v3 v3.16.13 // indirect

0 commit comments

Comments
 (0)