Add Lido-Exporter app

Makefile

@@ -9,10 +9,28 @@ LDFLAGS=-X github.com/NethermindEth/sedge/internal/utils.Version="${SEDGE_VERSIO
 compile: ## compile:
 	@mkdir -p build
 	@go build -ldflags "${LDFLAGS}" -o build/sedge cmd/sedge/main.go
+	@go build -ldflags "${LDFLAGS}" -o build/lido-exporter cmd/lido-exporter/main.go
 
 compile-linux: ## compile:
 	@mkdir -p build
 	@env GOOS=linux go build -ldflags="${LDFLAGS[*]}" -o build/sedge cmd/sedge/main.go
+	@env GOOS=linux go build -ldflags="${LDFLAGS[*]}" -o build/lido-exporter cmd/lido-exporter/main.go
+
+compile-sedge:
+	@mkdir -p build
+	@go build -ldflags "${LDFLAGS}" -o build/sedge cmd/sedge/main.go
+
+compile-lido-exporter:
+	@mkdir -p build
+	@go build -ldflags "${LDFLAGS}" -o build/lido-exporter cmd/lido-exporter/main.go
+
+compile-sedge-linux:
+	@mkdir -p build
+	@env GOOS=linux go build -ldflags="${LDFLAGS[*]}" -o build/sedge cmd/sedge/main.go
+
+compile-lido-exporter-linux:
+	@mkdir -p build
+	@env GOOS=linux go build -ldflags="${LDFLAGS[*]}" -o build/lido-exporter cmd/lido-exporter/main.go
 
 install: compile ## compile the binary and copy it to PATH
 	@sudo cp build/sedge /usr/local/bin
@@ -28,25 +46,26 @@ generate: ## generate go files
 	@abigen --abi ./internal/lido/contracts/csfeedistributor/CSFeeDistributor.abi --bin ./internal/lido/contracts/csfeedistributor/CSFeeDistributor.bin --pkg csfeedistributor --out ./internal/lido/contracts/csfeedistributor/CSFeeDistributor.go
 	@abigen --abi ./internal/lido/contracts/csaccounting/CSAccounting.abi --bin ./internal/lido/contracts/csaccounting/CSAccounting.bin --pkg csaccounting --out ./internal/lido/contracts/csaccounting/CSAccounting.go
 	@abigen --abi ./internal/lido/contracts/mevboostrelaylist/MEVBoostRelayAllowedList.abi --bin ./internal/lido/contracts/mevboostrelaylist/MEVBoostRelayAllowedList.bin --pkg mevboostrelaylist --out ./internal/lido/contracts/mevboostrelaylist/MEVBoostRelayAllowedList.go
+	@abigen --abi ./internal/lido/contracts/vebo/VEBO.abi --bin ./internal/lido/contracts/vebo/VEBO.bin --pkg vebo --out ./internal/lido/contracts/vebo/VEBO.go
 	@go generate ./...
 
 test: generate ## run tests
 	@mkdir -p coverage
-	@go test -coverprofile=coverage/coverage.out -covermode=count ./...
+	@go test -coverprofile=coverage/coverage.out -covermode=count -timeout 25m ./...
 
 e2e-test: generate ## Run e2e tests
-	@go test -timeout 20m -count=1 ./e2e/...
+	@go test -timeout 25m -count=1 ./e2e/sedge/...
 
 e2e-test-windows: generate ## Run e2e tests on Windows
-	@go test -timeout 20m -count=1 -skip TestE2E_MonitoringStack ./e2e/...
+	@go test -timeout 25m -count=1 -skip TestE2E_MonitoringStack ./e2e/sedge/...
 
 test-no-e2e: generate ## run tests excluding e2e
 	@mkdir -p coverage
 	@go test -coverprofile=coverage/coverage.out -covermode=count ./... -skip TestE2E
 
 codecov-test: generate ## unit tests with coverage using the courtney tool
 	@mkdir -p coverage
-	@courtney/courtney -v -o coverage/coverage.out -t="-skip=TestE2E" ./...
+	@go test -coverprofile=coverage/coverage.out -covermode=count -timeout 25m ./... -skip TestE2E
 	@go tool cover -html=coverage/coverage.out -o coverage/coverage.html
 
 install-gofumpt: ## install gofumpt
@@ -55,15 +74,10 @@ install-gofumpt: ## install gofumpt
 install-mockgen: ## install mockgen
 	go install github.com/golang/mock/[email protected] 
 
-install-courtney: ## Install courtney for code coverage
-	@git clone https://github.com/dave/courtney
-	@(cd courtney && go get  ./... && go build courtney.go)
-	@go get ./...
-
 install-abigen: ## install abigen
 	go install github.com/ethereum/go-ethereum/cmd/abigen@latest
 
-install-deps: | install-gofumpt install-courtney install-mockgen install-abigen ## Install some project dependencies
+install-deps: | install-gofumpt install-mockgen install-abigen ## Install some project dependencies
 
 coverage: ## show tests coverage
 	@go tool cover -html=coverage/coverage.out -o coverage/coverage.html

cmd/lido-exporter/Dockerfile

@@ -0,0 +1,33 @@
+# Use the official Golang image to build the application
+FROM golang:1.22 as builder
+
+# Set the Current Working Directory inside the container
+WORKDIR /app
+
+# Copy go mod and sum files
+COPY go.mod go.sum ./
+
+# Download all dependencies. Dependencies will be cached if the go.mod and go.sum files are not changed
+RUN go mod download
+
+# Copy the source from the current directory to the Working Directory inside the container
+COPY . .
+
+# Build the Go app
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o lido-exporter ./cmd/lido-exporter
+
+# Start a new stage from scratch
+FROM alpine:latest  
+
+RUN apk --no-cache add ca-certificates
+
+WORKDIR /root/
+
+# Copy the Pre-built binary file from the previous stage
+COPY --from=builder /app/lido-exporter .
+
+# Expose port 8080 to the outside world
+EXPOSE 8080
+
+# Command to run the executable
+ENTRYPOINT ["./lido-exporter"]

cmd/lido-exporter/README.md

@@ -0,0 +1,53 @@
+# Lido Exporter
+
+Lido Exporter is a service that exports data from the Lido CSM smart contracts as Prometheus metrics.
+
+## Features
+
+- Collects metrics for penalties, exit requests, node operator info, keys info, bond info, and rewards info
+- Supports Holesky network (Mainnet support can be easily added)
+- Exports metrics in Prometheus format
+
+## Usage
+
+### Running with Docker
+
+1. Build the Docker image:
+   ```
+   docker build -t lido-exporter .
+   ```
+
+2. Run the Docker container:
+   ```
+   docker run -d -p 8080:8080 -e LIDO_EXPORTER_NODE_OPERATOR_ID=<your_node_operator_id> -e LIDO_EXPORTER_NETWORK=<network_name> lido-exporter
+   ```
+
+### Running as a CLI Application
+
+1. Build the application:
+   ```
+   go build -o lido-exporter cmd/lido-exporter/main.go
+   ```
+
+2. Run the application:
+   ```
+   ./lido-exporter --node-operator-id <your_node_operator_id> --network <network_name>
+   ```
+
+## Configuration
+
+The service can be configured using the following methods (in order of precedence):
+
+1. Environment variables
+2. Command-line flags
+
+Available settings:
+
+- `LIDO_EXPORTER_NODE_OPERATOR_ID` (required): Node Operator ID
+- `LIDO_EXPORTER_REWARD_ADDRESS` (optional): Reward address of Node Operator. It is used to calculate Node Operator ID if not set
+- `LIDO_EXPORTER_NETWORK`: Network name (default: "holesky")
+- `LIDO_EXPORTER_RPC_ENDPOINTS`: Comma-separated list of Ethereum RPC endpoints
+- `LIDO_EXPORTER_WS_ENDPOINTS`: Comma-separated list of Ethereum WebSocket endpoints
+- `LIDO_EXPORTER_PORT`: Port to listen on (default: "8080")
+- `LIDO_EXPORTER_SCRAPE_TIME`: Scrape interval (default: 30s)
+- `LIDO_EXPORTER_LOG_LEVEL`: Log level (default: "info")

cmd/lido-exporter/cli/lido_exporter.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2022 Nethermind
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cli
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"net/http"
+	"os"
+	"os/signal"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/NethermindEth/sedge/cmd/lido-exporter/metrics"
+	"github.com/NethermindEth/sedge/configs"
+	"github.com/NethermindEth/sedge/internal/lido/contracts"
+	"github.com/NethermindEth/sedge/internal/lido/contracts/csmodule"
+	"github.com/NethermindEth/sedge/internal/utils"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	nested "github.com/antonfisher/nested-logrus-formatter"
+	log "github.com/sirupsen/logrus"
+)
+
+var logLevel string
+
+func RootCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "lido-exporter",
+		Short: "Lido Exporter exports Lido CSM metrics to Prometheus",
+		Long:  `Lido Exporter exports Lido CSM metrics to Prometheus`,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			logLevel = viper.GetString("log-level")
+			level, err := log.ParseLevel(strings.ToLower(logLevel))
+			if err != nil {
+				log.WithField(configs.Component, "Logger Init").Error(err)
+				return
+			}
+			log.SetLevel(level)
+		},
+		Run: run,
+	}
+
+	// Disable completion default cmd
+	cmd.CompletionOptions.DisableDefaultCmd = true
+
+	// Persistent flags
+	cmd.PersistentFlags().String("node-operator-id", "", "Node Operator ID")
+	cmd.PersistentFlags().String("reward-address", "", "Reward address of Node Operator. It is used to calculate Node Operator ID if not set")
+	cmd.PersistentFlags().String("network", "holesky", "Network name")
+	cmd.PersistentFlags().StringSlice("rpc-endpoints", nil, "List of Ethereum HTTP RPC endpoints")
+	cmd.PersistentFlags().StringSlice("ws-endpoints", nil, "List of Ethereum WebSocket RPC endpoints")
+	cmd.PersistentFlags().String("port", "8080", "Port where the metrics will be exported.")
+	cmd.PersistentFlags().Duration("scrape-time", 30*time.Second, "Time interval for scraping metrics. Values should be in the format of 10s, 1m, 1h, etc.")
+	cmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "Set Log Level, e.g panic, fatal, error, warn, warning, info, debug, trace")
+
+	viper.BindPFlag("node-operator-id", cmd.PersistentFlags().Lookup("node-operator-id"))
+	viper.BindPFlag("reward-address", cmd.PersistentFlags().Lookup("reward-address"))
+	viper.BindPFlag("network", cmd.PersistentFlags().Lookup("network"))
+	viper.BindPFlag("rpc-endpoints", cmd.PersistentFlags().Lookup("rpc-endpoints"))
+	viper.BindPFlag("ws-endpoints", cmd.PersistentFlags().Lookup("ws-endpoints"))
+	viper.BindPFlag("port", cmd.PersistentFlags().Lookup("port"))
+	viper.BindPFlag("scrape-time", cmd.PersistentFlags().Lookup("scrape-time"))
+	viper.BindPFlag("log-level", cmd.PersistentFlags().Lookup("log-level"))
+	viper.SetEnvPrefix("LIDO_EXPORTER")
+	viper.AutomaticEnv()
+	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+
+	return cmd
+}
+
+func run(cmd *cobra.Command, args []string) {
+	nodeOperatorID := viper.GetString("node-operator-id")
+	rewardAddress := viper.GetString("reward-address")
+	if nodeOperatorID == "" && rewardAddress == "" {
+		log.Fatal("Node Operator ID or Reward Address is required")
+	}
+
+	// Validate port
+	port := viper.GetString("port")
+	_, err := strconv.Atoi(port)
+	if err != nil {
+		log.Fatalf("Invalid port: %s", port)
+	}
+
+	network := viper.GetString("network")
+	if !slices.Contains(configs.NetworkSupported(), network) {
+		log.Fatalf("Invalid network: %s", network)
+	}
+
+	var nodeOperatorIDBigInt *big.Int
+	if nodeOperatorID != "" {
+		var ok bool
+		nodeOperatorIDBigInt, ok = new(big.Int).SetString(nodeOperatorID, 10)
+		if !ok {
+			log.Fatalf("Failed to convert Node Operator ID to big.Int: %s", nodeOperatorID)
+		}
+		if nodeOperatorIDBigInt.Sign() < 0 { // Check if the value is negative
+			log.Fatalf("Node Operator ID cannot be negative: %s", nodeOperatorID)
+		}
+	} else {
+		if !utils.IsAddress(rewardAddress) {
+			log.Fatalf("Invalid reward address: %s", rewardAddress)
+		}
+
+		var err error
+		nodeOperatorIDBigInt, err = csmodule.NodeID(network, rewardAddress)
+		if err != nil {
+			log.Fatalf("Failed to get Node Operator ID: %v", err)
+		}
+	}
+
+	rpcEndpoints := viper.GetStringSlice("rpc-endpoints")
+	wsEndpoints := viper.GetStringSlice("ws-endpoints")
+
+	client, err := contracts.ConnectClient(network, false, rpcEndpoints...)
+	if err != nil {
+		log.Fatalf("Failed to connect to Ethereum RPC: %v", err)
+	}
+	wsClient, err := contracts.ConnectClient(network, true, wsEndpoints...)
+	if err != nil {
+		log.Fatalf("Failed to connect to Ethereum WebSocket: %v", err)
+	}
+
+	// Initialize metrics
+	metrics.InitMetrics(nodeOperatorID, network)
+
+	// Start the metrics server
+	go func() {
+		http.Handle("/metrics", promhttp.Handler())
+		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", viper.GetString("port")), nil))
+	}()
+
+	// Start collecting metrics
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go metrics.CollectMetrics(ctx, client, wsClient, nodeOperatorIDBigInt, network, viper.GetDuration("scrape-time"))
+
+	// Wait for interrupt signal to gracefully shutdown the exporter
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+	<-sigCh
+	log.Info("Shutting down Lido Exporter...")
+}
+
+func init() {
+	log.SetFormatter(&nested.Formatter{
+		HideKeys:        true,
+		FieldsOrder:     []string{configs.Component},
+		TimestampFormat: "2006-01-02 15:04:05 --",
+	})
+
+	level, err := log.ParseLevel(strings.ToLower("error"))
+	if err != nil {
+		log.WithField(configs.Component, "Logger Init").Error(err)
+		return
+	}
+	log.SetLevel(level)
+	log.WithField(configs.Component, "Logger Init").Infof("Log level: %+v", logLevel)
+}

cmd/lido-exporter/main.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2022 Nethermind
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import (
+	"github.com/NethermindEth/sedge/cmd/lido-exporter/cli"
+	log "github.com/sirupsen/logrus"
+)
+
+func main() {
+	rootCmd := cli.RootCmd()
+	rootCmd.SilenceErrors = true
+	rootCmd.SilenceUsage = true
+
+	if err := rootCmd.Execute(); err != nil {
+		log.Fatal(err)
+	}
+}