fix(metrics): grpc prometheus stats missing (#23877) (#23838)

Signed-off-by: pbhatnagar-oss <pbhatifiwork@gmail.com>

docs/operator-manual/metrics.md

@@ -209,6 +209,7 @@ argocd_cluster_labels{label_environment="production",label_team_name="team3",nam
 
 Metrics about API Server API request and response activity (request totals, response codes, etc...).
 Scraped at the `argocd-server-metrics:8083/metrics` endpoint.
+For GRPC metrics to show up environment variable ARGOCD_ENABLE_GRPC_TIME_HISTOGRAM must be set to true. 
 
 | Metric                                            |   Type    | Description                                                                        
 |---------------------------------------------------|:---------:|---------------------------------------------------------------------------------------------|
@@ -249,9 +250,11 @@ Scraped at the `argocd-server-metrics:8083/metrics` endpoint.
 
 ## Repo Server Metrics
 
-Metrics about the Repo Server.
+Metrics about the Repo Server. The gRPC metrics are not exposed by default.  Metrics can be enabled using
+`ARGOCD_ENABLE_GRPC_TIME_HISTOGRAM=true` environment variable.  
 Scraped at the `argocd-repo-server:8084/metrics` endpoint.
 
+
 | Metric                                  |   Type    | Description                                                               |
 | --------------------------------------- | :-------: | ------------------------------------------------------------------------- |
 | `argocd_git_request_duration_seconds`   | histogram | Git requests duration seconds.                                            |

reposerver/metrics/metrics.go

@@ -19,6 +19,7 @@ type MetricsServer struct {
 	repoPendingRequestsGauge *prometheus.GaugeVec
 	redisRequestCounter      *prometheus.CounterVec
 	redisRequestHistogram    *prometheus.HistogramVec
+	PrometheusRegistry       *prometheus.Registry
 }
 
 type GitRequestType string
@@ -108,6 +109,7 @@ func NewMetricsServer() *MetricsServer {
 		repoPendingRequestsGauge: repoPendingRequestsGauge,
 		redisRequestCounter:      redisRequestCounter,
 		redisRequestHistogram:    redisRequestHistogram,
+		PrometheusRegistry:       registry,
 	}
 }
 

reposerver/server.go

@@ -9,7 +9,6 @@ import (
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
 	"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging"
 	"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery"
-	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
@@ -63,8 +62,7 @@ func NewServer(metricsServer *metrics.MetricsServer, cache *reposervercache.Cach
 		serverMetricsOptions = append(serverMetricsOptions, grpc_prometheus.WithServerHandlingTimeHistogram())
 	}
 	serverMetrics := grpc_prometheus.NewServerMetrics(serverMetricsOptions...)
-	reg := prometheus.NewRegistry()
+	metricsServer.PrometheusRegistry.MustRegister(serverMetrics)
-	reg.MustRegister(serverMetrics)
 
 	serverLog := log.NewEntry(log.StandardLogger())
 	streamInterceptors := []grpc.StreamServerInterceptor{

server/metrics/metrics.go

@@ -21,6 +21,7 @@ type MetricsServer struct {
 	extensionRequestCounter  *prometheus.CounterVec
 	extensionRequestDuration *prometheus.HistogramVec
 	loginRequestCounter      *prometheus.CounterVec
+	PrometheusRegistry       *prometheus.Registry
 }
 
 var (
@@ -102,6 +103,7 @@ func NewMetricsServer(host string, port int) *MetricsServer {
 		extensionRequestCounter:  extensionRequestCounter,
 		extensionRequestDuration: extensionRequestDuration,
 		loginRequestCounter:      loginRequestCounter,
+		PrometheusRegistry:       registry,
 	}
 }
 

server/server.go

@@ -561,7 +561,6 @@ func (server *ArgoCDServer) Run(ctx context.Context, listeners *Listeners) {
 			server.Shutdown()
 		}
 	}()
-
 	metricsServ := metrics.NewMetricsServer(server.MetricsHost, server.MetricsPort)
 	if server.RedisClient != nil {
 		cacheutil.CollectMetrics(server.RedisClient, metricsServ, server.userStateStorage.GetLockObject())
@@ -576,7 +575,7 @@ func (server *ArgoCDServer) Run(ctx context.Context, listeners *Listeners) {
 		server.sessionMgr.CollectMetrics(metricsServ)
 	}
 	server.serviceSet = svcSet
-	grpcS, appResourceTreeFn := server.newGRPCServer()
+	grpcS, appResourceTreeFn := server.newGRPCServer(metricsServ.PrometheusRegistry)
 	grpcWebS := grpcweb.WrapServer(grpcS)
 	var httpS *http.Server
 	var httpsS *http.Server
@@ -899,14 +898,13 @@ func (server *ArgoCDServer) useTLS() bool {
 	return true
 }
 
-func (server *ArgoCDServer) newGRPCServer() (*grpc.Server, application.AppResourceTreeFn) {
+func (server *ArgoCDServer) newGRPCServer(prometheusRegistry *prometheus.Registry) (*grpc.Server, application.AppResourceTreeFn) {
 	var serverMetricsOptions []grpc_prometheus.ServerMetricsOption
 	if enableGRPCTimeHistogram {
 		serverMetricsOptions = append(serverMetricsOptions, grpc_prometheus.WithServerHandlingTimeHistogram())
 	}
 	serverMetrics := grpc_prometheus.NewServerMetrics(serverMetricsOptions...)
-	reg := prometheus.NewRegistry()
+	prometheusRegistry.MustRegister(serverMetrics)
-	reg.MustRegister(serverMetrics)
 
 	sOpts := []grpc.ServerOption{
 		// Set the both send and receive the bytes limit to be 100MB

test/e2e/metrics_test.go

@@ -103,4 +103,6 @@ func TestKubectlMetrics(t *testing.T) {
 	assert.Contains(t, string(body), "argocd_kubectl_response_size_bytes", "metrics should have contained argocd_kubectl_response_size_bytes")
 	assert.Contains(t, string(body), "argocd_kubectl_rate_limiter_duration_seconds", "metrics should have contained argocd_kubectl_rate_limiter_duration_seconds")
 	assert.Contains(t, string(body), "argocd_kubectl_requests_total", "metrics should have contained argocd_kubectl_requests_total")
+	assert.Contains(t, string(body), "grpc_server_handled_total", "metrics should have contained grpc_server_handled_total for all the reflected methods")
+	assert.Contains(t, string(body), "grpc_server_msg_received_total", "metrics should have contained grpc_server_msg_received_total for all the reflected methods")
 }