diff --git a/controller/metrics/metrics.go b/controller/metrics/metrics.go index dcb3717538..f58034c2c5 100644 --- a/controller/metrics/metrics.go +++ b/controller/metrics/metrics.go @@ -3,6 +3,7 @@ package metrics import ( "context" "net/http" + "os" "time" "github.com/prometheus/client_golang/prometheus" @@ -30,6 +31,8 @@ type MetricsServer struct { const ( // MetricsPath is the endpoint to collect application metrics MetricsPath = "/metrics" + // EnvVarLegacyControllerMetrics is a env var to re-enable deprecated prometheus metrics + EnvVarLegacyControllerMetrics = "ARGOCD_LEGACY_CONTROLLER_METRICS" ) // Follow Prometheus naming practices @@ -40,27 +43,71 @@ var ( descAppInfo = prometheus.NewDesc( "argocd_app_info", "Information about application.", - append(descAppDefaultLabels, "repo", "dest_server", "dest_namespace"), + append(descAppDefaultLabels, "repo", "dest_server", "dest_namespace", "sync_status", "health_status", "operation"), nil, ) + // DEPRECATED descAppCreated = prometheus.NewDesc( "argocd_app_created_time", "Creation time in unix timestamp for an application.", descAppDefaultLabels, nil, ) + // DEPRECATED: superceded by sync_status label in argocd_app_info descAppSyncStatusCode = prometheus.NewDesc( "argocd_app_sync_status", "The application current sync status.", append(descAppDefaultLabels, "sync_status"), nil, ) + // DEPRECATED: superceded by health_status label in argocd_app_info descAppHealthStatus = prometheus.NewDesc( "argocd_app_health_status", "The application current health status.", append(descAppDefaultLabels, "health_status"), nil, ) + + syncCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "argocd_app_sync_total", + Help: "Number of application syncs.", + }, + append(descAppDefaultLabels, "dest_server", "phase"), + ) + + k8sRequestCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "argocd_app_k8s_request_total", + Help: "Number of kubernetes requests executed during application reconciliation.", + }, + append(descAppDefaultLabels, "server", "response_code", "verb", "resource_kind", "resource_namespace"), + ) + + kubectlExecCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "argocd_kubectl_exec_total", + Help: "Number of kubectl executions", + }, []string{"command"}) + + kubectlExecPendingGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "argocd_kubectl_exec_pending", + Help: "Number of pending kubectl executions", + }, []string{"command"}) + + reconcileHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "argocd_app_reconcile", + Help: "Application reconciliation performance.", + // Buckets chosen after observing a ~2100ms mean reconcile time + Buckets: []float64{0.25, .5, 1, 2, 4, 8, 16}, + }, + append(descAppDefaultLabels, "dest_server"), + ) + + clusterEventsCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "argocd_cluster_events_total", + Help: "Number of processes k8s resource events.", + }, append(descClusterDefaultLabels, "group", "kind")) ) // NewMetricsServer returns a new prometheus server which collects application metrics @@ -75,50 +122,11 @@ func NewMetricsServer(addr string, appLister applister.ApplicationLister, health }, promhttp.HandlerOpts{})) healthz.ServeHealthCheck(mux, healthCheck) - syncCounter := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "argocd_app_sync_total", - Help: "Number of application syncs.", - }, - append(descAppDefaultLabels, "phase"), - ) registry.MustRegister(syncCounter) - - k8sRequestCounter := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "argocd_app_k8s_request_total", - Help: "Number of kubernetes requests executed during application reconciliation.", - }, - append(descAppDefaultLabels, "server", "response_code", "verb", "resource_kind", "resource_namespace"), - ) registry.MustRegister(k8sRequestCounter) - - kubectlExecCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "argocd_kubectl_exec_total", - Help: "Number of kubectl executions", - }, []string{"command"}) registry.MustRegister(kubectlExecCounter) - kubectlExecPendingGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "argocd_kubectl_exec_pending", - Help: "Number of pending kubectl executions", - }, []string{"command"}) registry.MustRegister(kubectlExecPendingGauge) - - reconcileHistogram := prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "argocd_app_reconcile", - Help: "Application reconciliation performance.", - // Buckets chosen after observing a ~2100ms mean reconcile time - Buckets: []float64{0.25, .5, 1, 2, 4, 8, 16}, - }, - descAppDefaultLabels, - ) - registry.MustRegister(reconcileHistogram) - clusterEventsCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "argocd_cluster_events_total", - Help: "Number of processes k8s resource events.", - }, append(descClusterDefaultLabels, "group", "kind")) registry.MustRegister(clusterEventsCounter) return &MetricsServer{ @@ -147,7 +155,7 @@ func (m *MetricsServer) IncSync(app *argoappv1.Application, state *argoappv1.Ope if !state.Phase.Completed() { return } - m.syncCounter.WithLabelValues(app.Namespace, app.Name, app.Spec.GetProject(), string(state.Phase)).Inc() + m.syncCounter.WithLabelValues(app.Namespace, app.Name, app.Spec.GetProject(), app.Spec.Destination.Server, string(state.Phase)).Inc() } func (m *MetricsServer) IncKubectlExec(command string) { @@ -183,7 +191,7 @@ func (m *MetricsServer) IncKubernetesRequest(app *argoappv1.Application, server, // IncReconcile increments the reconcile counter for an application func (m *MetricsServer) IncReconcile(app *argoappv1.Application, duration time.Duration) { - m.reconcileHistogram.WithLabelValues(app.Namespace, app.Name, app.Spec.GetProject()).Observe(duration.Seconds()) + m.reconcileHistogram.WithLabelValues(app.Namespace, app.Name, app.Spec.GetProject(), app.Spec.Destination.Server).Observe(duration.Seconds()) } type appCollector struct { @@ -207,7 +215,6 @@ func NewAppRegistry(appLister applister.ApplicationLister) *prometheus.Registry // Describe implements the prometheus.Collector interface func (c *appCollector) Describe(ch chan<- *prometheus.Desc) { ch <- descAppInfo - ch <- descAppCreated ch <- descAppSyncStatusCode ch <- descAppHealthStatus } @@ -241,20 +248,36 @@ func collectApps(ch chan<- prometheus.Metric, app *argoappv1.Application) { addConstMetric(desc, prometheus.GaugeValue, v, lv...) } - addGauge(descAppInfo, 1, git.NormalizeGitURL(app.Spec.Source.RepoURL), app.Spec.Destination.Server, app.Spec.Destination.Namespace) - - addGauge(descAppCreated, float64(app.CreationTimestamp.Unix())) - + var operation string + if app.DeletionTimestamp != nil { + operation = "delete" + } else if app.Operation != nil && app.Operation.Sync != nil { + operation = "sync" + } syncStatus := app.Status.Sync.Status - addGauge(descAppSyncStatusCode, boolFloat64(syncStatus == argoappv1.SyncStatusCodeSynced), string(argoappv1.SyncStatusCodeSynced)) - addGauge(descAppSyncStatusCode, boolFloat64(syncStatus == argoappv1.SyncStatusCodeOutOfSync), string(argoappv1.SyncStatusCodeOutOfSync)) - addGauge(descAppSyncStatusCode, boolFloat64(syncStatus == argoappv1.SyncStatusCodeUnknown || syncStatus == ""), string(argoappv1.SyncStatusCodeUnknown)) - + if syncStatus == "" { + syncStatus = argoappv1.SyncStatusCodeUnknown + } healthStatus := app.Status.Health.Status - addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusUnknown || healthStatus == ""), argoappv1.HealthStatusUnknown) - addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusProgressing), argoappv1.HealthStatusProgressing) - addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusSuspended), argoappv1.HealthStatusSuspended) - addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusHealthy), argoappv1.HealthStatusHealthy) - addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusDegraded), argoappv1.HealthStatusDegraded) - addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusMissing), argoappv1.HealthStatusMissing) + if healthStatus == "" { + healthStatus = argoappv1.HealthStatusUnknown + } + + addGauge(descAppInfo, 1, git.NormalizeGitURL(app.Spec.Source.RepoURL), app.Spec.Destination.Server, app.Spec.Destination.Namespace, string(syncStatus), healthStatus, operation) + + // Deprecated controller metrics + if os.Getenv(EnvVarLegacyControllerMetrics) == "true" { + addGauge(descAppCreated, float64(app.CreationTimestamp.Unix())) + + addGauge(descAppSyncStatusCode, boolFloat64(syncStatus == argoappv1.SyncStatusCodeSynced), string(argoappv1.SyncStatusCodeSynced)) + addGauge(descAppSyncStatusCode, boolFloat64(syncStatus == argoappv1.SyncStatusCodeOutOfSync), string(argoappv1.SyncStatusCodeOutOfSync)) + addGauge(descAppSyncStatusCode, boolFloat64(syncStatus == argoappv1.SyncStatusCodeUnknown || syncStatus == ""), string(argoappv1.SyncStatusCodeUnknown)) + + addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusUnknown || healthStatus == ""), argoappv1.HealthStatusUnknown) + addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusProgressing), argoappv1.HealthStatusProgressing) + addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusSuspended), argoappv1.HealthStatusSuspended) + addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusHealthy), argoappv1.HealthStatusHealthy) + addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusDegraded), argoappv1.HealthStatusDegraded) + addGauge(descAppHealthStatus, boolFloat64(healthStatus == argoappv1.HealthStatusMissing), argoappv1.HealthStatusMissing) + } } diff --git a/controller/metrics/metrics_test.go b/controller/metrics/metrics_test.go index 3524044bb8..867b9977f5 100644 --- a/controller/metrics/metrics_test.go +++ b/controller/metrics/metrics_test.go @@ -5,6 +5,7 @@ import ( "log" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" @@ -42,25 +43,52 @@ status: status: Healthy ` -const expectedResponse = `# HELP argocd_app_created_time Creation time in unix timestamp for an application. -# TYPE argocd_app_created_time gauge -argocd_app_created_time{name="my-app",namespace="argocd",project="important-project"} -6.21355968e+10 -# HELP argocd_app_health_status The application current health status. -# TYPE argocd_app_health_status gauge -argocd_app_health_status{health_status="Degraded",name="my-app",namespace="argocd",project="important-project"} 0 -argocd_app_health_status{health_status="Healthy",name="my-app",namespace="argocd",project="important-project"} 1 -argocd_app_health_status{health_status="Missing",name="my-app",namespace="argocd",project="important-project"} 0 -argocd_app_health_status{health_status="Progressing",name="my-app",namespace="argocd",project="important-project"} 0 -argocd_app_health_status{health_status="Suspended",name="my-app",namespace="argocd",project="important-project"} 0 -argocd_app_health_status{health_status="Unknown",name="my-app",namespace="argocd",project="important-project"} 0 -# HELP argocd_app_info Information about application. -# TYPE argocd_app_info gauge -argocd_app_info{dest_namespace="dummy-namespace",dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",repo="https://github.com/argoproj/argocd-example-apps"} 1 -# HELP argocd_app_sync_status The application current sync status. -# TYPE argocd_app_sync_status gauge -argocd_app_sync_status{name="my-app",namespace="argocd",project="important-project",sync_status="OutOfSync"} 0 -argocd_app_sync_status{name="my-app",namespace="argocd",project="important-project",sync_status="Synced"} 1 -argocd_app_sync_status{name="my-app",namespace="argocd",project="important-project",sync_status="Unknown"} 0 +const fakeApp2 = ` +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: my-app-2 + namespace: argocd +spec: + destination: + namespace: dummy-namespace + server: https://localhost:6443 + project: important-project + source: + path: some/path + repoURL: https://github.com/argoproj/argocd-example-apps.git +status: + sync: + status: Synced + health: + status: Healthy +operation: + sync: + revision: 041eab7439ece92c99b043f0e171788185b8fc1d + syncStrategy: + hook: {} +` + +const fakeApp3 = ` +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: my-app-3 + namespace: argocd + deletionTimestamp: "2020-03-16T09:17:45Z" +spec: + destination: + namespace: dummy-namespace + server: https://localhost:6443 + project: important-project + source: + path: some/path + repoURL: https://github.com/argoproj/argocd-example-apps.git +status: + sync: + status: OutOfSync + health: + status: Degraded ` const fakeDefaultApp = ` @@ -83,46 +111,26 @@ status: status: Healthy ` -const expectedDefaultResponse = `# HELP argocd_app_created_time Creation time in unix timestamp for an application. -# TYPE argocd_app_created_time gauge -argocd_app_created_time{name="my-app",namespace="argocd",project="default"} -6.21355968e+10 -# HELP argocd_app_health_status The application current health status. -# TYPE argocd_app_health_status gauge -argocd_app_health_status{health_status="Degraded",name="my-app",namespace="argocd",project="default"} 0 -argocd_app_health_status{health_status="Healthy",name="my-app",namespace="argocd",project="default"} 1 -argocd_app_health_status{health_status="Missing",name="my-app",namespace="argocd",project="default"} 0 -argocd_app_health_status{health_status="Progressing",name="my-app",namespace="argocd",project="default"} 0 -argocd_app_health_status{health_status="Suspended",name="my-app",namespace="argocd",project="default"} 0 -argocd_app_health_status{health_status="Unknown",name="my-app",namespace="argocd",project="default"} 0 -# HELP argocd_app_info Information about application. -# TYPE argocd_app_info gauge -argocd_app_info{dest_namespace="dummy-namespace",dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="default",repo="https://github.com/argoproj/argocd-example-apps"} 1 -# HELP argocd_app_sync_status The application current sync status. -# TYPE argocd_app_sync_status gauge -argocd_app_sync_status{name="my-app",namespace="argocd",project="default",sync_status="OutOfSync"} 0 -argocd_app_sync_status{name="my-app",namespace="argocd",project="default",sync_status="Synced"} 1 -argocd_app_sync_status{name="my-app",namespace="argocd",project="default",sync_status="Unknown"} 0 -` - var noOpHealthCheck = func() error { return nil } -func newFakeApp(fakeApp string) *argoappv1.Application { +func newFakeApp(fakeAppYAML string) *argoappv1.Application { var app argoappv1.Application - err := yaml.Unmarshal([]byte(fakeApp), &app) + err := yaml.Unmarshal([]byte(fakeAppYAML), &app) if err != nil { panic(err) } return &app } -func newFakeLister(fakeApp ...string) (context.CancelFunc, applister.ApplicationLister) { +func newFakeLister(fakeAppYAMLs ...string) (context.CancelFunc, applister.ApplicationLister) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var fakeApps []runtime.Object - for _, name := range fakeApp { - fakeApps = append(fakeApps, newFakeApp(name)) + for _, appYAML := range fakeAppYAMLs { + a := newFakeApp(appYAML) + fakeApps = append(fakeApps, a) } appClientset := appclientset.NewSimpleClientset(fakeApps...) factory := appinformer.NewFilteredSharedInformerFactory(appClientset, 0, "argocd", func(options *metav1.ListOptions) {}) @@ -134,8 +142,8 @@ func newFakeLister(fakeApp ...string) (context.CancelFunc, applister.Application return cancel, factory.Argoproj().V1alpha1().Applications().Lister() } -func testApp(t *testing.T, fakeApp string, expectedResponse string) { - cancel, appLister := newFakeLister(fakeApp) +func testApp(t *testing.T, fakeAppYAMLs []string, expectedResponse string) { + cancel, appLister := newFakeLister(fakeAppYAMLs...) defer cancel() metricsServ := NewMetricsServer("localhost:8082", appLister, noOpHealthCheck) req, err := http.NewRequest("GET", "/metrics", nil) @@ -149,39 +157,75 @@ func testApp(t *testing.T, fakeApp string, expectedResponse string) { } type testCombination struct { - application string + applications []string expectedResponse string } func TestMetrics(t *testing.T) { combinations := []testCombination{ { - application: fakeApp, - expectedResponse: expectedResponse, + applications: []string{fakeApp, fakeApp2, fakeApp3}, + expectedResponse: ` +# HELP argocd_app_info Information about application. +# TYPE argocd_app_info gauge +argocd_app_info{dest_namespace="dummy-namespace",dest_server="https://localhost:6443",health_status="Degraded",name="my-app-3",namespace="argocd",operation="delete",project="important-project",repo="https://github.com/argoproj/argocd-example-apps",sync_status="OutOfSync"} 1 +argocd_app_info{dest_namespace="dummy-namespace",dest_server="https://localhost:6443",health_status="Healthy",name="my-app",namespace="argocd",operation="",project="important-project",repo="https://github.com/argoproj/argocd-example-apps",sync_status="Synced"} 1 +argocd_app_info{dest_namespace="dummy-namespace",dest_server="https://localhost:6443",health_status="Healthy",name="my-app-2",namespace="argocd",operation="sync",project="important-project",repo="https://github.com/argoproj/argocd-example-apps",sync_status="Synced"} 1 +`, }, { - application: fakeDefaultApp, - expectedResponse: expectedDefaultResponse, + applications: []string{fakeDefaultApp}, + expectedResponse: ` +# HELP argocd_app_info Information about application. +# TYPE argocd_app_info gauge +argocd_app_info{dest_namespace="dummy-namespace",dest_server="https://localhost:6443",health_status="Healthy",name="my-app",namespace="argocd",operation="",project="default",repo="https://github.com/argoproj/argocd-example-apps",sync_status="Synced"} 1 +`, }, } for _, combination := range combinations { - testApp(t, combination.application, combination.expectedResponse) + testApp(t, combination.applications, combination.expectedResponse) } } -const appSyncTotal = `# HELP argocd_app_sync_total Number of application syncs. -# TYPE argocd_app_sync_total counter -argocd_app_sync_total{name="my-app",namespace="argocd",phase="Error",project="important-project"} 1 -argocd_app_sync_total{name="my-app",namespace="argocd",phase="Failed",project="important-project"} 1 -argocd_app_sync_total{name="my-app",namespace="argocd",phase="Succeeded",project="important-project"} 2 +func TestLegacyMetrics(t *testing.T) { + os.Setenv(EnvVarLegacyControllerMetrics, "true") + defer os.Unsetenv(EnvVarLegacyControllerMetrics) + + expectedResponse := ` +# HELP argocd_app_created_time Creation time in unix timestamp for an application. +# TYPE argocd_app_created_time gauge +argocd_app_created_time{name="my-app",namespace="argocd",project="important-project"} -6.21355968e+10 +# HELP argocd_app_health_status The application current health status. +# TYPE argocd_app_health_status gauge +argocd_app_health_status{health_status="Degraded",name="my-app",namespace="argocd",project="important-project"} 0 +argocd_app_health_status{health_status="Healthy",name="my-app",namespace="argocd",project="important-project"} 1 +argocd_app_health_status{health_status="Missing",name="my-app",namespace="argocd",project="important-project"} 0 +argocd_app_health_status{health_status="Progressing",name="my-app",namespace="argocd",project="important-project"} 0 +argocd_app_health_status{health_status="Suspended",name="my-app",namespace="argocd",project="important-project"} 0 +argocd_app_health_status{health_status="Unknown",name="my-app",namespace="argocd",project="important-project"} 0 +# HELP argocd_app_sync_status The application current sync status. +# TYPE argocd_app_sync_status gauge +argocd_app_sync_status{name="my-app",namespace="argocd",project="important-project",sync_status="OutOfSync"} 0 +argocd_app_sync_status{name="my-app",namespace="argocd",project="important-project",sync_status="Synced"} 1 +argocd_app_sync_status{name="my-app",namespace="argocd",project="important-project",sync_status="Unknown"} 0 ` + testApp(t, []string{fakeApp}, expectedResponse) +} func TestMetricsSyncCounter(t *testing.T) { cancel, appLister := newFakeLister() defer cancel() metricsServ := NewMetricsServer("localhost:8082", appLister, noOpHealthCheck) + appSyncTotal := ` +# HELP argocd_app_sync_total Number of application syncs. +# TYPE argocd_app_sync_total counter +argocd_app_sync_total{dest_server="https://localhost:6443",name="my-app",namespace="argocd",phase="Error",project="important-project"} 1 +argocd_app_sync_total{dest_server="https://localhost:6443",name="my-app",namespace="argocd",phase="Failed",project="important-project"} 1 +argocd_app_sync_total{dest_server="https://localhost:6443",name="my-app",namespace="argocd",phase="Succeeded",project="important-project"} 2 +` + fakeApp := newFakeApp(fakeApp) metricsServ.IncSync(fakeApp, &argoappv1.OperationState{Phase: argoappv1.OperationRunning}) metricsServ.IncSync(fakeApp, &argoappv1.OperationState{Phase: argoappv1.OperationFailed}) @@ -202,27 +246,31 @@ func TestMetricsSyncCounter(t *testing.T) { // assertMetricsPrinted asserts every line in the expected lines appears in the body func assertMetricsPrinted(t *testing.T, expectedLines, body string) { for _, line := range strings.Split(expectedLines, "\n") { + if line == "" { + continue + } assert.Contains(t, body, line) } } -const appReconcileMetrics = `argocd_app_reconcile_bucket{name="my-app",namespace="argocd",project="important-project",le="0.25"} 0 -argocd_app_reconcile_bucket{name="my-app",namespace="argocd",project="important-project",le="0.5"} 0 -argocd_app_reconcile_bucket{name="my-app",namespace="argocd",project="important-project",le="1"} 0 -argocd_app_reconcile_bucket{name="my-app",namespace="argocd",project="important-project",le="2"} 0 -argocd_app_reconcile_bucket{name="my-app",namespace="argocd",project="important-project",le="4"} 0 -argocd_app_reconcile_bucket{name="my-app",namespace="argocd",project="important-project",le="8"} 1 -argocd_app_reconcile_bucket{name="my-app",namespace="argocd",project="important-project",le="16"} 1 -argocd_app_reconcile_bucket{name="my-app",namespace="argocd",project="important-project",le="+Inf"} 1 -argocd_app_reconcile_sum{name="my-app",namespace="argocd",project="important-project"} 5 -argocd_app_reconcile_count{name="my-app",namespace="argocd",project="important-project"} 1 -` - func TestReconcileMetrics(t *testing.T) { cancel, appLister := newFakeLister() defer cancel() metricsServ := NewMetricsServer("localhost:8082", appLister, noOpHealthCheck) - + appReconcileMetrics := ` +# HELP argocd_app_reconcile Application reconciliation performance. +# TYPE argocd_app_reconcile histogram +argocd_app_reconcile_bucket{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",le="0.25"} 0 +argocd_app_reconcile_bucket{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",le="0.5"} 0 +argocd_app_reconcile_bucket{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",le="1"} 0 +argocd_app_reconcile_bucket{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",le="2"} 0 +argocd_app_reconcile_bucket{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",le="4"} 0 +argocd_app_reconcile_bucket{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",le="8"} 1 +argocd_app_reconcile_bucket{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",le="16"} 1 +argocd_app_reconcile_bucket{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project",le="+Inf"} 1 +argocd_app_reconcile_sum{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project"} 5 +argocd_app_reconcile_count{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project"} 1 +` fakeApp := newFakeApp(fakeApp) metricsServ.IncReconcile(fakeApp, 5*time.Second)