feat: Implement Server-Side Diff (#13663)

* feat: Implement Server-Side Diff Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * propagate the refreshtype to the diff config Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * Create the serverSideDiff config Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * chore: add featureflag utility package Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * remove featureflag package Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * add param Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * make ssd configurable with app annotation Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * add server-side-diff flags Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * apply the same logic regardless of the refresh type Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * fix gitops-engine reference Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * address review comments Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * Address review comments Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * docs: add docs related to server-side-diff Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * docs: update doc Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * Add config to include mutation webhooks Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * Address review comments Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * go mod update Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * Add sdd cache test case Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * fix ssd cache unit test Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * Update clidocs Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * update manifests Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * Fix procfile Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * additional doc changes Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> * update gitops-engine version Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com> --------- Signed-off-by: Leonardo Luz Almeida <leonardo_almeida@intuit.com>
2026-04-21 17:07:16 +00:00 · 2023-12-18 15:37:13 -05:00 · 2023-12-18 15:37:13 -05:00 · 82ca7a7f9c
commit 82ca7a7f9c
parent 9abc1cc837
26 changed files with 487 additions and 182 deletions
--- a/1
+++ b/1
@ -492,6 +492,7 @@ start-local: mod-vendor-local dep-ui-local cli-local
 	ARGOCD_ZJWT_FEATURE_FLAG=always \
 	ARGOCD_IN_CI=false \
 	ARGOCD_GPG_ENABLED=$(ARGOCD_GPG_ENABLED) \
+	BIN_MODE=$(ARGOCD_BIN_MODE) \
 	ARGOCD_E2E_TEST=false \
 	ARGOCD_APPLICATION_NAMESPACES=$(ARGOCD_APPLICATION_NAMESPACES) \
 		goreman -f $(ARGOCD_PROCFILE) start ${ARGOCD_START}
--- a/2
+++ b/2
@ -1,4 +1,4 @@
-controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-application-controller $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''}"
+controller: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-application-controller $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''} --server-side-diff-enabled=${ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF:-'false'}"
 api-server: [ "$BIN_MODE" = 'true' ] && COMMAND=./dist/argocd || COMMAND='go run ./cmd/main.go' && sh -c "FORCE_LOG_COLORS=1 ARGOCD_FAKE_IN_CLUSTER=true ARGOCD_TLS_DATA_PATH=${ARGOCD_TLS_DATA_PATH:-/tmp/argocd-local/tls} ARGOCD_SSH_DATA_PATH=${ARGOCD_SSH_DATA_PATH:-/tmp/argocd-local/ssh} ARGOCD_BINARY_NAME=argocd-server $COMMAND --loglevel debug --redis localhost:${ARGOCD_E2E_REDIS_PORT:-6379} --disable-auth=${ARGOCD_E2E_DISABLE_AUTH:-'true'} --insecure --dex-server http://localhost:${ARGOCD_E2E_DEX_PORT:-5556} --repo-server localhost:${ARGOCD_E2E_REPOSERVER_PORT:-8081} --port ${ARGOCD_E2E_APISERVER_PORT:-8080} --otlp-address=${ARGOCD_OTLP_ADDRESS} --application-namespaces=${ARGOCD_APPLICATION_NAMESPACES:-''}"
 dex: sh -c "ARGOCD_BINARY_NAME=argocd-dex go run github.com/argoproj/argo-cd/v2/cmd gendexcfg -o `pwd`/dist/dex.yaml && (test -f dist/dex.yaml || { echo 'Failed to generate dex configuration'; exit 1; }) && docker run --rm -p ${ARGOCD_E2E_DEX_PORT:-5556}:${ARGOCD_E2E_DEX_PORT:-5556} -v `pwd`/dist/dex.yaml:/dex.yaml ghcr.io/dexidp/dex:$(grep "image: ghcr.io/dexidp/dex" manifests/base/dex/argocd-dex-server-deployment.yaml | cut -d':' -f3) dex serve /dex.yaml"
 redis: bash -c "if [ \"$ARGOCD_REDIS_LOCAL\" = 'true' ]; then redis-server --save '' --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}; else docker run --rm --name argocd-redis -i -p ${ARGOCD_E2E_REDIS_PORT:-6379}:${ARGOCD_E2E_REDIS_PORT:-6379} docker.io/library/redis:$(grep "image: redis" manifests/base/redis/argocd-redis-deployment.yaml | cut -d':' -f3) --save '' --appendonly no --port ${ARGOCD_E2E_REDIS_PORT:-6379}; fi"
--- a/cmd/argocd-application-controller/commands/argocd_application_controller.go
+++ b/cmd/argocd-application-controller/commands/argocd_application_controller.go
@ -73,6 +73,7 @@ func NewCommand() *cobra.Command {
 		persistResourceHealth            bool
 		shardingAlgorithm                string
 		enableDynamicClusterDistribution bool
+		serverSideDiff                   bool
 	)
 	var command = cobra.Command{
 		Use:               cliName,
@ -166,6 +167,7 @@ func NewCommand() *cobra.Command {
 				clusterFilter,
 				applicationNamespaces,
 				&workqueueRateLimit,
+				serverSideDiff,
 			)
 			errors.CheckError(err)
 			cacheutil.CollectMetrics(redisClient, appController.GetMetricsServer())
@ -224,6 +226,7 @@ func NewCommand() *cobra.Command {
 	command.Flags().DurationVar(&workqueueRateLimit.MaxDelay, "wq-maxdelay-ns", time.Duration(env.ParseInt64FromEnv("WORKQUEUE_MAX_DELAY_NS", time.Second.Nanoseconds(), 1*time.Millisecond.Nanoseconds(), (24*time.Hour).Nanoseconds())), "Set Workqueue Per Item Rate Limiter Max Delay duration in nanoseconds, default 1000000000 (1s)")
 	command.Flags().Float64Var(&workqueueRateLimit.BackoffFactor, "wq-backoff-factor", env.ParseFloat64FromEnv("WORKQUEUE_BACKOFF_FACTOR", 1.5, 0, math.MaxFloat64), "Set Workqueue Per Item Rate Limiter Backoff Factor, default is 1.5")
 	command.Flags().BoolVar(&enableDynamicClusterDistribution, "dynamic-cluster-distribution-enabled", env.ParseBoolFromEnv(common.EnvEnableDynamicClusterDistribution, false), "Enables dynamic cluster distribution.")
+	command.Flags().BoolVar(&serverSideDiff, "server-side-diff-enabled", env.ParseBoolFromEnv(common.EnvServerSideDiff, false), "Feature flag to enable ServerSide diff. Default (\"false\")")
 	cacheSource = appstatecache.AddCacheFlagsToCmd(&command, func(client *redis.Client) {
 		redisClient = client
 	})
--- a/cmd/argocd/commands/admin/app.go
+++ b/cmd/argocd/commands/admin/app.go
@ -243,6 +243,7 @@ func NewReconcileCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command
 		repoServerAddress string
 		outputFormat      string
 		refresh           bool
+		serverSideDiff    bool
 	)

 	var command = &cobra.Command{
@ -280,7 +281,7 @@ func NewReconcileCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command

 				appClientset := appclientset.NewForConfigOrDie(cfg)
 				kubeClientset := kubernetes.NewForConfigOrDie(cfg)
-				result, err = reconcileApplications(ctx, kubeClientset, appClientset, namespace, repoServerClient, selector, newLiveStateCache)
+				result, err = reconcileApplications(ctx, kubeClientset, appClientset, namespace, repoServerClient, selector, newLiveStateCache, serverSideDiff)
 				errors.CheckError(err)
 			} else {
 				appClientset := appclientset.NewForConfigOrDie(cfg)
@ -295,6 +296,7 @@ func NewReconcileCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command
 	command.Flags().StringVar(&selector, "l", "", "Label selector")
 	command.Flags().StringVar(&outputFormat, "o", "yaml", "Output format (yaml|json)")
 	command.Flags().BoolVar(&refresh, "refresh", false, "If set to true then recalculates apps reconciliation")
+	command.Flags().BoolVar(&serverSideDiff, "server-side-diff", false, "If set to \"true\" will use server-side diff while comparing resources. Default (\"false\")")

 	return command
 }
@ -344,6 +346,7 @@ func reconcileApplications(
 	repoServerClient reposerverclient.Clientset,
 	selector string,
 	createLiveStateCache func(argoDB db.ArgoDB, appInformer kubecache.SharedIndexInformer, settingsMgr *settings.SettingsManager, server *metrics.MetricsServer) cache.LiveStateCache,
+	serverSideDiff bool,
 ) ([]appReconcileResult, error) {
 	settingsMgr := settings.NewSettingsManager(ctx, kubeClientset, namespace)
 	argoDB := db.NewDB(namespace, settingsMgr, kubeClientset)
@ -384,7 +387,7 @@ func reconcileApplications(
 	)

 	appStateManager := controller.NewAppStateManager(
-		argoDB, appClientset, repoServerClient, namespace, kubeutil.NewKubectl(), settingsMgr, stateCache, projInformer, server, cache, time.Second, argo.NewResourceTracking(), false, 0)
+		argoDB, appClientset, repoServerClient, namespace, kubeutil.NewKubectl(), settingsMgr, stateCache, projInformer, server, cache, time.Second, argo.NewResourceTracking(), false, 0, serverSideDiff)

 	appsList, err := appClientset.ArgoprojV1alpha1().Applications(namespace).List(ctx, v1.ListOptions{LabelSelector: selector})
 	if err != nil {
--- a/cmd/argocd/commands/admin/app_test.go
+++ b/cmd/argocd/commands/admin/app_test.go
@ -113,6 +113,7 @@ func TestGetReconcileResults_Refresh(t *testing.T) {
 		func(argoDB db.ArgoDB, appInformer cache.SharedIndexInformer, settingsMgr *settings.SettingsManager, server *metrics.MetricsServer) statecache.LiveStateCache {
 			return &liveStateCache
 		},
+		false,
 	)

 	if !assert.NoError(t, err) {
--- a/common/common.go
+++ b/common/common.go
@ -260,6 +260,9 @@ const (
 	EnvRedisHaProxyName = "ARGOCD_REDIS_HAPROXY_NAME"
 	// EnvGRPCKeepAliveMin defines the GRPCKeepAliveEnforcementMinimum, used in the grpc.KeepaliveEnforcementPolicy. Expects a "Duration" format (e.g. 10s).
 	EnvGRPCKeepAliveMin = "ARGOCD_GRPC_KEEP_ALIVE_MIN"
+	// EnvServerSideDiff defines the env var used to enable ServerSide Diff feature.
+	// If defined, value must be "true" or "false".
+	EnvServerSideDiff = "ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF"
 )

 // Config Management Plugin related constants
--- a/controller/appcontroller.go
+++ b/controller/appcontroller.go
@ -152,6 +152,7 @@ func NewApplicationController(
 	clusterFilter func(cluster *appv1.Cluster) bool,
 	applicationNamespaces []string,
 	rateLimiterConfig *ratelimiter.AppControllerRateLimiterConfig,
+	serverSideDiff bool,
 ) (*ApplicationController, error) {
 	log.Infof("appResyncPeriod=%v, appHardResyncPeriod=%v", appResyncPeriod, appHardResyncPeriod)
 	db := db.NewDB(namespace, settingsMgr, kubeClientset)
@ -260,7 +261,7 @@ func NewApplicationController(
 		}
 	}
 	stateCache := statecache.NewLiveStateCache(db, appInformer, ctrl.settingsMgr, kubectl, ctrl.metricsServer, ctrl.handleObjectUpdated, clusterFilter, argo.NewResourceTracking())
-	appStateManager := NewAppStateManager(db, applicationClientset, repoClientset, namespace, kubectl, ctrl.settingsMgr, stateCache, projInformer, ctrl.metricsServer, argoCache, ctrl.statusRefreshTimeout, argo.NewResourceTracking(), persistResourceHealth, repoErrorGracePeriod)
+	appStateManager := NewAppStateManager(db, applicationClientset, repoClientset, namespace, kubectl, ctrl.settingsMgr, stateCache, projInformer, ctrl.metricsServer, argoCache, ctrl.statusRefreshTimeout, argo.NewResourceTracking(), persistResourceHealth, repoErrorGracePeriod, serverSideDiff)
 	ctrl.appInformer = appInformer
 	ctrl.appLister = appLister
 	ctrl.projInformer = projInformer
--- a/controller/appcontroller_test.go
+++ b/controller/appcontroller_test.go
@ -152,6 +152,7 @@ func newFakeController(data *fakeData, repoErr error) *ApplicationController {
 		nil,
 		data.applicationNamespaces,
 		nil,
+		false,
 	)
 	if err != nil {
 		panic(err)
--- a/controller/state.go
+++ b/controller/state.go
@ -116,6 +116,7 @@ type appStateManager struct {
 	persistResourceHealth bool
 	repoErrorCache        goSync.Map
 	repoErrorGracePeriod  time.Duration
+	serverSideDiff        bool
 }

 // GetRepoObjs will generate the manifests for the given application delegating the
@ -592,7 +593,16 @@ func (m *appStateManager) CompareAppState(app *v1alpha1.Application, project *v1
 		manifestRevisions = append(manifestRevisions, manifestInfo.Revision)
 	}

-	useDiffCache := useDiffCache(noCache, manifestInfos, sources, app, manifestRevisions, m.statusRefreshTimeout, logCtx)
+	serverSideDiff := m.serverSideDiff ||
+		resourceutil.HasAnnotationOption(app, common.AnnotationCompareOptions, "ServerSideDiff=true")
+
+	// This allows turning SSD off for a given app if it is enabled at the
+	// controller level
+	if resourceutil.HasAnnotationOption(app, common.AnnotationCompareOptions, "ServerSideDiff=false") {
+		serverSideDiff = false
+	}
+
+	useDiffCache := useDiffCache(noCache, manifestInfos, sources, app, manifestRevisions, m.statusRefreshTimeout, serverSideDiff, logCtx)

 	diffConfigBuilder := argodiff.NewDiffConfigBuilder().
 		WithDiffSettings(app.Spec.IgnoreDifferences, resourceOverrides, compareOptions.IgnoreAggregatedRoles).
@ -604,6 +614,10 @@ func (m *appStateManager) CompareAppState(app *v1alpha1.Application, project *v1
 		diffConfigBuilder.WithNoCache()
 	}

+	if resourceutil.HasAnnotationOption(app, common.AnnotationCompareOptions, "IncludeMutationWebhook=true") {
+		diffConfigBuilder.WithIgnoreMutationWebhook(false)
+	}
+
 	gvkParser, err := m.getGVKParser(app.Spec.Destination.Server)
 	if err != nil {
 		conditions = append(conditions, v1alpha1.ApplicationCondition{Type: v1alpha1.ApplicationConditionUnknownError, Message: err.Error(), LastTransitionTime: &now})
@ -611,6 +625,18 @@ func (m *appStateManager) CompareAppState(app *v1alpha1.Application, project *v1
 	diffConfigBuilder.WithGVKParser(gvkParser)
 	diffConfigBuilder.WithManager(common.ArgoCDSSAManager)

+	diffConfigBuilder.WithServerSideDiff(serverSideDiff)
+
+	if serverSideDiff {
+		resourceOps, cleanup, err := m.getResourceOperations(app.Spec.Destination.Server)
+		if err != nil {
+			log.Errorf("CompareAppState error getting resource operations: %s", err)
+			conditions = append(conditions, v1alpha1.ApplicationCondition{Type: v1alpha1.ApplicationConditionUnknownError, Message: err.Error(), LastTransitionTime: &now})
+		}
+		defer cleanup()
+		diffConfigBuilder.WithServerSideDryRunner(diff.NewK8sServerSideDryRunner(resourceOps))
+	}
+
 	// enable structured merge diff if application syncs with server-side apply
 	if app.Spec.SyncPolicy != nil && app.Spec.SyncPolicy.SyncOptions.HasOption("ServerSideApply=true") {
 		diffConfigBuilder.WithStructuredMergeDiff(true)
@ -811,18 +837,23 @@ func (m *appStateManager) CompareAppState(app *v1alpha1.Application, project *v1

 // useDiffCache will determine if the diff should be calculated based
 // on the existing live state cache or not.
-func useDiffCache(noCache bool, manifestInfos []*apiclient.ManifestResponse, sources []v1alpha1.ApplicationSource, app *v1alpha1.Application, manifestRevisions []string, statusRefreshTimeout time.Duration, log *log.Entry) bool {
+func useDiffCache(noCache bool, manifestInfos []*apiclient.ManifestResponse, sources []v1alpha1.ApplicationSource, app *v1alpha1.Application, manifestRevisions []string, statusRefreshTimeout time.Duration, serverSideDiff bool, log *log.Entry) bool {

 	if noCache {
 		log.WithField("useDiffCache", "false").Debug("noCache is true")
 		return false
 	}
-	_, refreshRequested := app.IsRefreshRequested()
+	refreshType, refreshRequested := app.IsRefreshRequested()
 	if refreshRequested {
-		log.WithField("useDiffCache", "false").Debug("refreshRequested")
+		log.WithField("useDiffCache", "false").Debugf("refresh type %s requested", string(refreshType))
 		return false
 	}
-	if app.Status.Expired(statusRefreshTimeout) {
+	// serverSideDiff should still use cache even if status is expired.
+	// This is an attempt to avoid hitting k8s API server too frequently during
+	// app refresh with serverSideDiff is enabled. If there are negative side
+	// effects identified with this approach, the serverSideDiff should be removed
+	// from this condition.
+	if app.Status.Expired(statusRefreshTimeout) && !serverSideDiff {
 		log.WithField("useDiffCache", "false").Debug("app.status.expired")
 		return false
 	}
@ -903,6 +934,7 @@ func NewAppStateManager(
 	resourceTracking argo.ResourceTracking,
 	persistResourceHealth bool,
 	repoErrorGracePeriod time.Duration,
+	serverSideDiff bool,
 ) AppStateManager {
 	return &appStateManager{
 		liveStateCache:        liveStateCache,
@ -919,6 +951,7 @@ func NewAppStateManager(
 		resourceTracking:      resourceTracking,
 		persistResourceHealth: persistResourceHealth,
 		repoErrorGracePeriod:  repoErrorGracePeriod,
+		serverSideDiff:        serverSideDiff,
 	}
 }

--- a/controller/state_test.go
+++ b/controller/state_test.go
@ -1407,6 +1407,7 @@ func TestUseDiffCache(t *testing.T) {
 		manifestRevisions    []string
 		statusRefreshTimeout time.Duration
 		expectedUseCache     bool
+		serverSideDiff       bool
 	}

 	manifestInfos := func(revision string) []*apiclient.ManifestResponse {
@ -1505,6 +1506,7 @@ func TestUseDiffCache(t *testing.T) {
 			manifestRevisions:    []string{"rev1"},
 			statusRefreshTimeout: time.Hour * 24,
 			expectedUseCache:     true,
+			serverSideDiff:       false,
 		},
 		{
 			testName:      "will use diff cache for multisource",
@ -1548,6 +1550,7 @@ func TestUseDiffCache(t *testing.T) {
 			manifestRevisions:    []string{"rev1", "rev2"},
 			statusRefreshTimeout: time.Hour * 24,
 			expectedUseCache:     true,
+			serverSideDiff:       false,
 		},
 		{
 			testName:             "will return false if nocache is true",
@ -1558,6 +1561,7 @@ func TestUseDiffCache(t *testing.T) {
 			manifestRevisions:    []string{"rev1"},
 			statusRefreshTimeout: time.Hour * 24,
 			expectedUseCache:     false,
+			serverSideDiff:       false,
 		},
 		{
 			testName:             "will return false if requested refresh",
@ -1568,6 +1572,7 @@ func TestUseDiffCache(t *testing.T) {
 			manifestRevisions:    []string{"rev1"},
 			statusRefreshTimeout: time.Hour * 24,
 			expectedUseCache:     false,
+			serverSideDiff:       false,
 		},
 		{
 			testName:             "will return false if status expired",
@ -1578,6 +1583,18 @@ func TestUseDiffCache(t *testing.T) {
 			manifestRevisions:    []string{"rev1"},
 			statusRefreshTimeout: time.Minute,
 			expectedUseCache:     false,
+			serverSideDiff:       false,
+		},
+		{
+			testName:             "will return true if status expired and server-side diff",
+			noCache:              false,
+			manifestInfos:        manifestInfos("rev1"),
+			sources:              sources(),
+			app:                  app("httpbin", "rev1", false, nil),
+			manifestRevisions:    []string{"rev1"},
+			statusRefreshTimeout: time.Minute,
+			expectedUseCache:     true,
+			serverSideDiff:       true,
 		},
 		{
 			testName:             "will return false if there is a new revision",
@ -1588,6 +1605,7 @@ func TestUseDiffCache(t *testing.T) {
 			manifestRevisions:    []string{"rev2"},
 			statusRefreshTimeout: time.Hour * 24,
 			expectedUseCache:     false,
+			serverSideDiff:       false,
 		},
 		{
 			testName:      "will return false if app spec repo changed",
@ -1604,6 +1622,7 @@ func TestUseDiffCache(t *testing.T) {
 			manifestRevisions:    []string{"rev1"},
 			statusRefreshTimeout: time.Hour * 24,
 			expectedUseCache:     false,
+			serverSideDiff:       false,
 		},
 		{
 			testName:      "will return false if app spec IgnoreDifferences changed",
@ -1626,6 +1645,7 @@ func TestUseDiffCache(t *testing.T) {
 			manifestRevisions:    []string{"rev1"},
 			statusRefreshTimeout: time.Hour * 24,
 			expectedUseCache:     false,
+			serverSideDiff:       false,
 		},
 	}

@ -1638,7 +1658,7 @@ func TestUseDiffCache(t *testing.T) {
 			log := logrus.NewEntry(logger)

 			// When
-			useDiffCache := useDiffCache(tc.noCache, tc.manifestInfos, tc.sources, tc.app, tc.manifestRevisions, tc.statusRefreshTimeout, log)
+			useDiffCache := useDiffCache(tc.noCache, tc.manifestInfos, tc.sources, tc.app, tc.manifestRevisions, tc.statusRefreshTimeout, tc.serverSideDiff, log)

 			// Then
 			assert.Equal(t, useDiffCache, tc.expectedUseCache)
--- a/controller/sync.go
+++ b/controller/sync.go
@ -57,6 +57,27 @@ func (m *appStateManager) getGVKParser(server string) (*managedfields.GvkParser,
 	return cluster.GetGVKParser(), nil
 }

+// getResourceOperations will return the kubectl implementation of the ResourceOperations
+// interface that provides functionality to manage kubernetes resources. Returns a
+// cleanup function that must be called to remove the generated kube config for this
+// server.
+func (m *appStateManager) getResourceOperations(server string) (kube.ResourceOperations, func(), error) {
+	clusterCache, err := m.liveStateCache.GetClusterCache(server)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error getting cluster cache: %w", err)
+	}
+
+	cluster, err := m.db.GetCluster(context.Background(), server)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error getting cluster: %w", err)
+	}
+	ops, cleanup, err := m.kubectl.ManageResources(cluster.RawRestConfig(), clusterCache.GetOpenAPISchema())
+	if err != nil {
+		return nil, nil, fmt.Errorf("error creating kubectl ResourceOperations: %w", err)
+	}
+	return ops, cleanup, nil
+}
+
 func (m *appStateManager) SyncAppState(app *v1alpha1.Application, state *v1alpha1.OperationState) {
 	// Sync requests might be requested with ambiguous revisions (e.g. master, HEAD, v1.2.3).
 	// This can change meaning when resuming operations (e.g a hook sync). After calculating a
--- a/docs/operator-manual/argocd-cmd-params-cm.yaml
+++ b/docs/operator-manual/argocd-cmd-params-cm.yaml
@ -68,6 +68,10 @@ data:
  controller.k8sclient.retry.base.backoff: "100"
  # Grace period in seconds for ignoring consecutive errors while communicating with repo server.
  controller.repo.error.grace.period.seconds: "180"
+  # Enables the server side diff feature at the application controller level.
+  # Diff calculation will be done by running a server side apply dryrun (when
+  # diff cache is unavailable).
+  controller.diff.server.side: "false"

  ## Server properties
  # Listen on given address for incoming connections (default "0.0.0.0")
--- a/docs/operator-manual/server-commands/argocd-application-controller.md
+++ b/docs/operator-manual/server-commands/argocd-application-controller.md
@ -67,6 +67,7 @@ argocd-application-controller [flags]
      --sentinel stringArray                   Redis sentinel hostname and port (e.g. argocd-redis-ha-announce-0:6379). 
      --sentinelmaster string                  Redis sentinel master group name. (default "master")
      --server string                          The address and port of the Kubernetes API server
+      --server-side-diff-enabled               Feature flag to enable ServerSide diff. Default ("false")
      --sharding-method string                 Enables choice of sharding method. Supported sharding methods are : [legacy, round-robin]  (default "legacy")
      --status-processors int                  Number of application status processors (default 20)
      --tls-server-name string                 If provided, this name will be used to validate server certificate. If this is not provided, hostname used to contact the server is used.
--- a/docs/user-guide/commands/argocd_admin_app_get-reconcile-results.md
+++ b/docs/user-guide/commands/argocd_admin_app_get-reconcile-results.md
@ -32,6 +32,7 @@ argocd admin app get-reconcile-results PATH [flags]
      --repo-server string             Repo server address.
      --request-timeout string         The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
      --server string                  The address and port of the Kubernetes API server
+      --server-side-diff               If set to "true" will use server-side diff while comparing resources. Default ("false")
      --tls-server-name string         If provided, this name will be used to validate server certificate. If this is not provided, hostname used to contact the server is used.
      --token string                   Bearer token for authentication to the API server
      --user string                    The name of the kubeconfig user to use
--- a/docs/user-guide/diff-strategies.md
+++ b/docs/user-guide/diff-strategies.md
@ -0,0 +1,125 @@
+# Diff Strategies
+
+Argo CD calculates the diff between the desired state and the live
+state in order to define if an Application is out-of-sync. This same
+logic is also used in Argo CD UI to display the differences between
+live and desired states for all resources belonging to an application.
+
+Argo CD currently has 3 different strategies to calculate diffs:
+
+- **Legacy**: This is the main diff strategy used by default. It
+  applies a 3-way diff based on live state, desired state and
+  last-applied-configuration (annotation).
+- **Structured-Merge Diff**: Strategy automatically applied when
+  enabling Server-Side Apply sync option. 
+- **Server-Side Diff**: New strategy that invokes a Server-Side Apply
+  in dryrun mode in order to generate the predicted live state.
+
+## Structured-Merge Diff
+*Current Status: [Beta][1] (Since v2.5.0)*
+
+This is diff strategy is automatically used when Server-Side Apply
+sync option is enabled. It uses the [structured-merge-diff][2] library
+used by Kubernetes to calculate diffs based on fields ownership. There
+are some challenges using this strategy to calculate diffs for CRDs
+that define default values. After different issues were identified by
+the community, this strategy is being discontinued in favour of
+Server-Side Diff.
+
+## Server-Side Diff
+*Current Status: [Beta][1] (Since v2.10.0)*
+
+This diff strategy will execute a Server-Side Apply in dryrun mode for
+each resource of the application. The response of this operation is then
+compared with the live state in order to provide the diff results. The
+diff results are cached and new Server-Side Apply requests to Kube API
+are only triggered when:
+
+- An Application refresh or hard-refresh is requested.
+- There is a new revision in the repo which the Argo CD Application is
+  targeting.
+- The Argo CD Application spec changed.
+
+One advantage of Server-Side Diff is that Kubernetes Admission
+Controllers will participate in the diff calculation. If for example
+a validation webhook identifies a resource to be invalid, that will be
+informed to Argo CD during the diff stage rather than during the sync 
+stage.
+
+### Enabling it
+
+Server-Side Diff can be enabled at the Argo CD Controller level or per
+Application.
+
+**Enabling Server-Side Diff for all Applications**
+
+Add the following entry in the argocd-cmd-params-cm configmap:
+
+```
+controller.diff.server.side: "true"
+```
+
+Note: It is necessary to restart the `argocd-application-controller`
+after applying this configuration.
+
+**Enabling Server-Side Diff for one application**
+
+Add the following annotation in the Argo CD Application resource:
+
+```
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/compare-options: ServerSideDiff=true
+...
+```
+
+**Disabling Server-Side Diff for one application**
+
+If Server-Side Diff is enabled globally in your Argo CD instance, it
+is possible to disable it at the application level. In order to do so,
+add the following annotation in the Application resource:
+
+```
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/compare-options: ServerSideDiff=false
+...
+```
+
+*Note: Please report any issues that forced you to disable the
+Server-Side Diff feature*
+
+### Mutation Webhooks
+
+Server-Side Diff does not include changes made by mutation webhooks by
+default. If you want to include mutation webhooks in Argo CD diffs add
+the following annotation in the Argo CD Application resource:
+
+```
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/compare-options: IncludeMutationWebhook=true
+...
+```
+
+Note: This annoation is only effective when Server-Side Diff is
+enabled. To enable both options for a given application add the
+following annotation in the Argo CD Application resource:
+
+```
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  annotations:
+    argocd.argoproj.io/compare-options: ServerSideDiff=true,IncludeMutationWebhook=true
+...
+```
+
+[1]: https://github.com/argoproj/argoproj/blob/main/community/feature-status.md#beta
+[2]: https://github.com/kubernetes-sigs/structured-merge-diff
--- a/go.mod
+++ b/go.mod
@ -13,7 +13,7 @@ require (
 	github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d
 	github.com/alicebob/miniredis/v2 v2.30.4
 	github.com/antonmedv/expr v1.15.2
-	github.com/argoproj/gitops-engine v0.7.1-0.20231102154024-c0c2dd1f6f48
+	github.com/argoproj/gitops-engine v0.7.1-0.20231218194513-aba38192fb16
 	github.com/argoproj/notifications-engine v0.4.1-0.20231027194313-a8d185ecc0a9
 	github.com/argoproj/pkg v0.13.7-0.20230626144333-d56162821bd1
 	github.com/aws/aws-sdk-go v1.44.317
@ -104,7 +104,7 @@ require (
 	layeh.com/gopher-json v0.0.0-20190114024228-97fed8db8427
 	oras.land/oras-go/v2 v2.3.0
 	sigs.k8s.io/controller-runtime v0.14.6
-	sigs.k8s.io/structured-merge-diff/v4 v4.3.0
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1
 	sigs.k8s.io/yaml v1.3.0
 )

--- a/go.sum
+++ b/go.sum
@ -696,8 +696,8 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
 github.com/appscode/go v0.0.0-20191119085241-0887d8ec2ecc/go.mod h1:OawnOmAL4ZX3YaPdN+8HTNwBveT1jMsqP74moa9XUbE=
-github.com/argoproj/gitops-engine v0.7.1-0.20231102154024-c0c2dd1f6f48 h1:vnXMrNkBFC0H0KBkH1Jno31OVgJQR4KSd0ypEcfzQRA=
-github.com/argoproj/gitops-engine v0.7.1-0.20231102154024-c0c2dd1f6f48/go.mod h1:1TchqKw9XmYYZluyEHa1dTJQoZgbV6PhabB/e8Wf3KY=
+github.com/argoproj/gitops-engine v0.7.1-0.20231218194513-aba38192fb16 h1:kR15L8UsSVr7oitABKU88msQirMT0/RO/KRla1jkq/s=
+github.com/argoproj/gitops-engine v0.7.1-0.20231218194513-aba38192fb16/go.mod h1:gWE8uROi7hIkWGNAVM+8FWkMfo0vZ03SLx/aFw/DBzg=
 github.com/argoproj/notifications-engine v0.4.1-0.20231027194313-a8d185ecc0a9 h1:1lt0VXzmLK7Vv0kaeal3S6/JIfzPyBORkUWXhiqF3l0=
 github.com/argoproj/notifications-engine v0.4.1-0.20231027194313-a8d185ecc0a9/go.mod h1:E/vv4+by868m0mmflaRfGBmKBtAupoF+mmyfekP8QCk=
 github.com/argoproj/pkg v0.13.7-0.20230626144333-d56162821bd1 h1:qsHwwOJ21K2Ao0xPju1sNuqphyMnMYkyB3ZLoLtxWpo=
@ -2715,8 +2715,8 @@ sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCY
 sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk=
 sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
-sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk=
-sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
--- a/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml
+++ b/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml
@ -34,24 +34,30 @@ spec:
              name: argocd-cm
              key: timeout.hard.reconciliation
              optional: true
+        - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.repo.error.grace.period.seconds
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: repo.server
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: repo.server
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_TIMEOUT_SECONDS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.repo.server.timeout.seconds
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.repo.server.timeout.seconds
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_STATUS_PROCESSORS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.status.processors
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.status.processors
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_OPERATION_PROCESSORS
          valueFrom:
            configMapKeyRef:
@ -78,22 +84,22 @@ spec:
              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_SELF_HEAL_TIMEOUT_SECONDS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.self.heal.timeout.seconds
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.self.heal.timeout.seconds
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_PLAINTEXT
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.repo.server.plaintext
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.repo.server.plaintext
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_STRICT_TLS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.repo.server.strict.tls
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.repo.server.strict.tls
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_PERSIST_RESOURCE_HEALTH
          valueFrom:
            configMapKeyRef:
@ -102,16 +108,16 @@ spec:
              optional: true
        - name: ARGOCD_APP_STATE_CACHE_EXPIRATION
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.app.state.cache.expiration
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.app.state.cache.expiration
+              optional: true
        - name: REDIS_SERVER
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: redis.server
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: redis.server
+              optional: true
        - name: REDIS_COMPRESSION
          valueFrom:
            configMapKeyRef:
@ -120,70 +126,70 @@ spec:
              optional: true
        - name: REDISDB
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: redis.db
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: redis.db
+              optional: true
        - name: ARGOCD_DEFAULT_CACHE_EXPIRATION
-          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.default.cache.expiration
-                optional: true
-        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_ADDRESS
-          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: otlp.address
-                optional: true
-        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_INSECURE
-          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: otlp.insecure
-                optional: true
-        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_HEADERS
-          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: otlp.headers
-                optional: true
-        - name: ARGOCD_APPLICATION_NAMESPACES
-          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: application.namespaces
-                optional: true
-        - name: ARGOCD_CONTROLLER_SHARDING_ALGORITHM
-          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.sharding.algorithm
-                optional: true
-        - name: ARGOCD_APPLICATION_CONTROLLER_KUBECTL_PARALLELISM_LIMIT
-          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.kubectl.parallelism.limit
-                optional: true
-        - name: ARGOCD_CONTROLLER_HEARTBEAT_TIME
          valueFrom:
            configMapKeyRef:
              name: argocd-cmd-params-cm
-              key: controller.heatbeat.time
+              key: controller.default.cache.expiration
+              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_ADDRESS
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: otlp.address
+              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_INSECURE
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: otlp.insecure
+              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_HEADERS
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: otlp.headers
+              optional: true
+        - name: ARGOCD_APPLICATION_NAMESPACES
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: application.namespaces
+              optional: true
+        - name: ARGOCD_CONTROLLER_SHARDING_ALGORITHM
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.sharding.algorithm
+              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_KUBECTL_PARALLELISM_LIMIT
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.kubectl.parallelism.limit
              optional: true
        - name: ARGOCD_K8SCLIENT_RETRY_MAX
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.k8sclient.retry.max
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.k8sclient.retry.max
+              optional: true
        - name: ARGOCD_K8SCLIENT_RETRY_BASE_BACKOFF
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.k8sclient.retry.base.backoff
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.k8sclient.retry.base.backoff
+              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.diff.server.side
+              optional: true
        image: quay.io/argoproj/argocd:latest
        imagePullPolicy: Always
        name: argocd-application-controller
--- a/manifests/base/application-controller/argocd-application-controller-statefulset.yaml
+++ b/manifests/base/application-controller/argocd-application-controller-statefulset.yaml
@ -43,22 +43,22 @@ spec:
              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: repo.server
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: repo.server
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_TIMEOUT_SECONDS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.repo.server.timeout.seconds
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.repo.server.timeout.seconds
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_STATUS_PROCESSORS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.status.processors
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.status.processors
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_OPERATION_PROCESSORS
          valueFrom:
            configMapKeyRef:
@ -85,22 +85,22 @@ spec:
              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_SELF_HEAL_TIMEOUT_SECONDS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.self.heal.timeout.seconds
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.self.heal.timeout.seconds
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_PLAINTEXT
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.repo.server.plaintext
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.repo.server.plaintext
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_STRICT_TLS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.repo.server.strict.tls
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.repo.server.strict.tls
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_PERSIST_RESOURCE_HEALTH
          valueFrom:
            configMapKeyRef:
@ -109,16 +109,16 @@ spec:
              optional: true
        - name: ARGOCD_APP_STATE_CACHE_EXPIRATION
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.app.state.cache.expiration
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.app.state.cache.expiration
+              optional: true
        - name: REDIS_SERVER
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: redis.server
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: redis.server
+              optional: true
        - name: REDIS_COMPRESSION
          valueFrom:
            configMapKeyRef:
@ -127,64 +127,70 @@ spec:
              optional: true
        - name: REDISDB
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: redis.db
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: redis.db
+              optional: true
        - name: ARGOCD_DEFAULT_CACHE_EXPIRATION
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.default.cache.expiration
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.default.cache.expiration
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_ADDRESS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: otlp.address
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: otlp.address
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_INSECURE
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: otlp.insecure
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: otlp.insecure
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_HEADERS
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: otlp.headers
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: otlp.headers
+              optional: true
        - name: ARGOCD_APPLICATION_NAMESPACES
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: application.namespaces
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: application.namespaces
+              optional: true
        - name: ARGOCD_CONTROLLER_SHARDING_ALGORITHM
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.sharding.algorithm
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.sharding.algorithm
+              optional: true
        - name: ARGOCD_APPLICATION_CONTROLLER_KUBECTL_PARALLELISM_LIMIT
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.kubectl.parallelism.limit
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.kubectl.parallelism.limit
+              optional: true
        - name: ARGOCD_K8SCLIENT_RETRY_MAX
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.k8sclient.retry.max
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.k8sclient.retry.max
+              optional: true
        - name: ARGOCD_K8SCLIENT_RETRY_BASE_BACKOFF
          valueFrom:
-              configMapKeyRef:
-                name: argocd-cmd-params-cm
-                key: controller.k8sclient.retry.base.backoff
-                optional: true
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.k8sclient.retry.base.backoff
+              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cmd-params-cm
+              key: controller.diff.server.side
+              optional: true
        image: quay.io/argoproj/argocd:latest
        imagePullPolicy: Always
        name: argocd-application-controller
--- a/manifests/core-install.yaml
+++ b/manifests/core-install.yaml
@ -21643,6 +21643,12 @@ spec:
              key: controller.k8sclient.retry.base.backoff
              name: argocd-cmd-params-cm
              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF
+          valueFrom:
+            configMapKeyRef:
+              key: controller.diff.server.side
+              name: argocd-cmd-params-cm
+              optional: true
        image: quay.io/argoproj/argocd:latest
        imagePullPolicy: Always
        name: argocd-application-controller
--- a/manifests/ha/install.yaml
+++ b/manifests/ha/install.yaml
@ -23476,6 +23476,12 @@ spec:
              key: controller.k8sclient.retry.base.backoff
              name: argocd-cmd-params-cm
              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF
+          valueFrom:
+            configMapKeyRef:
+              key: controller.diff.server.side
+              name: argocd-cmd-params-cm
+              optional: true
        image: quay.io/argoproj/argocd:latest
        imagePullPolicy: Always
        name: argocd-application-controller
--- a/manifests/ha/namespace-install.yaml
+++ b/manifests/ha/namespace-install.yaml
@ -2861,6 +2861,12 @@ spec:
              key: controller.k8sclient.retry.base.backoff
              name: argocd-cmd-params-cm
              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF
+          valueFrom:
+            configMapKeyRef:
+              key: controller.diff.server.side
+              name: argocd-cmd-params-cm
+              optional: true
        image: quay.io/argoproj/argocd:latest
        imagePullPolicy: Always
        name: argocd-application-controller
--- a/manifests/install.yaml
+++ b/manifests/install.yaml
@ -22520,6 +22520,12 @@ spec:
              key: controller.k8sclient.retry.base.backoff
              name: argocd-cmd-params-cm
              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF
+          valueFrom:
+            configMapKeyRef:
+              key: controller.diff.server.side
+              name: argocd-cmd-params-cm
+              optional: true
        image: quay.io/argoproj/argocd:latest
        imagePullPolicy: Always
        name: argocd-application-controller
--- a/manifests/namespace-install.yaml
+++ b/manifests/namespace-install.yaml
@ -1905,6 +1905,12 @@ spec:
              key: controller.k8sclient.retry.base.backoff
              name: argocd-cmd-params-cm
              optional: true
+        - name: ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF
+          valueFrom:
+            configMapKeyRef:
+              key: controller.diff.server.side
+              name: argocd-cmd-params-cm
+              optional: true
        image: quay.io/argoproj/argocd:latest
        imagePullPolicy: Always
        name: argocd-application-controller
--- a/mkdocs.yml
+++ b/mkdocs.yml
@ -162,7 +162,9 @@ nav:
  - user-guide/multiple_sources.md
  - GnuPG verification: user-guide/gpg-verification.md
  - user-guide/auto_sync.md
-  - user-guide/diffing.md
+  - Diffing:
+    - Diff Strategies: user-guide/diff-strategies.md
+    - Diff Customization: user-guide/diffing.md
  - user-guide/orphaned-resources.md
  - user-guide/compare-options.md
  - user-guide/sync-options.md
--- a/util/argo/diff/diff.go
+++ b/util/argo/diff/diff.go
@ -4,6 +4,7 @@ import (
 	"fmt"

 	"github.com/go-logr/logr"
+	log "github.com/sirupsen/logrus"

 	k8smanagedfields "k8s.io/apimachinery/pkg/util/managedfields"

@ -26,7 +27,9 @@ type DiffConfigBuilder struct {
 // NewDiffConfigBuilder create a new DiffConfigBuilder instance.
 func NewDiffConfigBuilder() *DiffConfigBuilder {
 	return &DiffConfigBuilder{
-		diffConfig: &diffConfig{},
+		diffConfig: &diffConfig{
+			ignoreMutationWebhook: true,
+		},
 	}
 }

@ -63,7 +66,6 @@ func (b *DiffConfigBuilder) WithNoCache() *DiffConfigBuilder {
 // WithCache sets the appstatecache.Cache and the appName in the diff config. Those the
 // are two objects necessary to retrieve a cached diff.
 func (b *DiffConfigBuilder) WithCache(s *appstatecache.Cache, appName string) *DiffConfigBuilder {
-	b.diffConfig.noCache = false
 	b.diffConfig.stateCache = s
 	b.diffConfig.appName = appName
 	return b
@ -95,6 +97,21 @@ func (b *DiffConfigBuilder) WithManager(manager string) *DiffConfigBuilder {
 	return b
 }

+func (b *DiffConfigBuilder) WithServerSideDryRunner(ssdr diff.ServerSideDryRunner) *DiffConfigBuilder {
+	b.diffConfig.serverSideDryRunner = ssdr
+	return b
+}
+
+func (b *DiffConfigBuilder) WithServerSideDiff(ssd bool) *DiffConfigBuilder {
+	b.diffConfig.serverSideDiff = ssd
+	return b
+}
+
+func (b *DiffConfigBuilder) WithIgnoreMutationWebhook(m bool) *DiffConfigBuilder {
+	b.diffConfig.ignoreMutationWebhook = m
+	return b
+}
+
 // Build will first validate the current state of the diff config and return the
 // DiffConfig implementation if no errors are found. Will return nil and the error
 // details otherwise.
@ -140,6 +157,10 @@ type DiffConfig interface {
 	// Manager returns the manager that should be used by the diff while
 	// calculating the structured merge diff.
 	Manager() string
+
+	ServerSideDiff() bool
+	ServerSideDryRunner() diff.ServerSideDryRunner
+	IgnoreMutationWebhook() bool
 }

 // diffConfig defines the configurations used while applying diffs.
@ -156,6 +177,9 @@ type diffConfig struct {
 	gvkParser             *k8smanagedfields.GvkParser
 	structuredMergeDiff   bool
 	manager               string
+	serverSideDiff        bool
+	serverSideDryRunner   diff.ServerSideDryRunner
+	ignoreMutationWebhook bool
 }

 func (c *diffConfig) Ignores() []v1alpha1.ResourceIgnoreDifferences {
@ -194,6 +218,15 @@ func (c *diffConfig) StructuredMergeDiff() bool {
 func (c *diffConfig) Manager() string {
 	return c.manager
 }
+func (c *diffConfig) ServerSideDryRunner() diff.ServerSideDryRunner {
+	return c.serverSideDryRunner
+}
+func (c *diffConfig) ServerSideDiff() bool {
+	return c.serverSideDiff
+}
+func (c *diffConfig) IgnoreMutationWebhook() bool {
+	return c.ignoreMutationWebhook
+}

 // Validate will check the current state of this diffConfig and return
 // error if it finds any required configuration missing.
@ -213,6 +246,9 @@ func (c *diffConfig) Validate() error {
 			return fmt.Errorf("%s: StateCache must be set when retrieving from cache", msg)
 		}
 	}
+	if c.serverSideDiff && c.serverSideDryRunner == nil {
+		return fmt.Errorf("%s: serverSideDryRunner must be set when using server side diff", msg)
+	}
 	return nil
 }

@ -254,6 +290,9 @@ func StateDiffs(lives, configs []*unstructured.Unstructured, diffConfig DiffConf
 		diff.WithStructuredMergeDiff(diffConfig.StructuredMergeDiff()),
 		diff.WithGVKParser(diffConfig.GVKParser()),
 		diff.WithManager(diffConfig.Manager()),
+		diff.WithServerSideDiff(diffConfig.ServerSideDiff()),
+		diff.WithServerSideDryRunner(diffConfig.ServerSideDryRunner()),
+		diff.WithIgnoreMutationWebhook(diffConfig.IgnoreMutationWebhook()),
 	}

 	if diffConfig.Logger() != nil {
@ -282,9 +321,8 @@ func diffArrayCached(configArray []*unstructured.Unstructured, liveArray []*unst
 	}

 	diffByKey := map[kube.ResourceKey]*v1alpha1.ResourceDiff{}
-	for i := range cachedDiff {
-		res := cachedDiff[i]
-		diffByKey[kube.NewResourceKey(res.Group, res.Kind, res.Namespace, res.Name)] = cachedDiff[i]
+	for _, res := range cachedDiff {
+		diffByKey[kube.NewResourceKey(res.Group, res.Kind, res.Namespace, res.Name)] = res
 	}

 	diffResultList := diff.DiffResultList{
@ -335,7 +373,12 @@ func (c *diffConfig) DiffFromCache(appName string) (bool, []*v1alpha1.ResourceDi
 		return false, nil
 	}
 	cachedDiff := make([]*v1alpha1.ResourceDiff, 0)
-	if c.stateCache != nil && c.stateCache.GetAppManagedResources(appName, &cachedDiff) == nil {
+	if c.stateCache != nil {
+		err := c.stateCache.GetAppManagedResources(appName, &cachedDiff)
+		if err != nil {
+			log.Errorf("DiffFromCache error: error getting managed resources for app %s: %s", appName, err)
+			return false, nil
+		}
 		return true, cachedDiff
 	}
 	return false, nil