feat: add force promote actions for Numaplane rollouts (#22141)

Signed-off-by: Dillen Padhiar <dillen_padhiar@intuit.com>

docs/operator-manual/argocd-cm.yaml

@@ -428,4 +428,4 @@ data:
   webhook.maxPayloadSizeMB: "50"
 
   # application.sync.impersonation.enabled enables application sync to use a custom service account, via impersonation. This allows decoupling sync from control-plane service account.
-  application.sync.impersonation.enabled: "false"
+  application.sync.impersonation.enabled: "false"

docs/operator-manual/resource_actions_builtin.md

@@ -37,10 +37,16 @@
 - [numaflow.numaproj.io/MonoVertex/unpause](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaflow.numaproj.io/MonoVertex/actions/unpause/action.lua)
 - [numaflow.numaproj.io/Pipeline/pause](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaflow.numaproj.io/Pipeline/actions/pause/action.lua)
 - [numaflow.numaproj.io/Pipeline/unpause](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaflow.numaproj.io/Pipeline/actions/unpause/action.lua)
+- [numaplane.numaproj.io/ISBServiceRollout/disable-force-promote](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/ISBServiceRollout/actions/disable-force-promote/action.lua)
+- [numaplane.numaproj.io/ISBServiceRollout/enable-force-promote](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/ISBServiceRollout/actions/enable-force-promote/action.lua)
+- [numaplane.numaproj.io/MonoVertexRollout/disable-force-promote](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/disable-force-promote/action.lua)
+- [numaplane.numaproj.io/MonoVertexRollout/enable-force-promote](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/enable-force-promote/action.lua)
 - [numaplane.numaproj.io/MonoVertexRollout/pause](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/pause/action.lua)
 - [numaplane.numaproj.io/MonoVertexRollout/unpause](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/unpause/action.lua)
 - [numaplane.numaproj.io/PipelineRollout/allow-data-loss](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/allow-data-loss/action.lua)
+- [numaplane.numaproj.io/PipelineRollout/disable-force-promote](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/disable-force-promote/action.lua)
 - [numaplane.numaproj.io/PipelineRollout/disallow-data-loss](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/disallow-data-loss/action.lua)
+- [numaplane.numaproj.io/PipelineRollout/enable-force-promote](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/enable-force-promote/action.lua)
 - [numaplane.numaproj.io/PipelineRollout/pause](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/pause/action.lua)
 - [numaplane.numaproj.io/PipelineRollout/unpause](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/unpause/action.lua)
 - [source.toolkit.fluxcd.io/Bucket/reconcile](https://github.com/argoproj/argo-cd/blob/master/resource_customizations/source.toolkit.fluxcd.io/Bucket/actions/reconcile/action.lua)

resource_customizations/numaplane.numaproj.io/ISBServiceRollout/actions/action_test.yaml

@@ -0,0 +1,7 @@
+actionTests:
+- action: enable-force-promote
+  inputPath: testdata/ISBServiceRollout/rollout-enable-force-promote.yaml
+  expectedOutputPath: testdata/ISBServiceRollout/rollout-disable-force-promote.yaml
+- action: disable-force-promote
+  inputPath: testdata/ISBServiceRollout/rollout-disable-force-promote.yaml
+  expectedOutputPath: testdata/ISBServiceRollout/rollout-enable-force-promote.yaml

resource_customizations/numaplane.numaproj.io/ISBServiceRollout/actions/disable-force-promote/action.lua

@@ -0,0 +1,9 @@
+if (obj.spec.strategy == nil) then
+    obj.spec.strategy = {}
+    obj.spec.strategy.progressive = {}
+elseif (obj.spec.strategy.progressive == nil) then
+    obj.spec.strategy.progressive = {}
+end
+
+obj.spec.strategy.progressive.forcePromote = false
+return obj 

resource_customizations/numaplane.numaproj.io/ISBServiceRollout/actions/discovery.lua

@@ -0,0 +1,19 @@
+local actions = {}
+actions["enable-force-promote"] = {
+  ["disabled"] = true,
+  ["displayName"] = "Enable Force Promote"
+}
+actions["disable-force-promote"] = {
+  ["disabled"] = true,
+  ["displayName"] = "Disable Force Promote"
+}
+
+-- force-promote
+if (obj.status ~= nil and obj.status.upgradeInProgress == "Progressive" and obj.status.phase == "Pending") then
+  actions["enable-force-promote"]["disabled"] = false
+end
+if (obj.spec ~= nil and obj.spec.strategy ~= nil and obj.spec.strategy.progressive ~= nil and obj.spec.strategy.progressive.forcePromote == true) then
+  actions["disable-force-promote"]["disabled"] = false
+end
+  
+return actions

resource_customizations/numaplane.numaproj.io/ISBServiceRollout/actions/enable-force-promote/action.lua

@@ -0,0 +1,9 @@
+if (obj.spec.strategy == nil) then
+    obj.spec.strategy = {}
+    obj.spec.strategy.progressive = {}
+elseif (obj.spec.strategy.progressive == nil) then
+    obj.spec.strategy.progressive = {}
+end
+
+obj.spec.strategy.progressive.forcePromote = true
+return obj 

resource_customizations/numaplane.numaproj.io/ISBServiceRollout/actions/testdata/ISBServiceRollout/rollout-disable-force-promote.yaml

@@ -0,0 +1,71 @@
+apiVersion: numaplane.numaproj.io/v1alpha1
+kind: ISBServiceRollout
+metadata:
+  creationTimestamp: "2025-02-21T19:43:06Z"
+  finalizers:
+  - numaplane.numaproj.io/numaplane-controller
+  generation: 4
+  name: test-isbservice-rollout
+  namespace: numaplane-system
+  resourceVersion: "6984"
+  uid: 0c926b94-f7a7-4580-a865-f9dd8d54525f
+spec:
+  strategy:
+    progressive:
+      assessmentSchedule: 60,60,10
+      forcePromote: true
+  interStepBufferService:
+    metadata: {}
+    spec:
+      jetstream:
+        containerTemplate:
+          resources:
+            limits:
+              memory: 2Gi
+        persistence:
+          volumeSize: 20Mi
+        version: 2.10.11
+status:
+  conditions:
+  - lastTransitionTime: "2025-02-21T19:43:06Z"
+    message: Successful
+    observedGeneration: 3
+    reason: Successful
+    status: "True"
+    type: ChildResourceDeployed
+  - lastTransitionTime: "2025-02-21T19:55:13Z"
+    message: Successful
+    observedGeneration: 4
+    reason: Successful
+    status: "True"
+    type: ChildResourcesHealthy
+  - lastTransitionTime: "2025-02-21T19:43:06Z"
+    message: no need for pause
+    observedGeneration: 4
+    reason: NoPause
+    status: "False"
+    type: PausingPipelines
+  - lastTransitionTime: "2025-02-21T19:57:27Z"
+    message: New Child Object numaplane-system/test-isbservice-rollout-2 Failed
+    observedGeneration: 4
+    reason: Failed
+    status: "False"
+    type: ProgressiveUpgradeSucceeded
+  lastFailureTime: "2025-02-21T19:53:30Z"
+  message: Progressing
+  nameCount: 3
+  observedGeneration: 4
+  pauseRequestStatus:
+    lastPauseBeginTime: null
+    lastPauseEndTime: null
+    lastPausePhaseChangeTime: null
+  phase: Pending
+  progressiveStatus:
+    promotedISBServiceStatus:
+      name: test-isbservice-rollout-1
+    upgradingISBServiceStatus:
+      assessmentEndTime: "2025-02-21T19:58:27Z"
+      assessmentResult: Failure
+      assessmentStartTime: "2025-02-21T19:57:05Z"
+      name: test-isbservice-rollout-2
+  upgradeInProgress: Progressive

resource_customizations/numaplane.numaproj.io/ISBServiceRollout/actions/testdata/ISBServiceRollout/rollout-enable-force-promote.yaml

@@ -0,0 +1,71 @@
+apiVersion: numaplane.numaproj.io/v1alpha1
+kind: ISBServiceRollout
+metadata:
+  creationTimestamp: "2025-02-21T19:43:06Z"
+  finalizers:
+  - numaplane.numaproj.io/numaplane-controller
+  generation: 4
+  name: test-isbservice-rollout
+  namespace: numaplane-system
+  resourceVersion: "6984"
+  uid: 0c926b94-f7a7-4580-a865-f9dd8d54525f
+spec:
+  strategy:
+    progressive:
+      assessmentSchedule: 60,60,10
+      forcePromote: false
+  interStepBufferService:
+    metadata: {}
+    spec:
+      jetstream:
+        containerTemplate:
+          resources:
+            limits:
+              memory: 2Gi
+        persistence:
+          volumeSize: 20Mi
+        version: 2.10.11
+status:
+  conditions:
+  - lastTransitionTime: "2025-02-21T19:43:06Z"
+    message: Successful
+    observedGeneration: 3
+    reason: Successful
+    status: "True"
+    type: ChildResourceDeployed
+  - lastTransitionTime: "2025-02-21T19:55:13Z"
+    message: Successful
+    observedGeneration: 4
+    reason: Successful
+    status: "True"
+    type: ChildResourcesHealthy
+  - lastTransitionTime: "2025-02-21T19:43:06Z"
+    message: no need for pause
+    observedGeneration: 4
+    reason: NoPause
+    status: "False"
+    type: PausingPipelines
+  - lastTransitionTime: "2025-02-21T19:57:27Z"
+    message: New Child Object numaplane-system/test-isbservice-rollout-2 Failed
+    observedGeneration: 4
+    reason: Failed
+    status: "False"
+    type: ProgressiveUpgradeSucceeded
+  lastFailureTime: "2025-02-21T19:53:30Z"
+  message: Progressing
+  nameCount: 3
+  observedGeneration: 4
+  pauseRequestStatus:
+    lastPauseBeginTime: null
+    lastPauseEndTime: null
+    lastPausePhaseChangeTime: null
+  phase: Pending
+  progressiveStatus:
+    promotedISBServiceStatus:
+      name: test-isbservice-rollout-1
+    upgradingISBServiceStatus:
+      assessmentEndTime: "2025-02-21T19:58:27Z"
+      assessmentResult: Failure
+      assessmentStartTime: "2025-02-21T19:57:05Z"
+      name: test-isbservice-rollout-2
+  upgradeInProgress: Progressive

resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/action_test.yaml

@@ -4,4 +4,10 @@ actionTests:
   expectedOutputPath: testdata/MonoVertexRollout/rollout-paused.yaml
 - action: unpause
   inputPath: testdata/MonoVertexRollout/rollout-paused.yaml
-  expectedOutputPath: testdata/MonoVertexRollout/rollout-running.yaml
+  expectedOutputPath: testdata/MonoVertexRollout/rollout-running.yaml
+- action: enable-force-promote
+  inputPath: testdata/MonoVertexRollout/rollout-enable-force-promote.yaml
+  expectedOutputPath: testdata/MonoVertexRollout/rollout-disable-force-promote.yaml
+- action: disable-force-promote
+  inputPath: testdata/MonoVertexRollout/rollout-disable-force-promote.yaml
+  expectedOutputPath: testdata/MonoVertexRollout/rollout-enable-force-promote.yaml

resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/disable-force-promote/action.lua

@@ -0,0 +1,9 @@
+if (obj.spec.strategy == nil) then
+    obj.spec.strategy = {}
+    obj.spec.strategy.progressive = {}
+elseif (obj.spec.strategy.progressive == nil) then
+    obj.spec.strategy.progressive = {}
+end
+
+obj.spec.strategy.progressive.forcePromote = false
+return obj 

resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/discovery.lua

@@ -1,7 +1,16 @@
 local actions = {}
 actions["pause"] = {["disabled"] = true}
 actions["unpause"] = {["disabled"] = true}
+actions["enable-force-promote"] = {
+  ["disabled"] = true,
+  ["displayName"] = "Enable Force Promote"
+}
+actions["disable-force-promote"] = {
+  ["disabled"] = true,
+  ["displayName"] = "Disable Force Promote"
+}
 
+-- pause/unpause
 local paused = false
 if obj.spec.monoVertex.spec.lifecycle ~= nil and obj.spec.monoVertex.spec.lifecycle.desiredPhase ~= nil and obj.spec.monoVertex.spec.lifecycle.desiredPhase == "Paused" then
   paused = true
@@ -11,4 +20,13 @@ if paused then
 else
   actions["pause"]["disabled"] = false
 end
+
+-- force-promote
+if (obj.status ~= nil and obj.status.upgradeInProgress == "Progressive" and obj.status.phase == "Pending") then
+  actions["enable-force-promote"]["disabled"] = false
+end
+if (obj.spec ~= nil and obj.spec.strategy ~= nil and obj.spec.strategy.progressive ~= nil and obj.spec.strategy.progressive.forcePromote == true) then
+  actions["disable-force-promote"]["disabled"] = false
+end
+
 return actions

resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/enable-force-promote/action.lua

@@ -0,0 +1,9 @@
+if (obj.spec.strategy == nil) then
+    obj.spec.strategy = {}
+    obj.spec.strategy.progressive = {}
+elseif (obj.spec.strategy.progressive == nil) then
+    obj.spec.strategy.progressive = {}
+end
+
+obj.spec.strategy.progressive.forcePromote = true
+return obj 

resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/testdata/MonoVertexRollout/rollout-disable-force-promote.yaml

@@ -0,0 +1,70 @@
+apiVersion: numaplane.numaproj.io/v1alpha1
+kind: MonoVertexRollout
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"numaplane.numaproj.io/v1alpha1","kind":"MonoVertexRollout","metadata":{"annotations":{},"name":"my-monovertex","namespace":"example-namespace"},"spec":{"monoVertex":{"spec":{"sink":{"udsink":{"container":{"image":"quay.io/numaio/numaflow-go/sink-log:stable"}}},"source":{"udsource":{"container":{"image":"quay.io/numaio/numaflow-go/source-simple-source:bad-image"}}}}},"strategy":{"progressive":{"assessmentSchedule":"60,60,10"}}}}
+  creationTimestamp: "2025-03-03T18:51:52Z"
+  finalizers:
+  - numaplane.numaproj.io/numaplane-controller
+  generation: 3
+  name: my-monovertex
+  namespace: example-namespace
+  resourceVersion: "314047"
+  uid: d948cd56-e383-4a18-8100-34b513976614
+spec:
+  monoVertex:
+    metadata: {}
+    spec:
+      sink:
+        udsink:
+          container:
+            image: quay.io/numaio/numaflow-go/sink-log:stable
+      source:
+        udsource:
+          container:
+            image: quay.io/numaio/numaflow-go/source-simple-source:bad-image
+  strategy:
+    progressive:
+      assessmentSchedule: 60,60,10
+      forcePromote: true
+status:
+  conditions:
+  - lastTransitionTime: "2025-03-03T18:51:52Z"
+    message: Successful
+    observedGeneration: 2
+    reason: Successful
+    status: "True"
+    type: ChildResourceDeployed
+  - lastTransitionTime: "2025-03-03T18:54:08Z"
+    message: Successful
+    observedGeneration: 3
+    reason: Successful
+    status: "True"
+    type: ChildResourcesHealthy
+  - lastTransitionTime: "2025-03-03T18:51:52Z"
+    message: MonoVertex unpaused
+    observedGeneration: 3
+    reason: Unpaused
+    status: "False"
+    type: MonoVertexPausingOrPaused
+  - lastTransitionTime: "2025-03-03T18:54:02Z"
+    message: New Child Object example-namespace/my-monovertex-1 Failed
+    observedGeneration: 3
+    reason: Failed
+    status: "False"
+    type: ProgressiveUpgradeSucceeded
+  message: Progressing
+  nameCount: 2
+  observedGeneration: 3
+  phase: Pending
+  progressiveStatus:
+    promotedMonoVertexStatus:
+      name: my-monovertex-0
+      scaleValuesRestoredToOriginal: true
+    upgradingMonoVertexStatus:
+      assessmentEndTime: "2025-03-03T18:54:56Z"
+      assessmentResult: Failure
+      assessmentStartTime: "2025-03-03T18:53:53Z"
+      name: my-monovertex-1
+  upgradeInProgress: Progressive

resource_customizations/numaplane.numaproj.io/MonoVertexRollout/actions/testdata/MonoVertexRollout/rollout-enable-force-promote.yaml

@@ -0,0 +1,70 @@
+apiVersion: numaplane.numaproj.io/v1alpha1
+kind: MonoVertexRollout
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"numaplane.numaproj.io/v1alpha1","kind":"MonoVertexRollout","metadata":{"annotations":{},"name":"my-monovertex","namespace":"example-namespace"},"spec":{"monoVertex":{"spec":{"sink":{"udsink":{"container":{"image":"quay.io/numaio/numaflow-go/sink-log:stable"}}},"source":{"udsource":{"container":{"image":"quay.io/numaio/numaflow-go/source-simple-source:bad-image"}}}}},"strategy":{"progressive":{"assessmentSchedule":"60,60,10"}}}}
+  creationTimestamp: "2025-03-03T18:51:52Z"
+  finalizers:
+  - numaplane.numaproj.io/numaplane-controller
+  generation: 3
+  name: my-monovertex
+  namespace: example-namespace
+  resourceVersion: "314047"
+  uid: d948cd56-e383-4a18-8100-34b513976614
+spec:
+  monoVertex:
+    metadata: {}
+    spec:
+      sink:
+        udsink:
+          container:
+            image: quay.io/numaio/numaflow-go/sink-log:stable
+      source:
+        udsource:
+          container:
+            image: quay.io/numaio/numaflow-go/source-simple-source:bad-image
+  strategy:
+    progressive:
+      assessmentSchedule: 60,60,10
+      forcePromote: false
+status:
+  conditions:
+  - lastTransitionTime: "2025-03-03T18:51:52Z"
+    message: Successful
+    observedGeneration: 2
+    reason: Successful
+    status: "True"
+    type: ChildResourceDeployed
+  - lastTransitionTime: "2025-03-03T18:54:08Z"
+    message: Successful
+    observedGeneration: 3
+    reason: Successful
+    status: "True"
+    type: ChildResourcesHealthy
+  - lastTransitionTime: "2025-03-03T18:51:52Z"
+    message: MonoVertex unpaused
+    observedGeneration: 3
+    reason: Unpaused
+    status: "False"
+    type: MonoVertexPausingOrPaused
+  - lastTransitionTime: "2025-03-03T18:54:02Z"
+    message: New Child Object example-namespace/my-monovertex-1 Failed
+    observedGeneration: 3
+    reason: Failed
+    status: "False"
+    type: ProgressiveUpgradeSucceeded
+  message: Progressing
+  nameCount: 2
+  observedGeneration: 3
+  phase: Pending
+  progressiveStatus:
+    promotedMonoVertexStatus:
+      name: my-monovertex-0
+      scaleValuesRestoredToOriginal: true
+    upgradingMonoVertexStatus:
+      assessmentEndTime: "2025-03-03T18:54:56Z"
+      assessmentResult: Failure
+      assessmentStartTime: "2025-03-03T18:53:53Z"
+      name: my-monovertex-1
+  upgradeInProgress: Progressive

resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/action_test.yaml

@@ -10,4 +10,10 @@ actionTests:
   expectedOutputPath: testdata/PipelineRollout/rollout-allowing-data-loss.yaml
 - action: disallow-data-loss
   inputPath: testdata/PipelineRollout/rollout-allowing-data-loss.yaml
-  expectedOutputPath: testdata/PipelineRollout/rollout-disallowing-data-loss.yaml
+  expectedOutputPath: testdata/PipelineRollout/rollout-disallowing-data-loss.yaml
+- action: enable-force-promote
+  inputPath: testdata/PipelineRollout/rollout-enable-force-promote.yaml
+  expectedOutputPath: testdata/PipelineRollout/rollout-disable-force-promote.yaml
+- action: disable-force-promote
+  inputPath: testdata/PipelineRollout/rollout-disable-force-promote.yaml
+  expectedOutputPath: testdata/PipelineRollout/rollout-enable-force-promote.yaml

resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/disable-force-promote/action.lua

@@ -0,0 +1,9 @@
+if (obj.spec.strategy == nil) then
+    obj.spec.strategy = {}
+    obj.spec.strategy.progressive = {}
+elseif (obj.spec.strategy.progressive == nil) then
+    obj.spec.strategy.progressive = {}
+end
+
+obj.spec.strategy.progressive.forcePromote = false
+return obj 

resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/discovery.lua

@@ -3,6 +3,14 @@ actions["pause"] = {["disabled"] = true}
 actions["unpause"] = {["disabled"] = true}
 actions["allow-data-loss"] = {["disabled"] = true}
 actions["disallow-data-loss"] = {["disabled"] = true}
+actions["enable-force-promote"] = {
+  ["disabled"] = true,
+  ["displayName"] = "Enable Force Promote"
+}
+actions["disable-force-promote"] = {
+  ["disabled"] = true,
+  ["displayName"] = "Disable Force Promote"
+}
 
 -- pause/unpause
 local paused = false
@@ -23,4 +31,12 @@ if obj.metadata.annotations ~= nil and obj.metadata.annotations["numaplane.numap
   actions["disallow-data-loss"]["disabled"] = false
 end
 
+-- force-promote
+if (obj.status ~= nil and obj.status.upgradeInProgress == "Progressive" and obj.status.phase == "Pending") then
+  actions["enable-force-promote"]["disabled"] = false
+end
+if (obj.spec ~= nil and obj.spec.strategy ~= nil and obj.spec.strategy.progressive ~= nil and obj.spec.strategy.progressive.forcePromote == true) then
+  actions["disable-force-promote"]["disabled"] = false
+end
+
 return actions

resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/enable-force-promote/action.lua

@@ -0,0 +1,9 @@
+if (obj.spec.strategy == nil) then
+    obj.spec.strategy = {}
+    obj.spec.strategy.progressive = {}
+elseif (obj.spec.strategy.progressive == nil) then
+    obj.spec.strategy.progressive = {}
+end
+
+obj.spec.strategy.progressive.forcePromote = true
+return obj 

resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/testdata/PipelineRollout/rollout-disable-force-promote.yaml

@@ -0,0 +1,93 @@
+apiVersion: numaplane.numaproj.io/v1alpha1
+kind: PipelineRollout
+metadata:
+  creationTimestamp: "2025-02-21T19:43:43Z"
+  finalizers:
+  - numaplane.numaproj.io/numaplane-controller
+  generation: 3
+  name: another-pipeline-rollout
+  namespace: numaplane-system
+  resourceVersion: "6904"
+  uid: 8365e0f1-18fe-47ed-a26e-cfa2963cced6
+spec:
+  strategy:
+    progressive:
+      assessmentSchedule: 60,60,10
+      forcePromote: true
+  pipeline:
+    metadata: {}
+    spec:
+      edges:
+      - conditions: null
+        from: in
+        to: out
+      interStepBufferServiceName: test-isbservice-rollout
+      lifecycle:
+        desiredPhase: Running
+      vertices:
+      - name: in
+        scale:
+          max: 3
+          min: 3
+          zeroReplicaSleepSeconds: 15
+        source:
+          generator:
+            duration: 1s
+            rpu: 5
+        updateStrategy: {}
+      - name: out
+        scale:
+          max: 3
+          min: 3
+          zeroReplicaSleepSeconds: 15
+        sink:
+          log: {}
+          retryStrategy: {}
+        updateStrategy: {}
+      watermark: {}
+status:
+  conditions:
+  - lastTransitionTime: "2025-02-21T19:43:43Z"
+    message: Successful
+    observedGeneration: 3
+    reason: Successful
+    status: "True"
+    type: ChildResourceDeployed
+  - lastTransitionTime: "2025-02-21T19:55:22Z"
+    message: Successful
+    observedGeneration: 3
+    reason: Successful
+    status: "True"
+    type: ChildResourcesHealthy
+  - lastTransitionTime: "2025-02-21T19:48:13Z"
+    message: Pipeline unpaused
+    observedGeneration: 3
+    reason: Unpaused
+    status: "False"
+    type: PipelinePausingOrPaused
+  - lastTransitionTime: "2025-02-21T19:57:20Z"
+    message: New Child Object numaplane-system/another-pipeline-rollout-2 Failed
+    observedGeneration: 3
+    reason: Failed
+    status: "False"
+    type: ProgressiveUpgradeSucceeded
+  lastFailureTime: null
+  message: Progressing
+  nameCount: 3
+  observedGeneration: 3
+  pauseStatus:
+    lastPauseBeginTime: "2025-02-21T19:47:53Z"
+    lastPauseEndTime: "2025-02-21T19:48:13Z"
+    lastPausePhaseChangeTime: "2025-02-21T19:47:54Z"
+  phase: Pending
+  progressiveStatus:
+    promotedPipelineStatus:
+      name: another-pipeline-rollout-1
+      scaleValuesRestoredToOriginal: true
+    upgradingPipelineStatus:
+      assessmentEndTime: "2025-02-21T19:58:20Z"
+      assessmentResult: Failure
+      assessmentStartTime: "2025-02-21T19:57:19Z"
+      interStepBufferServiceName: test-isbservice-rollout-2
+      name: another-pipeline-rollout-2
+  upgradeInProgress: Progressive

resource_customizations/numaplane.numaproj.io/PipelineRollout/actions/testdata/PipelineRollout/rollout-enable-force-promote.yaml

@@ -0,0 +1,93 @@
+apiVersion: numaplane.numaproj.io/v1alpha1
+kind: PipelineRollout
+metadata:
+  creationTimestamp: "2025-02-21T19:43:43Z"
+  finalizers:
+  - numaplane.numaproj.io/numaplane-controller
+  generation: 3
+  name: another-pipeline-rollout
+  namespace: numaplane-system
+  resourceVersion: "6904"
+  uid: 8365e0f1-18fe-47ed-a26e-cfa2963cced6
+spec:
+  strategy:
+    progressive:
+      assessmentSchedule: 60,60,10
+      forcePromote: false
+  pipeline:
+    metadata: {}
+    spec:
+      edges:
+      - conditions: null
+        from: in
+        to: out
+      interStepBufferServiceName: test-isbservice-rollout
+      lifecycle:
+        desiredPhase: Running
+      vertices:
+      - name: in
+        scale:
+          max: 3
+          min: 3
+          zeroReplicaSleepSeconds: 15
+        source:
+          generator:
+            duration: 1s
+            rpu: 5
+        updateStrategy: {}
+      - name: out
+        scale:
+          max: 3
+          min: 3
+          zeroReplicaSleepSeconds: 15
+        sink:
+          log: {}
+          retryStrategy: {}
+        updateStrategy: {}
+      watermark: {}
+status:
+  conditions:
+  - lastTransitionTime: "2025-02-21T19:43:43Z"
+    message: Successful
+    observedGeneration: 3
+    reason: Successful
+    status: "True"
+    type: ChildResourceDeployed
+  - lastTransitionTime: "2025-02-21T19:55:22Z"
+    message: Successful
+    observedGeneration: 3
+    reason: Successful
+    status: "True"
+    type: ChildResourcesHealthy
+  - lastTransitionTime: "2025-02-21T19:48:13Z"
+    message: Pipeline unpaused
+    observedGeneration: 3
+    reason: Unpaused
+    status: "False"
+    type: PipelinePausingOrPaused
+  - lastTransitionTime: "2025-02-21T19:57:20Z"
+    message: New Child Object numaplane-system/another-pipeline-rollout-2 Failed
+    observedGeneration: 3
+    reason: Failed
+    status: "False"
+    type: ProgressiveUpgradeSucceeded
+  lastFailureTime: null
+  message: Progressing
+  nameCount: 3
+  observedGeneration: 3
+  pauseStatus:
+    lastPauseBeginTime: "2025-02-21T19:47:53Z"
+    lastPauseEndTime: "2025-02-21T19:48:13Z"
+    lastPausePhaseChangeTime: "2025-02-21T19:47:54Z"
+  phase: Pending
+  progressiveStatus:
+    promotedPipelineStatus:
+      name: another-pipeline-rollout-1
+      scaleValuesRestoredToOriginal: true
+    upgradingPipelineStatus:
+      assessmentEndTime: "2025-02-21T19:58:20Z"
+      assessmentResult: Failure
+      assessmentStartTime: "2025-02-21T19:57:19Z"
+      interStepBufferServiceName: test-isbservice-rollout-2
+      name: another-pipeline-rollout-2
+  upgradeInProgress: Progressive