From 16ca0526bc92d797c351176a0607e57971d50f8b Mon Sep 17 00:00:00 2001 From: Rabi Mishra Date: Sat, 3 Oct 2026 09:43:33 +0530 Subject: [PATCH] Collect dataplane AnsibleEE jobs only when deployment has finished An OpenStackDataPlaneDeployment re-derives every service outcome from the AnsibleEE Job of that service: the deployment conditions are rebuilt on each reconcile pass, and lib-common does not recreate a Job whose hash is unchanged. With preserveJobs: false the Jobs are deleted ten minutes after they finish, so a deployment still running once that happens waits forever for an execution it can neither find nor re-run, and reports no error. Create the Jobs with cleanup deferred while the deployment is in flight, and apply the cleanup deadline only once the deployment reaches a finished state: deployed, or failed with the backoff limit exceeded. Signed-off-by: Rabi Mishra --- ...ack.org_openstackdataplanedeployments.yaml | 6 +- .../openstackdataplanedeployment_types.go | 42 ++++- bindata/crds/crds.yaml | 6 +- ...ack.org_openstackdataplanedeployments.yaml | 6 +- docs/assemblies/dataplane_resources.adoc | 2 +- ...openstackdataplanedeployment_controller.go | 113 ++++++++++++ internal/dataplane/util/ansible_execution.go | 17 +- ...tackdataplanedeployment_controller_test.go | 173 ++++++++++++++++++ 8 files changed, 356 insertions(+), 9 deletions(-) diff --git a/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml b/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml index 126a0a476..29e89d469 100644 --- a/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml +++ b/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml @@ -105,7 +105,11 @@ spec: preserveJobs: default: true description: |- - PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs - do not delete jobs after they finished e.g. to check logs. + When false the jobs are deleted after a delay once the deployment is + finished - deployed, or failed with the backoff limit exceeded - and not + when an individual job finishes. While a deployment is in flight its + jobs are what tells the operator which services already ran. PreserveJobs default: true enum: - true diff --git a/api/dataplane/v1beta1/openstackdataplanedeployment_types.go b/api/dataplane/v1beta1/openstackdataplanedeployment_types.go index 062e23f1e..affb0136b 100644 --- a/api/dataplane/v1beta1/openstackdataplanedeployment_types.go +++ b/api/dataplane/v1beta1/openstackdataplanedeployment_types.go @@ -39,7 +39,11 @@ type OpenStackDataPlaneDeploymentSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"} BackoffLimit *int32 `json:"backoffLimit,omitempty"` - // PreserveJobs - do not delete jobs after they finished e.g. to check logs + // PreserveJobs - do not delete jobs after they finished e.g. to check logs. + // When false the jobs are deleted after a delay once the deployment is + // finished - deployed, or failed with the backoff limit exceeded - and not + // when an individual job finishes. While a deployment is in flight its + // jobs are what tells the operator which services already ran. // PreserveJobs default: true // +kubebuilder:validation:Enum:=true;false // +kubebuilder:default:=true @@ -189,6 +193,42 @@ func (instance OpenStackDataPlaneDeployment) IsReady() bool { return instance.Status.Conditions.IsTrue(condition.ReadyCondition) } +// IsFinished reports whether the deployment reached a state it will not advance +// from. The failure verdict is only readable before InitConditions resets the +// conditions. +func (instance *OpenStackDataPlaneDeployment) IsFinished() bool { + return instance.Status.Deployed || instance.isTerminalFailure() +} + +// FinishedAt reports when the deployment reached that state, or nil while it can +// still progress. LastTransitionTime only moves when the condition state +// changes, so the value is stable across reconciles. +func (instance *OpenStackDataPlaneDeployment) FinishedAt() *metav1.Time { + if instance.Status.Conditions == nil { + return nil + } + deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition) + if deploymentCondition == nil || deploymentCondition.LastTransitionTime.IsZero() { + return nil + } + if !instance.Status.Deployed && !instance.isTerminalFailure() { + return nil + } + + finishedAt := deploymentCondition.LastTransitionTime + return &finishedAt +} + +func (instance *OpenStackDataPlaneDeployment) isTerminalFailure() bool { + if instance.Status.Conditions == nil { + return false + } + deploymentCondition := instance.Status.Conditions.Get(condition.DeploymentReadyCondition) + return deploymentCondition != nil && + deploymentCondition.Severity == condition.SeverityError && + deploymentCondition.Reason == condition.JobReasonBackoffLimitExceeded +} + // InitConditions - Initializes Status Conditons func (instance *OpenStackDataPlaneDeployment) InitConditions() { instance.Status.Conditions = condition.Conditions{} diff --git a/bindata/crds/crds.yaml b/bindata/crds/crds.yaml index 5641d6da5..c1e242c61 100644 --- a/bindata/crds/crds.yaml +++ b/bindata/crds/crds.yaml @@ -20732,7 +20732,11 @@ spec: preserveJobs: default: true description: |- - PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs - do not delete jobs after they finished e.g. to check logs. + When false the jobs are deleted after a delay once the deployment is + finished - deployed, or failed with the backoff limit exceeded - and not + when an individual job finishes. While a deployment is in flight its + jobs are what tells the operator which services already ran. PreserveJobs default: true enum: - true diff --git a/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml b/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml index 126a0a476..29e89d469 100644 --- a/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml +++ b/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml @@ -105,7 +105,11 @@ spec: preserveJobs: default: true description: |- - PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs - do not delete jobs after they finished e.g. to check logs. + When false the jobs are deleted after a delay once the deployment is + finished - deployed, or failed with the backoff limit exceeded - and not + when an individual job finishes. While a deployment is in flight its + jobs are what tells the operator which services already ran. PreserveJobs default: true enum: - true diff --git a/docs/assemblies/dataplane_resources.adoc b/docs/assemblies/dataplane_resources.adoc index 81e41c6b7..c1820ebee 100644 --- a/docs/assemblies/dataplane_resources.adoc +++ b/docs/assemblies/dataplane_resources.adoc @@ -93,7 +93,7 @@ OpenStackDataPlaneDeploymentSpec defines the desired state of OpenStackDataPlane | false | preserveJobs -| PreserveJobs - do not delete jobs after they finished e.g. to check logs PreserveJobs default: true +| PreserveJobs - do not delete jobs after they finished e.g. to check logs. When false the jobs are deleted after a delay once the deployment is finished - deployed, or failed with the backoff limit exceeded - and not when an individual job finishes. While a deployment is in flight its jobs are what tells the operator which services already ran. PreserveJobs default: true | bool | false diff --git a/internal/controller/dataplane/openstackdataplanedeployment_controller.go b/internal/controller/dataplane/openstackdataplanedeployment_controller.go index 771f12350..72833a482 100644 --- a/internal/controller/dataplane/openstackdataplanedeployment_controller.go +++ b/internal/controller/dataplane/openstackdataplanedeployment_controller.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "math" "slices" "time" @@ -29,6 +30,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -107,12 +109,30 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, Log, ) + // Whether Job cleanup may be scheduled: a verdict freshly computed by this + // reconcile is only trustworthy once the status patch below persists it. + finishedOnEntry := instance.IsFinished() + statusPersisted := false + + // Registered ahead of the status patch defer so it runs after the verdict persists. + defer func() { + if err := r.reconcileJobTTL(ctx, helper, instance, finishedOnEntry || statusPersisted); err != nil { + Log.Error(err, "Failed to reconcile AnsibleEE job cleanup", "deployment", instance.Name) + if _err == nil { + _err = err + } + } + }() + // If the deploy is already done, return immediately. if instance.Status.Deployed { Log.Info("Already deployed", "instance.Status.Deployed", instance.Status.Deployed) return ctrl.Result{}, nil } + // If the deployment is in a terminal state, do not reconcile even if + // nodesets or other watched resources change. The deployment should not + // be retried. if terminalFailure(instance) { Log.Info("Deployment is terminal, skipping reconciliation", "deployment", instance.Name) @@ -162,6 +182,7 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, _err = err return } + statusPersisted = true }() // Ensure NodeSets @@ -547,6 +568,98 @@ func (r *OpenStackDataPlaneDeploymentReconciler) deployGlobalServices( return nil } +// Mirrors the delay lib-common applies to finished Jobs. +const jobCleanupDelaySeconds int32 = 10 * 60 + +// reconcileJobTTL defers Job cleanup until the deployment is finished: while it +// runs the Jobs are the only record of which services already ran. scheduleCleanup +// is false while the terminal verdict exists only in memory. +func (r *OpenStackDataPlaneDeploymentReconciler) reconcileJobTTL( + ctx context.Context, + helper *helper.Helper, + instance *dataplanev1.OpenStackDataPlaneDeployment, + scheduleCleanup bool, +) error { + if instance.Spec.PreserveJobs { + return nil + } + + jobList := &batchv1.JobList{} + err := helper.GetClient().List( + ctx, + jobList, + client.InNamespace(instance.Namespace), + client.MatchingLabels{dataplaneutil.AnsibleEEDeploymentLabel: instance.Name}, + ) + if err != nil { + return err + } + + finished := scheduleCleanup && instance.IsFinished() + finishedAt := instance.FinishedAt() + + for i := range jobList.Items { + job := &jobList.Items[i] + if finished { + // Ageing starts at the Job's own completion time, so one schedule per Job + // is enough and the value must not be re-derived afterwards. + if job.Spec.TTLSecondsAfterFinished != nil { + continue + } + patch := client.MergeFrom(job.DeepCopy()) + ttl := jobCleanupDelay(job, finishedAt) + job.Spec.TTLSecondsAfterFinished = &ttl + + err := helper.GetClient().Patch(ctx, job, patch) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + continue + } + + // An older operator may have left a deadline on a running deployment. + if job.Spec.TTLSecondsAfterFinished == nil { + continue + } + patch := client.MergeFrom(job.DeepCopy()) + job.Spec.TTLSecondsAfterFinished = nil + + err := helper.GetClient().Patch(ctx, job, patch) + if err != nil && !k8s_errors.IsNotFound(err) { + return err + } + } + return nil +} + +// jobCleanupDelay returns the TTLSecondsAfterFinished that makes the Job expire +// jobCleanupDelaySeconds after the deployment finished. Ageing starts at the +// Job's own completion time, so a Job that finished while the deployment still +// ran gets that elapsed time added back - a flat delay would delete it at once. +func jobCleanupDelay(job *batchv1.Job, finishedAt *metav1.Time) int32 { + finishTime := jobFinishTime(job) + if finishedAt == nil || finishTime == nil || !finishedAt.After(finishTime.Time) { + return jobCleanupDelaySeconds + } + + delay := int64(jobCleanupDelaySeconds) + int64(math.Floor(finishedAt.Time.Sub(finishTime.Time).Seconds())) + return int32(min(delay, math.MaxInt32)) +} + +// jobFinishTime mirrors the TTL-after-finished controller, which measures from +// the Complete/Failed transition rather than CompletionTime. +func jobFinishTime(job *batchv1.Job) *metav1.Time { + for _, jobCondition := range job.Status.Conditions { + isFinished := (jobCondition.Type == batchv1.JobComplete || jobCondition.Type == batchv1.JobFailed) && + jobCondition.Status == corev1.ConditionTrue + if isFinished && !jobCondition.LastTransitionTime.IsZero() { + finishTime := jobCondition.LastTransitionTime + return &finishTime + } + } + return job.Status.CompletionTime +} + // GetService retrieves a service for the OpenStackDataPlaneDeployment func (r *OpenStackDataPlaneDeploymentReconciler) GetService( ctx context.Context, diff --git a/internal/dataplane/util/ansible_execution.go b/internal/dataplane/util/ansible_execution.go index 26ca9bd3a..a1a786785 100644 --- a/internal/dataplane/util/ansible_execution.go +++ b/internal/dataplane/util/ansible_execution.go @@ -247,6 +247,13 @@ func ParseAnsibleExecutionSummaryFromPod( return nil, nil } +// Labels the AnsibleEE Jobs carry, to build the Job spec and find them again. +const ( + AnsibleEEServiceLabel = "openstackdataplaneservice" + AnsibleEEDeploymentLabel = "openstackdataplanedeployment" + AnsibleEENodeSetLabel = "openstackdataplanenodeset" +) + // GetAnsibleExecutionNameAndLabels Name and Labels of AnsibleEE func GetAnsibleExecutionNameAndLabels(service *dataplanev1.OpenStackDataPlaneService, deploymentName string, @@ -267,7 +274,7 @@ func GetAnsibleExecutionNameAndLabels(service *dataplanev1.OpenStackDataPlaneSer } labels := GetGlobalAnsibleExecutionLabels(service.Name, deploymentName) - labels["openstackdataplanenodeset"] = nodeSetName + labels[AnsibleEENodeSetLabel] = nodeSetName return executionName, labels } @@ -276,8 +283,8 @@ func GetAnsibleExecutionNameAndLabels(service *dataplanev1.OpenStackDataPlaneSer // selector must not carry the nodeset label. func GetGlobalAnsibleExecutionLabels(serviceName string, deploymentName string) map[string]string { return map[string]string{ - "openstackdataplaneservice": serviceName, - "openstackdataplanedeployment": deploymentName, + AnsibleEEServiceLabel: serviceName, + AnsibleEEDeploymentLabel: deploymentName, } } @@ -306,7 +313,9 @@ func (a *EEJob) BuildAeeJobSpec( } a.BackoffLimit = deployment.Spec.BackoffLimit - a.PreserveJobs = deployment.Spec.PreserveJobs + // Cleanup must not start while the deployment still runs: lib-common never + // recreates a Job whose hash is unchanged. + a.PreserveJobs = deployment.Spec.PreserveJobs || !deployment.IsFinished() a.FormatAEECmdLineArguments(aeeSpec) a.FormatAEEExtraVars(aeeSpec, service, deployment, nodeSet) a.DetermineAeeImage(aeeSpec) diff --git a/test/functional/dataplane/openstackdataplanedeployment_controller_test.go b/test/functional/dataplane/openstackdataplanedeployment_controller_test.go index ab0862670..b7d22b735 100644 --- a/test/functional/dataplane/openstackdataplanedeployment_controller_test.go +++ b/test/functional/dataplane/openstackdataplanedeployment_controller_test.go @@ -3,6 +3,7 @@ package functional import ( "fmt" "os" + "time" . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports . "github.com/onsi/gomega" //revive:disable:dot-imports @@ -2156,6 +2157,178 @@ var _ = Describe("Dataplane Deployment Test", func() { }) }) + When("A dataplaneDeployment has preserveJobs false", func() { + var deploymentJobs func() []batchv1.Job + + BeforeEach(func() { + CreateSSHSecret(dataplaneSSHSecretName) + CreateCABundleSecret(caBundleSecretName) + CreateDataplaneService(dataplaneServiceName, false) + DeferCleanup(th.DeleteService, dataplaneServiceName) + DeferCleanup(th.DeleteInstance, CreateNetConfig(dataplaneNetConfigName, DefaultNetConfigSpec())) + DeferCleanup(th.DeleteInstance, CreateDNSMasq(dnsMasqName, DefaultDNSMasqSpec())) + SimulateDNSMasqComplete(dnsMasqName) + DeferCleanup(th.DeleteInstance, CreateDataplaneNodeSet(dataplaneNodeSetName, DefaultDataPlaneNodeSetSpec(dataplaneNodeSetName.Name))) + SimulateIPSetComplete(dataplaneNodeName) + SimulateDNSDataComplete(dataplaneNodeSetName) + DeferCleanup(th.DeleteInstance, CreateDataplaneDeployment(dataplaneDeploymentName, + map[string]interface{}{ + "nodeSets": []string{dataplaneNodeSetName.Name}, + "preserveJobs": false, + "servicesOverride": []string{"foo-service"}, + })) + + baremetal := baremetalv1.OpenStackBaremetalSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataplaneNodeSetName.Name, + Namespace: namespace, + }, + } + Eventually(func(g Gomega) { + g.Expect(th.K8sClient.Get(th.Ctx, dataplaneNodeSetName, &baremetal)).To(Succeed()) + baremetal.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + g.Expect(th.K8sClient.Status().Update(th.Ctx, &baremetal)).To(Succeed()) + }, th.Timeout, th.Interval).Should(Succeed()) + + deploymentJobs = func() []batchv1.Job { + jobList := &batchv1.JobList{} + Expect(k8sClient.List(ctx, jobList, + client.InNamespace(namespace), + client.MatchingLabels{"openstackdataplanedeployment": dataplaneDeploymentName.Name}, + )).To(Succeed()) + return jobList.Items + } + }) + + It("starts the cleanup clock when the deployment deployed", func() { + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + g.Expect(jobs[0].Spec.TTLSecondsAfterFinished).To(BeNil()) + }, th.Timeout, th.Interval).Should(Succeed()) + + // a deadline left behind by an older operator is dropped while the + // deployment still runs + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + ansibleEE := GetAnsibleee(types.NamespacedName{Name: jobs[0].Name, Namespace: namespace}) + ansibleEE.Spec.TTLSecondsAfterFinished = ptr.To(int32(600)) + g.Expect(th.K8sClient.Update(th.Ctx, ansibleEE)).To(Succeed()) + }, th.Timeout, th.Interval).Should(Succeed()) + + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + g.Expect(jobs[0].Spec.TTLSecondsAfterFinished).To(BeNil()) + }, th.Timeout, th.Interval).Should(Succeed()) + + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + ansibleEE := GetAnsibleee(types.NamespacedName{Name: jobs[0].Name, Namespace: namespace}) + ansibleEE.Status.Succeeded = 1 + g.Expect(th.K8sClient.Status().Update(th.Ctx, ansibleEE)).To(Succeed()) + }, th.Timeout, th.Interval).Should(Succeed()) + + th.ExpectCondition( + dataplaneDeploymentName, + ConditionGetterFunc(DataplaneDeploymentConditionGetter), + condition.DeploymentReadyCondition, + corev1.ConditionTrue, + ) + + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + g.Expect(jobs[0].Spec.TTLSecondsAfterFinished).ToNot(BeNil()) + g.Expect(*jobs[0].Spec.TTLSecondsAfterFinished).To(Equal(int32(600))) + }, th.Timeout, th.Interval).Should(Succeed()) + }) + + It("starts the cleanup clock when the deployment failed", func() { + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + g.Expect(jobs[0].Spec.TTLSecondsAfterFinished).To(BeNil()) + + ansibleEE := GetAnsibleee(types.NamespacedName{Name: jobs[0].Name, Namespace: namespace}) + startTime := metav1.Now() + ansibleEE.Status.StartTime = &startTime + ansibleEE.Status.Failed = *ansibleEE.Spec.BackoffLimit + 1 + ansibleEE.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobFailureTarget, + Status: corev1.ConditionTrue, + Reason: string(condition.JobReasonBackoffLimitExceeded), + Message: "Job has reached the specified backoff limit", + }, { + Type: batchv1.JobFailed, + Status: corev1.ConditionTrue, + Reason: string(condition.JobReasonBackoffLimitExceeded), + Message: "Simulated backoff limit exceeded for testing", + }} + g.Expect(th.K8sClient.Status().Update(th.Ctx, ansibleEE)).To(Succeed()) + }, th.Timeout, th.Interval).Should(Succeed()) + + Eventually(func(g Gomega) { + deploymentCondition := GetDataplaneDeployment(dataplaneDeploymentName). + Status.Conditions.Get(condition.DeploymentReadyCondition) + g.Expect(deploymentCondition).ToNot(BeNil()) + g.Expect(deploymentCondition.Status).To(Equal(corev1.ConditionFalse)) + g.Expect(string(deploymentCondition.Severity)).To(Equal(string(condition.SeverityError))) + g.Expect(string(deploymentCondition.Reason)).To(Equal(string(condition.JobReasonBackoffLimitExceeded))) + }, th.Timeout, th.Interval).Should(Succeed()) + + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + g.Expect(jobs[0].Spec.TTLSecondsAfterFinished).ToNot(BeNil()) + g.Expect(*jobs[0].Spec.TTLSecondsAfterFinished).To(Equal(int32(600))) + }, th.Timeout, th.Interval).Should(Succeed()) + }) + + It("gives a job that finished while the deployment still ran the time it waited", func() { + finishedThirtyMinutesAgo := metav1.NewTime(time.Now().Add(-30 * time.Minute)) + startedBeforeThat := metav1.NewTime(finishedThirtyMinutesAgo.Add(-10 * time.Minute)) + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + + ansibleEE := GetAnsibleee(types.NamespacedName{Name: jobs[0].Name, Namespace: namespace}) + ansibleEE.Status.Succeeded = 1 + ansibleEE.Status.StartTime = &startedBeforeThat + ansibleEE.Status.CompletionTime = &finishedThirtyMinutesAgo + ansibleEE.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobSuccessCriteriaMet, + Status: corev1.ConditionTrue, + LastTransitionTime: finishedThirtyMinutesAgo, + }, { + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + LastTransitionTime: finishedThirtyMinutesAgo, + }} + g.Expect(th.K8sClient.Status().Update(th.Ctx, ansibleEE)).To(Succeed()) + }, th.Timeout, th.Interval).Should(Succeed()) + + th.ExpectCondition( + dataplaneDeploymentName, + ConditionGetterFunc(DataplaneDeploymentConditionGetter), + condition.DeploymentReadyCondition, + corev1.ConditionTrue, + ) + + // Ageing starts at the Job's own completion time, so the TTL has to carry + // the time the deployment spent waiting for its other services. + Eventually(func(g Gomega) { + jobs := deploymentJobs() + g.Expect(jobs).To(HaveLen(1)) + g.Expect(jobs[0].Spec.TTLSecondsAfterFinished).ToNot(BeNil()) + g.Expect(*jobs[0].Spec.TTLSecondsAfterFinished).To(BeNumerically(">=", int32(30*60+600))) + g.Expect(*jobs[0].Spec.TTLSecondsAfterFinished).To(BeNumerically("<", int32(30*60+600+10*60))) + }, th.Timeout, th.Interval).Should(Succeed()) + }) + }) + When("A dataplaneDeployment is created with useParallelExecution true and services with dependsOn", func() { var svcAName types.NamespacedName var svcBName types.NamespacedName