Skip to content

Commit f52b39c

Browse files
metalcyclingmetalcycling
authored
Support for exponential requeuing time and user supplied requeuing time (#263)
* Updated CRD to expose new field 'requeuingTimeMinutes' allowing users to specify how long to wait for the first requeuing period after dispatching. * Modified requeuing strategy to use a requeuing time that grows exponentially. This time is set by default to 5 minutes but can be modified from the AppWrapper schedulingSpec field. * Added a test to evaluate the exponentially growing requeuing mechanism with user supplied initial time. This tests uses init containers to trick the requeuing mechanism into thinking the PODs have failed and are not going to complete. * Fixed naming issue in requeuing test * Updated requeuing test to separate the test for requeuing * Attempt to fix test issue by specifying requeuing time. 5 minutes of wait time seems to be too long. * Updated CRD to include different requeuing control options * New logic to requeu AppWrapper based on the latest condition based on time (not location in the list of conditions). Users can now control different ways to requeue the job and to stop it from requeuing. * Updated testing for new requeuing mechanism * Increased time for requeuing to completion test * Updated the CONTROLLER_VERSION
1 parent 65afa9e commit f52b39c

File tree

10 files changed

+426
-41
lines changed

10 files changed

+426
-41
lines changed

‎CONTROLLER_VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.29.50
1+
1.29.51

‎config/crd/bases/mcad.ibm.com_appwrappers.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,28 @@ spec:
257257
properties:
258258
minAvailable:
259259
type: integer
260+
requeuing:
261+
description: Specification of the requeuing strategy based on
262+
waiting time
263+
properties:
264+
initialTimeInSeconds:
265+
type: integer
266+
timeInSeconds:
267+
type: integer
268+
default: 300
269+
maxTimeInSeconds:
270+
type: integer
271+
default: 0
272+
growthType:
273+
type: string
274+
default: "exponential"
275+
numRequeuings:
276+
type: integer
277+
default: 0
278+
maxNumRequeuings:
279+
type: integer
280+
default: 0
281+
type: object
260282
nodeSelector:
261283
additionalProperties:
262284
type: string

‎config/crd/bases/mcad.ibm.com_queuejobs.yaml

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ spec:
3434
type: object
3535
spec:
3636
description: Specification of the desired behavior of a cron job, including
37-
the minAvailable
37+
the minAvailable and the requeuing strategy
3838
properties:
3939
schedulerName:
4040
type: string
@@ -43,6 +43,28 @@ spec:
4343
properties:
4444
minAvailable:
4545
type: integer
46+
requeuing:
47+
description: Specification of the requeuing strategy based on
48+
waiting time
49+
properties:
50+
initialTimeInSeconds:
51+
type: integer
52+
timeInSeconds:
53+
type: integer
54+
default: 300
55+
maxTimeInSeconds:
56+
type: integer
57+
default: 0
58+
growthType:
59+
type: string
60+
default: "exponential"
61+
numRequeuings:
62+
type: integer
63+
default: 0
64+
maxNumRequeuings:
65+
type: integer
66+
default: 0
67+
type: object
4668
nodeSelector:
4769
additionalProperties:
4870
type: string

‎config/crd/bases/mcad.ibm.com_schedulingspecs.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,28 @@ spec:
3636
properties:
3737
minAvailable:
3838
type: integer
39+
requeuing:
40+
description: Specification of the requeuing strategy based on
41+
waiting time
42+
properties:
43+
initialTimeInSeconds:
44+
type: integer
45+
timeInSeconds:
46+
type: integer
47+
default: 300
48+
maxTimeInSeconds:
49+
type: integer
50+
default: 0
51+
growthType:
52+
type: string
53+
default: "exponential"
54+
numRequeuings:
55+
type: integer
56+
default: 0
57+
maxNumRequeuings:
58+
type: integer
59+
default: 0
60+
type: object
3961
nodeSelector:
4062
additionalProperties:
4163
type: string

‎deployment/mcad-controller/templates/deployment.yaml

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,28 @@ spec:
136136
properties:
137137
minAvailable:
138138
type: integer
139+
requeuing:
140+
description: Specification of the requeuing strategy based on
141+
waiting time
142+
properties:
143+
initialTimeInSeconds:
144+
type: integer
145+
timeInSeconds:
146+
type: integer
147+
default: 300
148+
maxTimeInSeconds:
149+
type: integer
150+
default: 0
151+
growthType:
152+
type: string
153+
default: "exponential"
154+
numRequeuings:
155+
type: integer
156+
default: 0
157+
maxNumRequeuings:
158+
type: integer
159+
default: 0
160+
type: object
139161
nodeSelector:
140162
additionalProperties:
141163
type: string
@@ -181,7 +203,7 @@ spec:
181203
type: object
182204
spec:
183205
description: Specification of the desired behavior of a cron job, including
184-
the minAvailable
206+
the minAvailable and the requeuing strategy
185207
properties:
186208
schedulerName:
187209
type: string
@@ -190,6 +212,28 @@ spec:
190212
properties:
191213
minAvailable:
192214
type: integer
215+
requeuing:
216+
description: Specification of the requeuing strategy based on
217+
waiting time
218+
properties:
219+
initialTimeInSeconds:
220+
type: integer
221+
timeInSeconds:
222+
type: integer
223+
default: 300
224+
maxTimeInSeconds:
225+
type: integer
226+
default: 0
227+
growthType:
228+
type: string
229+
default: "exponential"
230+
numRequeuings:
231+
type: integer
232+
default: 0
233+
maxNumRequeuings:
234+
type: integer
235+
default: 0
236+
type: object
193237
nodeSelector:
194238
additionalProperties:
195239
type: string
@@ -7225,6 +7269,28 @@ spec:
72257269
properties:
72267270
minAvailable:
72277271
type: integer
7272+
requeuing:
7273+
description: Specification of the requeuing strategy based on
7274+
waiting time
7275+
properties:
7276+
initialTimeInSeconds:
7277+
type: integer
7278+
timeInSeconds:
7279+
type: integer
7280+
default: 300
7281+
maxTimeInSeconds:
7282+
type: integer
7283+
default: 0
7284+
growthType:
7285+
type: string
7286+
default: "exponential"
7287+
numRequeuings:
7288+
type: integer
7289+
default: 0
7290+
maxNumRequeuings:
7291+
type: integer
7292+
default: 0
7293+
type: object
72287294
nodeSelector:
72297295
additionalProperties:
72307296
type: string

‎deployment/mcad-operator/helm-charts/mcad-controller/templates/deployment.yaml

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,28 @@ spec:
118118
properties:
119119
minAvailable:
120120
type: integer
121+
requeuing:
122+
description: Specification of the requeuing strategy based on
123+
waiting time
124+
properties:
125+
initialTimeInSeconds:
126+
type: integer
127+
timeInSeconds:
128+
type: integer
129+
default: 300
130+
maxTimeInSeconds:
131+
type: integer
132+
default: 0
133+
growthType:
134+
type: string
135+
default: "exponential"
136+
numRequeuings:
137+
type: integer
138+
default: 0
139+
maxNumRequeuings:
140+
type: integer
141+
default: 0
142+
type: object
121143
nodeSelector:
122144
additionalProperties:
123145
type: string
@@ -163,7 +185,7 @@ spec:
163185
type: object
164186
spec:
165187
description: Specification of the desired behavior of a cron job, including
166-
the minAvailable
188+
the minAvailable and the requeuing strategy
167189
properties:
168190
schedulerName:
169191
type: string
@@ -172,6 +194,28 @@ spec:
172194
properties:
173195
minAvailable:
174196
type: integer
197+
requeuing:
198+
description: Specification of the requeuing strategy based on
199+
waiting time
200+
properties:
201+
initialTimeInSeconds:
202+
type: integer
203+
timeInSeconds:
204+
type: integer
205+
default: 300
206+
maxTimeInSeconds:
207+
type: integer
208+
default: 0
209+
growthType:
210+
type: string
211+
default: "exponential"
212+
numRequeuings:
213+
type: integer
214+
default: 0
215+
maxNumRequeuings:
216+
type: integer
217+
default: 0
218+
type: object
175219
nodeSelector:
176220
additionalProperties:
177221
type: string
@@ -7198,6 +7242,28 @@ spec:
71987242
properties:
71997243
minAvailable:
72007244
type: integer
7245+
requeuing:
7246+
description: Specification of the requeuing strategy based on
7247+
waiting time
7248+
properties:
7249+
initialTimeInSeconds:
7250+
type: integer
7251+
timeInSeconds:
7252+
type: integer
7253+
default: 300
7254+
maxTimeInSeconds:
7255+
type: integer
7256+
default: 0
7257+
growthType:
7258+
type: string
7259+
default: "exponential"
7260+
numRequeuings:
7261+
type: int
7262+
default: 0
7263+
maxNumRequeuings:
7264+
type: int
7265+
default: 0
7266+
type: object
72017267
nodeSelector:
72027268
additionalProperties:
72037269
type: string

‎pkg/apis/controller/v1beta1/schedulingspec.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,16 @@ type SchedulingSpec struct {
4848
type SchedulingSpecTemplate struct {
4949
NodeSelector map[string]string `json:"nodeSelector,omitempty" protobuf:"bytes,1,rep,name=nodeSelector"`
5050
MinAvailable int `json:"minAvailable,omitempty" protobuf:"bytes,2,rep,name=minAvailable"`
51+
Requeuing RequeuingTemplate `json:"requeuing,omitempty" protobuf:"bytes,1,rep,name=requeuing"`
52+
}
53+
54+
type RequeuingTemplate struct {
55+
InitialTimeInSeconds int `json:"initialTimeInSeconds,omitempty" protobuf:"bytes,1,rep,name=initialTimeInSeconds"`
56+
TimeInSeconds int `json:"timeInSeconds,omitempty" protobuf:"bytes,2,rep,name=timeInSeconds"`
57+
MaxTimeInSeconds int `json:"maxTimeInSeconds,omitempty" protobuf:"bytes,3,rep,name=maxTimeInSeconds"`
58+
GrowthType string `json:"growthType,omitempty" protobuf:"bytes,4,rep,name=growthType"`
59+
NumRequeuings int `json:"numRequeuings,omitempty" protobuf:"bytes,5,rep,name=numRequeuings"`
60+
MaxNumRequeuings int `json:"maxNumRequeuings,omitempty" protobuf:"bytes,6,rep,name=maxNumRequeuings"`
5161
}
5262

5363
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

0 commit comments

Comments
 (0)