Skip to content

Commit 8c8ee6b

Browse files
atantawiatantawi
committed
add support for try and undo allocate
1 parent a4d16e1 commit 8c8ee6b

File tree

8 files changed

+659
-32
lines changed

8 files changed

+659
-32
lines changed

‎pkg/quotaplugins/quota-forest/quota-manager/README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,11 @@ A summary of the API interface to the Quota Manager follows.
188188
- Forest Updates
189189
- refresh (effect) updates from caches
190190
- `UpdateForest(forestName)`
191+
- Undo consumer allocation: Two calls are provided to try to allocate a consumer, and if unaccepted, to undo the effect of the allocation trial. If the trial is accepted, no further action is needed. Otherwise, the undo has to be called right after the try allocation, without making any calls to change the trees or allocate/deallocate consumers. These operations are intended only during Normal mode.
192+
- `TryAllocate(treeName, consumerID)`
193+
- `UndoAllocate(treeName, consumerID)`
194+
- `TryAllocateForest(forestName, consumerID)`
195+
- `UndoAllocateForest(forestName, consumerID)`
191196

192197
Examples of using the Quota Manager in the case of a [single tree](demos/manager/tree/demo.go) and a [forest](demos/manager/forest/demo.go) are provided.
193198

‎pkg/quotaplugins/quota-forest/quota-manager/demos/undo/forest/demo.go

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
Copyright 2023 The Multi-Cluster App Dispatcher Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
package main
17+
18+
import (
19+
"flag"
20+
"fmt"
21+
"os"
22+
23+
"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/quota-forest/quota-manager/quota"
24+
"k8s.io/klog/v2"
25+
)
26+
27+
func main() {
28+
klog.InitFlags(nil)
29+
flag.Set("v", "4")
30+
flag.Set("skip_headers", "true")
31+
klog.SetOutput(os.Stdout)
32+
flag.Parse()
33+
defer klog.Flush()
34+
35+
fmt.Println("Demo of allocation and de-allocation of consumers on a forest using the quota manager.")
36+
fmt.Println()
37+
prefix := "../../../samples/forest/"
38+
indent := "===> "
39+
forestName := "Context-Service"
40+
treeNames := []string{"ContextTree", "ServiceTree"}
41+
42+
// create a quota manager
43+
fmt.Println(indent + "Creating quota manager ... " + "\n")
44+
quotaManager := quota.NewManager()
45+
quotaManager.SetMode(quota.Normal)
46+
fmt.Println(quotaManager.GetModeString())
47+
fmt.Println()
48+
49+
// create multiple trees
50+
fmt.Println(indent + "Creating multiple trees ..." + "\n")
51+
for _, treeName := range treeNames {
52+
fName := prefix + treeName + ".json"
53+
fmt.Printf("Tree file name: %s\n", fName)
54+
jsonTree, err := os.ReadFile(fName)
55+
if err != nil {
56+
fmt.Printf("error reading quota tree file: %s", fName)
57+
return
58+
}
59+
_, err = quotaManager.AddTreeFromString(string(jsonTree))
60+
if err != nil {
61+
fmt.Printf("error adding tree %s: %v", treeName, err)
62+
return
63+
}
64+
}
65+
66+
// create forest
67+
fmt.Println(indent + "Creating forest " + forestName + " ..." + "\n")
68+
quotaManager.AddForest(forestName)
69+
for _, treeName := range treeNames {
70+
quotaManager.AddTreeToForest(forestName, treeName)
71+
}
72+
fmt.Println(quotaManager)
73+
74+
// create consumer jobs
75+
fmt.Println(indent + "Allocating consumers on forest ..." + "\n")
76+
jobs := []string{"job1", "job2", "job3", "job4", "job5"}
77+
for _, job := range jobs {
78+
79+
// create consumer info
80+
fName := prefix + job + ".json"
81+
fmt.Printf("Consumer file name: %s\n", fName)
82+
consumerInfo, err := quota.NewConsumerInfoFromFile(fName)
83+
if err != nil {
84+
fmt.Printf("error reading consumer file: %s \n", fName)
85+
continue
86+
}
87+
consumerID := consumerInfo.GetID()
88+
89+
// add consumer info to quota manager
90+
quotaManager.AddConsumer(consumerInfo)
91+
92+
// allocate forest consumer instance of the consumer info
93+
if job == "job4" {
94+
_, err = quotaManager.TryAllocateForest(forestName, consumerID)
95+
if err != nil {
96+
fmt.Printf("error allocating consumer: %v \n", err)
97+
}
98+
err = quotaManager.UndoAllocateForest(forestName, "job-4")
99+
if err != nil {
100+
fmt.Printf("error undoing allocation consumer: %v \n", err)
101+
}
102+
_, err = quotaManager.AllocateForest(forestName, consumerID)
103+
} else {
104+
_, err = quotaManager.AllocateForest(forestName, consumerID)
105+
}
106+
107+
if err != nil {
108+
fmt.Printf("error allocating consumer: %v \n", err)
109+
quotaManager.RemoveConsumer((consumerID))
110+
continue
111+
}
112+
}
113+
114+
// de-allocate consumers from forest
115+
fmt.Println(indent + "De-allocating consumers from forest ..." + "\n")
116+
for _, id := range quotaManager.GetAllConsumerIDs() {
117+
quotaManager.DeAllocateForest(forestName, id)
118+
quotaManager.RemoveConsumer(id)
119+
}
120+
fmt.Println()
121+
fmt.Println(quotaManager)
122+
}

‎pkg/quotaplugins/quota-forest/quota-manager/demos/undo/tree/demo.go

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
Copyright 2023 The Multi-Cluster App Dispatcher Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
package main
17+
18+
import (
19+
"flag"
20+
"fmt"
21+
"os"
22+
23+
"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/quota-forest/quota-manager/quota"
24+
"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/quota-forest/quota-manager/quota/core"
25+
klog "k8s.io/klog/v2"
26+
)
27+
28+
func main() {
29+
klog.InitFlags(nil)
30+
flag.Set("v", "4")
31+
flag.Set("skip_headers", "true")
32+
klog.SetOutput(os.Stdout)
33+
flag.Parse()
34+
defer klog.Flush()
35+
36+
prefix := "../../../samples/tree/"
37+
treeFileName := prefix + "tree.json"
38+
caFileName := prefix + "ca.json"
39+
cbFileName := prefix + "cb.json"
40+
ccFileName := prefix + "cc.json"
41+
cdFileName := prefix + "cd.json"
42+
ceFileName := prefix + "ce.json"
43+
44+
// create a quota manager
45+
fmt.Println("==> Creating Quota Manager")
46+
fmt.Println("**************************")
47+
quotaManager := quota.NewManager()
48+
treeJsonString, err := os.ReadFile(treeFileName)
49+
if err != nil {
50+
fmt.Printf("error reading quota tree file: %s", treeFileName)
51+
return
52+
}
53+
quotaManager.SetMode(quota.Normal)
54+
55+
// add a quota tree from file
56+
treeName, err := quotaManager.AddTreeFromString(string(treeJsonString))
57+
if err != nil {
58+
fmt.Printf("error adding tree %s: %v", treeName, err)
59+
return
60+
}
61+
62+
// allocate consumers
63+
allocate(quotaManager, treeName, caFileName, false)
64+
allocate(quotaManager, treeName, cbFileName, false)
65+
allocate(quotaManager, treeName, ccFileName, false)
66+
67+
// try and undo allocation
68+
allocate(quotaManager, treeName, cdFileName, true)
69+
undoAllocate(quotaManager, treeName, cdFileName)
70+
71+
// allocate consumers
72+
allocate(quotaManager, treeName, ceFileName, false)
73+
}
74+
75+
// allocate consumer from file
76+
func allocate(quotaManager *quota.Manager, treeName string, consumerFileName string, try bool) {
77+
consumerInfo := getConsumerInfo(consumerFileName)
78+
if consumerInfo == nil {
79+
fmt.Printf("error reading consumer file: %s", consumerFileName)
80+
return
81+
}
82+
consumerID := consumerInfo.GetID()
83+
fmt.Println("==> Allocating consumer " + consumerID)
84+
fmt.Println("**************************")
85+
quotaManager.AddConsumer(consumerInfo)
86+
87+
var allocResponse *core.AllocationResponse
88+
var err error
89+
if try {
90+
allocResponse, err = quotaManager.TryAllocate(treeName, consumerID)
91+
} else {
92+
allocResponse, err = quotaManager.Allocate(treeName, consumerID)
93+
}
94+
if err != nil {
95+
fmt.Printf("error allocating consumer: %v", err)
96+
return
97+
}
98+
fmt.Println(allocResponse)
99+
fmt.Println(quotaManager)
100+
}
101+
102+
// undo most recent consumer allocation
103+
func undoAllocate(quotaManager *quota.Manager, treeName string, consumerFileName string) {
104+
consumerInfo := getConsumerInfo(consumerFileName)
105+
if consumerInfo == nil {
106+
fmt.Printf("error reading consumer file: %s", consumerFileName)
107+
return
108+
}
109+
consumerID := consumerInfo.GetID()
110+
fmt.Println("==> Undo allocating consumer " + consumerID)
111+
fmt.Println("**************************")
112+
quotaManager.UndoAllocate(treeName, consumerID)
113+
fmt.Println(quotaManager)
114+
}
115+
116+
// get consumer info from yaml file
117+
func getConsumerInfo(consumerFileName string) *quota.ConsumerInfo {
118+
consumerInfo, err := quota.NewConsumerInfoFromFile(consumerFileName)
119+
if err != nil {
120+
fmt.Printf("error reading consumer file: %s", consumerFileName)
121+
return nil
122+
}
123+
return consumerInfo
124+
}

‎pkg/quotaplugins/quota-forest/quota-manager/quota/core/forestcontroller.go

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
55
you may not use this file except in compliance with the License.
66
You may obtain a copy of the License at
77
8-
http://www.apache.org/licenses/LICENSE-2.0
8+
http://www.apache.org/licenses/LICENSE-2.0
99
1010
Unless required by applicable law or agreed to in writing, software
1111
distributed under the License is distributed on an "AS IS" BASIS,
@@ -144,7 +144,7 @@ func (fc *ForestController) Allocate(forestConsumer *ForestConsumer) *Allocation
144144
} else if len(groupID) == 0 {
145145
fmt.Fprintf(&msg, "No quota designations provided for '%s'", treeName)
146146
} else {
147-
fmt.Fprintf(&msg, "Explected %d resources for quota designations '%s', received %d",
147+
fmt.Fprintf(&msg, "Expected %d resources for quota designations '%s', received %d",
148148
controller.GetQuotaSize(), treeName, allocRequested.GetSize())
149149
}
150150
return fc.failureRecover(consumerID, processedTrees, deletedConsumers, msg.String())
@@ -187,6 +187,7 @@ func (fc *ForestController) Allocate(forestConsumer *ForestConsumer) *Allocation
187187
/*
188188
* allocation failed - undo deletions of prior preempted consumers and recover
189189
*/
190+
// TODO: make use of forest snapshot to recover
190191
for _, c := range treeDeletedConsumers {
191192
controller.Allocate(c)
192193
}
@@ -245,6 +246,46 @@ func (fc *ForestController) failureRecover(consumerID string, processedTrees []s
245246
return failedResponse
246247
}
247248

249+
// TryAllocate : try allocating a consumer by taking a snapshot before attempting allocation
250+
func (fc *ForestController) TryAllocate(forestConsumer *ForestConsumer) *AllocationResponse {
251+
consumerID := forestConsumer.GetID()
252+
consumers := forestConsumer.GetConsumers()
253+
allocResponse := NewAllocationResponse(consumerID)
254+
255+
// take a snapshot of the forest
256+
for treeName, consumer := range consumers {
257+
var msg bytes.Buffer
258+
controller := fc.controllers[treeName]
259+
controller.treeSnapshot = NewTreeSnapshot(controller.tree, consumer)
260+
// TODO: limit the number of potentially affected consumers by the allocation
261+
if !controller.treeSnapshot.Take(controller, controller.consumers) {
262+
fmt.Fprintf(&msg, "Failed to take a state snapshot of tree '%s'", controller.GetTreeName())
263+
treeAllocResponse := NewAllocationResponse(consumer.GetID())
264+
preemptedIds := make([]string, 0)
265+
treeAllocResponse.Append(false, msg.String(), &preemptedIds)
266+
allocResponse.Merge(treeAllocResponse)
267+
return allocResponse
268+
}
269+
}
270+
271+
ar := fc.Allocate(forestConsumer)
272+
allocResponse.Merge(ar)
273+
return allocResponse
274+
}
275+
276+
// UndoAllocate : undo the most recent allocation trial
277+
func (fc *ForestController) UndoAllocate(forestConsumer *ForestConsumer) bool {
278+
klog.V(4).Infof("Multi-quota undo allocation of consumer: %s\n", forestConsumer.GetID())
279+
consumers := forestConsumer.GetConsumers()
280+
success := true
281+
for treeName, consumer := range consumers {
282+
controller := fc.controllers[treeName]
283+
treeSuccess := controller.UndoAllocate(consumer)
284+
success = success && treeSuccess
285+
}
286+
return success
287+
}
288+
248289
// ForceAllocate : force allocate a consumer on a given set of nodes on trees;
249290
// no recovery if not allocated on some trees; partial allocation allowed
250291
func (fc *ForestController) ForceAllocate(forestConsumer *ForestConsumer, nodeIDs map[string]string) *AllocationResponse {

‎pkg/quotaplugins/quota-forest/quota-manager/quota/core/quotanode.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
55
you may not use this file except in compliance with the License.
66
You may obtain a copy of the License at
77
8-
http://www.apache.org/licenses/LICENSE-2.0
8+
http://www.apache.org/licenses/LICENSE-2.0
99
1010
Unless required by applicable law or agreed to in writing, software
1111
distributed under the License is distributed on an "AS IS" BASIS,
@@ -218,11 +218,21 @@ func (qn *QuotaNode) GetAllocated() *Allocation {
218218
return qn.allocated
219219
}
220220

221+
// SetAllocated :
222+
func (qn *QuotaNode) SetAllocated(alloc *Allocation) {
223+
qn.allocated = alloc
224+
}
225+
221226
// GetConsumers :
222227
func (qn *QuotaNode) GetConsumers() []*Consumer {
223228
return qn.consumers
224229
}
225230

231+
// SetConsumers :
232+
func (qn *QuotaNode) SetConsumers(consumers []*Consumer) {
233+
qn.consumers = consumers
234+
}
235+
226236
// String : print node with a specified level of indentation
227237
func (qn *QuotaNode) String(level int) string {
228238
var b bytes.Buffer

0 commit comments

Comments
 (0)