Skip to content

Commit a3d4eab

Browse files
Fiona-WatersFiona-Waters
committed
refactored to check for specific machine names
1 parent 18792be commit a3d4eab

File tree

3 files changed

+93
-89
lines changed

3 files changed

+93
-89
lines changed

‎test/e2e/instascale_test.go

Lines changed: 59 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package e2e
22

33
import (
4-
"sync"
54
"testing"
65
"time"
76

@@ -14,13 +13,6 @@ import (
1413
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1514
)
1615

17-
var (
18-
machinePoolsExist bool
19-
numInitialNodePools int
20-
numInitialMachineSets int
21-
wg = &sync.WaitGroup{}
22-
)
23-
2416
func TestInstascale(t *testing.T) {
2517

2618
test := With(t)
@@ -65,25 +57,25 @@ func TestInstascale(t *testing.T) {
6557
}
6658
defer connection.Close()
6759

68-
machinePoolsExist = true
6960
// check existing cluster resources
70-
numInitialMachinePools, err := MachinePoolsCount(connection)
71-
if err != nil {
72-
test.T().Errorf("Unable to count machine pools - Error : %v", err)
73-
}
61+
machinePoolsExist, err := MachinePoolsExist(connection)
62+
test.Expect(err).NotTo(HaveOccurred())
63+
nodePoolsExist, err := NodePoolsExist(connection)
64+
test.Expect(err).NotTo(HaveOccurred())
7465

75-
if numInitialMachinePools == 0 {
76-
machinePoolsExist = false
77-
numInitialNodePools, err = NodePoolsCount(connection)
78-
if err != nil {
79-
test.T().Errorf("Unable to count node pools - Error : %v", err)
80-
}
81-
if numInitialNodePools == 0 {
82-
numInitialMachineSets, err = MachineSetsCount()
83-
if err != nil {
84-
test.T().Errorf("Unable to count machine sets - Error : %v", err)
85-
}
86-
}
66+
if machinePoolsExist {
67+
// look for machine pool with aw name - expect not to find it
68+
foundMachinePool, err := CheckMachinePools(connection, "test-instascale")
69+
test.Expect(err).NotTo(HaveOccurred())
70+
test.Expect(foundMachinePool).To(BeFalse())
71+
} else if nodePoolsExist {
72+
// look for node pool with aw name - expect not to find it
73+
foundNodePool, err := CheckNodePools(connection, "test-instascale")
74+
test.Expect(err).NotTo(HaveOccurred())
75+
test.Expect(foundNodePool).To(BeFalse())
76+
} else {
77+
// TODO update to foundMachineSet
78+
8779
}
8880

8981
// Batch Job
@@ -103,13 +95,13 @@ func TestInstascale(t *testing.T) {
10395
Spec: corev1.PodSpec{
10496
Containers: []corev1.Container{
10597
{
106-
Name: "job",
107-
Image: GetPyTorchImage(),
98+
Name: "job",
99+
Image: GetPyTorchImage(),
108100
Env: []corev1.EnvVar{
109101
corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/test2"},
110102
},
111103
Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"},
112-
Args: []string{"$PYTHONUSERBASE"},
104+
Args: []string{"$PYTHONUSERBASE"},
113105
VolumeMounts: []corev1.VolumeMount{
114106
{
115107
Name: "test",
@@ -184,46 +176,41 @@ func TestInstascale(t *testing.T) {
184176
},
185177
},
186178
},
187-
GenericTemplate: Raw(test, job),
179+
GenericTemplate: Raw(test, job),
180+
CompletionStatus: "Complete",
188181
},
189182
},
190183
},
191184
},
192185
}
193186

194-
_, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
187+
_, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
195188
test.Expect(err).NotTo(HaveOccurred())
196189
test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name)
197190

198191
test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort).
199192
Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
200193

201-
// wait for required resources to be created before checking them again
202-
time.Sleep(TestTimeoutShort)
203-
if !machinePoolsExist {
204-
numNodePools, err := NodePoolsCount(connection)
205-
if err != nil {
206-
test.T().Errorf("Unable to count node pools - Error : %v", err)
207-
}
208-
test.Expect(numNodePools).To(BeNumerically(">", numInitialNodePools))
209-
test.T().Logf("number of node pools increased from %d to %d", numInitialNodePools, numNodePools)
210-
211-
} else if machinePoolsExist {
212-
numMachinePools, err := MachinePoolsCount(connection)
213-
if err != nil {
214-
test.T().Errorf("Unable to count machine pools - Error : %v", err)
215-
}
216-
test.Expect(numMachinePools).To(BeNumerically(">", numInitialMachinePools))
217-
test.T().Logf("number of machine pools increased from %d to %d", numInitialMachinePools, numMachinePools)
194+
// time.Sleep is used twice throughout the test, each for 30 seconds. Can look into using sync package waitGroup instead if that makes more sense
195+
// wait for required resources to scale up before checking them again
196+
time.Sleep(TestTimeoutThirtySeconds)
197+
198+
if machinePoolsExist {
199+
// look for machine pool with aw name - expect to find it
200+
foundMachinePool, err := CheckMachinePools(connection, "test-instascale")
201+
test.Expect(err).NotTo(HaveOccurred())
202+
test.Expect(foundMachinePool).To(BeTrue())
203+
} else if nodePoolsExist {
204+
// look for node pool with aw name - expect to find it
205+
foundNodePool, err := CheckNodePools(connection, "test-instascale")
206+
test.Expect(err).NotTo(HaveOccurred())
207+
test.Expect(foundNodePool).To(BeTrue())
218208
} else {
219-
numMachineSets, err := MachineSetsCount()
220-
if err != nil {
221-
test.T().Errorf("Unable to count machine sets - Error : %v", err)
222-
}
223-
test.Expect(numMachineSets).To(BeNumerically(">", numInitialMachineSets))
224-
test.T().Logf("number of machine sets increased from %d to %d", numInitialMachineSets, numMachineSets)
209+
// TODO update to foundMachineSet
210+
225211
}
226-
212+
213+
// Assert that the job has completed
227214
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
228215
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
229216
Or(
@@ -235,30 +222,24 @@ func TestInstascale(t *testing.T) {
235222
test.Expect(GetJob(test, job.Namespace, job.Name)).
236223
To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))
237224

238-
// AppWrapper not being updated to complete once job is finished
239-
240-
time.Sleep(TestTimeoutMedium)
241-
if !machinePoolsExist {
242-
numNodePoolsFinal, err := NodePoolsCount(connection)
243-
if err != nil {
244-
test.T().Errorf("Unable to count node pools - Error : %v", err)
245-
}
246-
test.Expect(numNodePoolsFinal).To(BeNumerically("==", numInitialNodePools))
247-
test.T().Logf("number of machine pools decreased")
248-
249-
} else if machinePoolsExist {
250-
numMachinePoolsFinal, err := MachinePoolsCount(connection)
251-
if err != nil {
252-
test.T().Errorf("Unable to count machine pools - Error : %v", err)
253-
}
254-
test.Expect(numMachinePoolsFinal).To(BeNumerically("==", numInitialMachinePools))
255-
test.T().Logf("number of machine pools decreased")
225+
test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort).
226+
Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)))
227+
228+
// allow time for the resources to scale down before checking them again
229+
time.Sleep(TestTimeoutThirtySeconds)
230+
231+
if machinePoolsExist {
232+
// look for machine pool with aw name - expect to find it
233+
foundMachinePool, err := CheckMachinePools(connection, "test-instascale")
234+
test.Expect(err).NotTo(HaveOccurred())
235+
test.Expect(foundMachinePool).To(BeFalse())
236+
} else if nodePoolsExist {
237+
// look for node pool with aw name - expect to find it
238+
foundNodePool, err := CheckNodePools(connection, "test-instascale")
239+
test.Expect(err).NotTo(HaveOccurred())
240+
test.Expect(foundNodePool).To(BeFalse())
256241
} else {
257-
numMachineSetsFinal, err := MachineSetsCount()
258-
if err != nil {
259-
test.T().Errorf("Unable to count machine sets - Error : %v", err)
260-
}
261-
test.Expect(numMachineSetsFinal).To(BeNumerically("==", numInitialMachineSets))
262-
test.T().Logf("number of machine sets decreased")
242+
// TODO update to foundMachineSet
243+
263244
}
264245
}

‎test/support/clusterpools.go

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,17 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"strings"
78

89
ocmsdk "github.com/openshift-online/ocm-sdk-go"
10+
cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
911
mapiclientset "github.com/openshift/client-go/machine/clientset/versioned"
1012
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1113
)
1214

1315
var (
14-
ClusterID string = os.Getenv("CLUSTERID")
15-
machineClient mapiclientset.Interface
16+
ClusterID string = os.Getenv("CLUSTERID")
17+
machineClient mapiclientset.Interface
1618
)
1719

1820
const (
@@ -39,30 +41,50 @@ func CreateOCMConnection(secret string) (*ocmsdk.Connection, error) {
3941
return connection, nil
4042
}
4143

44+
func MachinePoolsExist(connection *ocmsdk.Connection) (bool, error) {
45+
machinePools := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools()
46+
return machinePools != nil, nil
47+
}
48+
49+
func NodePoolsExist(connection *ocmsdk.Connection) (bool, error) {
50+
nodePools := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools()
51+
return nodePools != nil, nil
52+
}
4253

43-
func MachinePoolsCount(connection *ocmsdk.Connection) (numMachinePools int, err error) {
54+
func CheckMachinePools(connection *ocmsdk.Connection, awName string) (foundMachinePool bool, err error) {
4455
machinePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List()
4556
machinePoolsListResponse, err := machinePoolsConnection.Send()
4657
if err != nil {
47-
return 0, fmt.Errorf("unable to send request, error: %v", err)
58+
return false, fmt.Errorf("unable to send request, error: %v", err)
4859
}
4960
machinePoolsList := machinePoolsListResponse.Items()
50-
numMachinePools = machinePoolsList.Len()
61+
machinePoolsList.Range(func(index int, item *cmv1.MachinePool) bool {
62+
instanceName, _ := item.GetID()
63+
if strings.Contains(instanceName, awName) {
64+
foundMachinePool = true
65+
}
66+
return true
67+
})
5168

52-
return numMachinePools, nil
69+
return foundMachinePool, err
5370
}
5471

55-
func NodePoolsCount(connection *ocmsdk.Connection) (numNodePools int, err error) {
72+
func CheckNodePools(connection *ocmsdk.Connection, awName string) (foundNodePool bool, err error) {
5673
nodePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools().List()
5774
nodePoolsListResponse, err := nodePoolsConnection.SendContext(context.Background())
5875
if err != nil {
59-
return 0, fmt.Errorf("unable to send request, error: %v", err)
76+
return false, fmt.Errorf("unable to send request, error: %v", err)
6077
}
6178
nodePoolsList := nodePoolsListResponse.Items()
62-
numNodePools = nodePoolsList.Len()
63-
fmt.Println(numNodePools)
79+
nodePoolsList.Range(func(index int, item *cmv1.NodePool) bool {
80+
instanceName, _ := item.GetID()
81+
if strings.Contains(instanceName, awName) {
82+
foundNodePool = true
83+
}
84+
return true
85+
})
6486

65-
return numNodePools, nil
87+
return foundNodePool, err
6688
}
6789

6890
func MachineSetsCount() (numMachineSets int, err error) {

‎test/support/support.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929

3030
var (
3131
TestTimeoutShort = 1 * time.Minute
32+
TestTimeoutThirtySeconds = 30 * time.Second
3233
TestTimeoutMedium = 2 * time.Minute
3334
TestTimeoutLong = 5 * time.Minute
3435

0 commit comments

Comments
 (0)