Skip to content

Commit c156828

Browse files
jbuschejbusche
authored
Add KWOK scripts to perf folder (#342)
Signed-off-by: James Busche <jbusche@us.ibm.com>
1 parent a4ef3b0 commit c156828

File tree

6 files changed

+589
-0
lines changed

6 files changed

+589
-0
lines changed

‎test/perf-test/cleanup-mcad-kwok.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
for i in `kubectl get appwrapper -n default |grep fake-defaultaw | awk '{print $1}'`; do kubectl delete appwrapper $i -n default ; done

‎test/perf-test/kwokmcadperf.sh

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
#!/bin/bash
2+
3+
SCRIPT_DIR=$(readlink -f `dirname "${BASH_SOURCE[0]}"`)
4+
5+
function help() {
6+
echo "usage: kwokmcadperf.sh [-h]"
7+
echo
8+
echo "Description: Runs Appwrapper performance test script(s) in subdirectories under $SCRIPT_DIR."
9+
echo "NOTE: This runs on KWOK Fake nodes only."
10+
echo
11+
echo "Preconditions: "
12+
echo " - The script assumes you've logged into your cluster already. If not, it will tell you to login."
13+
echo " - The script checks that you have the mcad-controller installed, otherwise it'll tell you to install it first."
14+
echo " - The script checks that you have the kwok-controller installed, otherwise it'll tell you to install it first."
15+
echo
16+
echo "Options:"
17+
echo " -h Print this help message"
18+
echo
19+
}
20+
21+
function check_kubectl_login_status() {
22+
set +e
23+
kubectl get ns default &> /dev/null
24+
res="$?"
25+
set -e
26+
OCP="$res"
27+
if [ $OCP == 1 ]
28+
then
29+
echo "You need to login to your Kubernetes Cluster"
30+
exit 1
31+
else
32+
echo
33+
echo "Nice, looks like you're logged in"
34+
fi
35+
}
36+
37+
function check_mcad_installed_status() {
38+
set +e
39+
kubectl get pod -A |grep mcad-controller &> /dev/null
40+
res2="$?"
41+
kubectl get crd |grep appwrapper &> /dev/null
42+
res3="$?"
43+
set -e
44+
MCAD="$res2"
45+
CRD="$res3"
46+
if [[ $MCAD == 1 ]] || [[ $CRD == 1 ]]
47+
then
48+
echo "You need Install MCAD Controller first before running this script"
49+
exit 1
50+
else
51+
echo "Nice, MCAD Controller is installed"
52+
fi
53+
}
54+
55+
function check_kwok_installed_status() {
56+
set +e
57+
kubectl get pod -A |grep kwok-controller &> /dev/null
58+
res2="$?"
59+
set -e
60+
KWOK="$res2"
61+
if [[ $KWOK == 1 ]]
62+
then
63+
echo "You need Install the KWOK Controller first before running this script"
64+
exit 1
65+
else
66+
echo "Nice, the KWOK Controller is installed"
67+
fi
68+
}
69+
70+
71+
while getopts hf: option; do
72+
case $option in
73+
h)
74+
help
75+
exit 0
76+
;;
77+
*)
78+
;;
79+
esac
80+
done
81+
shift $((OPTIND-1))
82+
83+
# Track whether we have a valid kubectl login
84+
echo "Checking whether we have a valid cluster login or not..."
85+
check_kubectl_login_status
86+
87+
# Track whether you have the MCAD controller installed
88+
echo "Checking MCAD Controller installation status"
89+
echo
90+
check_mcad_installed_status
91+
92+
# Track whether you have the KWOK controller installed
93+
echo "Checking MCAD Controller installation status"
94+
echo
95+
check_kwok_installed_status
96+
97+
echo
98+
read -p "How many fake KWOK appwrapper jobs do you want?" jobs
99+
100+
# Start the timer now
101+
SECONDS=0
102+
103+
echo "jobs number is $jobs"
104+
export STARTTIME=`date +"%T"`
105+
echo " "
106+
echo "Jobs started at: $STARTTIME" |tee fake-job-$STARTTIME.log
107+
echo " "
108+
109+
# This fixes the number of jobs to be one less so the for loop gets the right amount
110+
((realjobs=$jobs-1))
111+
112+
for num in $(eval echo "{0.."$realjobs"}")
113+
do
114+
next_num=$(($num + 1))
115+
echo "Submitting job $next_num"
116+
# Had to do this OSTYPE because sed acts differently on Linux versus Mac
117+
case "$OSTYPE" in
118+
linux-gnu*)
119+
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
120+
darwin*)
121+
sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
122+
*)
123+
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
124+
esac
125+
kubectl apply -f ${SCRIPT_DIR}/preempt-exp-kwok.yaml
126+
done
127+
128+
# Let's reset the original preempt-exp-kwok.yaml file back to original value
129+
case "$OSTYPE" in
130+
linux-gnu*)
131+
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
132+
darwin*)
133+
sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
134+
*)
135+
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
136+
esac
137+
138+
# Check for all jobs to report complete
139+
jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
140+
141+
while [ $jobstatus -lt $jobs ]
142+
do
143+
echo "Number of completed jobs is: " $jobstatus " and the goal is: " $jobs
144+
sleep 10
145+
jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
146+
done
147+
148+
echo " "
149+
export FINISHTIME=`date +"%T"`
150+
echo "All $jobstatus jobs finished: $FINISHTIME" |tee -a fake-job-$STARTTIME.log
151+
echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/fake-job-$STARTTIME.log
152+
echo " "
153+
echo "Test results are stored in this file: ${SCRIPT_DIR}/fake-job-$next_num-$STARTTIME.log"
154+
155+
# Rename the log to show the number of jobs used
156+
mv ${SCRIPT_DIR}/fake-job-$STARTTIME.log ${SCRIPT_DIR}/fake-job-$next_num-$STARTTIME.log
157+
158+
#Ask if you want to auto-cleanup the appwrapper jobs
159+
echo "Do you want to cleanup the most recently created appwrappers? [Y/n]"
160+
read DELETE
161+
if [[ $DELETE == "Y" || $DELETE == "y" ]]; then
162+
echo "OK, deleting"
163+
${SCRIPT_DIR}/cleanup-mcad-kwok.sh
164+
else
165+
echo "OK, you'll need to cleanup yourself later using ./cleanup-mcad-kwok.sh"
166+
fi

‎test/perf-test/node.yaml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
apiVersion: v1
2+
kind: Node
3+
metadata:
4+
annotations:
5+
node.alpha.kubernetes.io/ttl: "0"
6+
kwok.x-k8s.io/node: fake
7+
labels:
8+
beta.kubernetes.io/arch: amd64
9+
beta.kubernetes.io/os: linux
10+
kubernetes.io/arch: amd64
11+
kubernetes.io/hostname: kwok-node-0
12+
kubernetes.io/os: linux
13+
kubernetes.io/role: agent
14+
node-role.kubernetes.io/agent: ""
15+
type: kwok
16+
name: kwok-node-0
17+
spec:
18+
taints: # Avoid scheduling actual running pods to fake Node
19+
- effect: NoSchedule
20+
key: kwok.x-k8s.io/node
21+
value: fake
22+
status:
23+
allocatable:
24+
cpu: 32
25+
memory: 256Gi
26+
pods: 110
27+
capacity:
28+
cpu: 32
29+
memory: 256Gi
30+
pods: 110
31+
nodeInfo:
32+
architecture: amd64
33+
bootID: ""
34+
containerRuntimeVersion: ""
35+
kernelVersion: ""
36+
kubeProxyVersion: fake
37+
kubeletVersion: fake
38+
machineID: ""
39+
operatingSystem: linux
40+
osImage: ""
41+
systemUUID: ""
42+
phase: Running

‎test/perf-test/nodes.sh

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/bin/bash
2+
3+
SCRIPT_DIR=$(readlink -f `dirname "${BASH_SOURCE[0]}"`)
4+
5+
function help() {
6+
echo "usage: nodes.sh [-h]"
7+
echo
8+
echo "Description: Creates fake KWOK nodes for performance testing"
9+
echo
10+
echo "Preconditions: "
11+
echo " - The script assumes you've logged into your cluster already. If not, it will tell you to login."
12+
echo " - The script checks that you have the kwok-controller installed, otherwise it'll tell you to install it first."
13+
echo
14+
echo "Options:"
15+
echo " -h Print this help message"
16+
echo
17+
}
18+
19+
function check_kubectl_login_status() {
20+
set +e
21+
kubectl get ns default &> /dev/null
22+
res="$?"
23+
set -e
24+
OCP="$res"
25+
if [ $OCP == 1 ]
26+
then
27+
echo "You need to login to your Kubernetes Cluster"
28+
exit 1
29+
else
30+
echo
31+
echo "Nice, looks like you're logged in"
32+
echo ""
33+
fi
34+
}
35+
36+
function check_kwok_installed_status() {
37+
set +e
38+
kubectl get pod -A |grep kwok-controller &> /dev/null
39+
res2="$?"
40+
set -e
41+
KWOK="$res2"
42+
if [[ $KWOK == 1 ]]
43+
then
44+
echo "You need Install the KWOK Controller first before running this script"
45+
exit 1
46+
else
47+
echo "Nice, the KWOK Controller is installed"
48+
fi
49+
}
50+
51+
while getopts hf: option; do
52+
case $option in
53+
h)
54+
help
55+
exit 0
56+
;;
57+
*)
58+
;;
59+
esac
60+
done
61+
shift $((OPTIND-1))
62+
63+
# Track whether we have a valid kubectl login
64+
echo "Checking whether we have a valid cluster login or not..."
65+
check_kubectl_login_status
66+
67+
# Track whether you have the KWOK controller installed
68+
echo "Checking MCAD Controller installation status"
69+
echo
70+
check_kwok_installed_status
71+
72+
echo
73+
read -p "How many simulated KWOK nodes do you want?" nodes
74+
75+
echo "Nodes number is $nodes"
76+
echo " "
77+
78+
# This fixes the number of jobs to be one less so the for loop gets the right amount
79+
((realnodes=$nodes-1))
80+
echo "The real number of nodes is $realnodes"
81+
82+
for num in $(eval echo "{0.."$realnodes"}")
83+
do
84+
next_num=$(($num + 1))
85+
echo "Submitting node $next_num"
86+
# Had to do this OSTYPE because sed acts differently on Linux versus Mac
87+
case "$OSTYPE" in
88+
linux-gnu*)
89+
sed -i "s/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ;;
90+
darwin*)
91+
sed -i '' "s/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ${SCRIPT_DIR}/node.yaml ;;
92+
*)
93+
sed -i "/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ;;
94+
esac
95+
kubectl apply -f ${SCRIPT_DIR}/node.yaml
96+
done
97+
98+
# Let's reset the original node.yaml file back to original value
99+
case "$OSTYPE" in
100+
linux-gnu*)
101+
sed -i "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
102+
darwin*)
103+
sed -i '' "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
104+
*)
105+
sed -i "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
106+
esac
107+
108+
# Check for all nodes to report complete
109+
echo "Waiting until all the simualted pods become ready:"
110+
kubectl wait --for=condition=Ready nodes --selector type=kwok --timeout=600s
111+
echo " "
112+
echo "Total amount of simulated nodes requested is: $nodes"
113+
echo "Total number of created nodes is: "`kubectl get nodes --selector type=kwok -o name |wc -l`
114+
kubectl get nodes --selector type=kwok
115+
116+
echo " "
117+
echo "FYI, to clean up the kwow nodes, issue this:"
118+
echo "kubectl get nodes --selector type=kwok -o name | xargs kubectl delete"

‎test/perf-test/preempt-exp-kwok.yaml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
apiVersion: mcad.ibm.com/v1beta1
2+
kind: AppWrapper
3+
metadata:
4+
name: fake-defaultaw-schd-spec-with-timeout-1
5+
namespace: default
6+
spec:
7+
schedulingSpec:
8+
minAvailable: 1
9+
requeuing:
10+
timeInSeconds: 120
11+
growthType: "exponential"
12+
priority: 9
13+
resources:
14+
Items: []
15+
GenericItems:
16+
- replicas: 1
17+
completionstatus: Complete
18+
custompodresources:
19+
- replicas: 1
20+
requests:
21+
cpu: 10m
22+
memory: 10M
23+
nvidia.com/gpu: 0
24+
limits:
25+
cpu: 500m
26+
memory: 128M
27+
nvidia.com/gpu: 0
28+
generictemplate:
29+
apiVersion: batch/v1
30+
kind: Job
31+
metadata:
32+
namespace: default
33+
name: fake-defaultaw-schd-spec-with-timeout-1
34+
spec:
35+
parallelism: 1
36+
completions: 1
37+
template:
38+
metadata:
39+
namespace: default
40+
labels:
41+
appwrapper.mcad.ibm.com: "fake-defaultaw-schd-spec-with-timeout-1"
42+
spec:
43+
affinity:
44+
nodeAffinity:
45+
requiredDuringSchedulingIgnoredDuringExecution:
46+
nodeSelectorTerms:
47+
- matchExpressions:
48+
- key: type
49+
operator: In
50+
values:
51+
- kwok
52+
# A taints was added to an automatically created Node.
53+
# You can remove taints of Node or add this tolerations.
54+
tolerations:
55+
- key: "kwok.x-k8s.io/node"
56+
operator: "Exists"
57+
effect: "NoSchedule"
58+
containers:
59+
- name: fake-defaultaw-schd-spec-with-timeout-1
60+
image: fake-image
61+
restartPolicy: Never

0 commit comments

Comments
 (0)