From 31838a06b946fcc6baa0321df730f62b8892a748 Mon Sep 17 00:00:00 2001 From: James Busche Date: Fri, 17 Mar 2023 14:47:52 -0700 Subject: [PATCH 1/5] Adding Performance test script Signed-off-by: James Busche --- test/perf-test/cleanup.sh | 3 ++ test/perf-test/perf.sh | 71 +++++++++++++++++++++++++++++++++ test/perf-test/preempt-exp.yaml | 57 ++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100755 test/perf-test/cleanup.sh create mode 100755 test/perf-test/perf.sh create mode 100644 test/perf-test/preempt-exp.yaml diff --git a/test/perf-test/cleanup.sh b/test/perf-test/cleanup.sh new file mode 100755 index 000000000..ae808704b --- /dev/null +++ b/test/perf-test/cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +for i in `kubectl get appwrapper -n default |grep defaultaw | awk '{print $1}'`; do kubectl delete appwrapper $i -n default ; done diff --git a/test/perf-test/perf.sh b/test/perf-test/perf.sh new file mode 100755 index 000000000..808212c93 --- /dev/null +++ b/test/perf-test/perf.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +read -p "How many appwrapper jobs do you want?" jobs + +# Start the timer now +SECONDS=0 + +echo "jobs number is $jobs" +export STARTTIME=`date +"%T"` +echo " " +echo "Jobs started at: $STARTTIME" |tee job-$STARTTIME.log +echo " " + +# This fixes the number of jobs to be one less so the for loop gets the right amount +((realjobs=$jobs-1)) + +for num in $(eval echo "{0.."$realjobs"}") +do + next_num=$(($num + 1)) + echo "Submitting job $next_num" +# Had to do this OSTYPE because sed acts differently on Linux versus Mac + case "$OSTYPE" in + linux-gnu*) + sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" preempt-exp.yaml ;; + darwin*) + sed -i '' "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" preempt-exp.yaml ;; + *) + sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" preempt-exp.yaml ;; + esac + oc apply -f preempt-exp.yaml +done + + # Let's reset the original preempt-exp.yaml file back to original value + case "$OSTYPE" in + linux-gnu*) + sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" preempt-exp.yaml ;; + darwin*) + sed -i '' "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" preempt-exp.yaml ;; + *) + sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" preempt-exp.yaml ;; + esac + +# Check for all jobs to report complete +jobstatus=`oc get jobs --no-headers --field-selector status.successful=1 |wc -l` + +while [ $jobstatus -lt $jobs ] +do + echo "Number of completed jobs is: " $jobstatus " and the goal is: " $jobs + sleep 10 + jobstatus=`oc get jobs --no-headers --field-selector status.successful=1 |wc -l` +done + +echo " " +export FINISHTIME=`date +"%T"` +echo "All $jobstatus jobs finished: $FINISHTIME" |tee -a job-$STARTTIME.log +echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a job-$STARTTIME.log +echo " " +echo "Test results are stored in this file: job-$next_num-$STARTTIME.log" + +# Rename the log to show the number of jobs used +mv job-$STARTTIME.log job-$next_num-$STARTTIME.log + +#Ask if you want to auto-cleanup the appwrapper jobs +echo "Do you want to cleanup the most recently created appwrappers? [Y/n]" +read DELETE +if [[ $DELETE == "Y" || $DELETE == "y" ]]; then + echo "OK, deleting" + ./cleanup.sh +else + echo "OK, you'll need to cleanup yourself later using ./cleanup.sh" +fi diff --git a/test/perf-test/preempt-exp.yaml b/test/perf-test/preempt-exp.yaml new file mode 100644 index 000000000..6eaf61f07 --- /dev/null +++ b/test/perf-test/preempt-exp.yaml @@ -0,0 +1,57 @@ +apiVersion: mcad.ibm.com/v1beta1 +kind: AppWrapper +metadata: + name: defaultaw-schd-spec-with-timeout-1 + namespace: default +spec: + schedulingSpec: + minAvailable: 1 + requeuing: + timeInSeconds: 120 + growthType: "exponential" + priority: 9 + resources: + Items: [] + GenericItems: + - replicas: 1 + completionstatus: Complete + custompodresources: + - replicas: 1 + requests: + cpu: 500m + memory: 512M + nvidia.com/gpu: 0 + limits: + cpu: 500m + memory: 512M + nvidia.com/gpu: 0 + generictemplate: + apiVersion: batch/v1 + kind: Job + metadata: + namespace: default + name: defaultaw-schd-spec-with-timeout-1 + # labels: + # appwrapper.mcad.ibm.com: defaultaw-schd-spec-with-timeout-1 + spec: + parallelism: 1 + completions: 1 + template: + metadata: + namespace: default + labels: + appwrapper.mcad.ibm.com: "defaultaw-schd-spec-with-timeout-1" + spec: + containers: + - name: defaultaw-schd-spec-with-timeout-1 + image: ubuntu:latest + command: [ "/bin/bash", "-c", "--" ] + args: [ "sleep 10" ] + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "500m" + restartPolicy: Never From 36016c699d5def862e3828b7a713d82200dff6a2 Mon Sep 17 00:00:00 2001 From: James Busche Date: Thu, 30 Mar 2023 13:08:35 -0700 Subject: [PATCH 2/5] red hat image and more checking Signed-off-by: James Busche --- test/perf-test/perf.sh | 69 +++++++++++++++++++++++++++------ test/perf-test/preempt-exp.yaml | 14 +++---- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/test/perf-test/perf.sh b/test/perf-test/perf.sh index 808212c93..e6183a80b 100755 --- a/test/perf-test/perf.sh +++ b/test/perf-test/perf.sh @@ -1,5 +1,52 @@ #!/bin/bash +SCRIPT_DIR=$(readlink -f `dirname "${BASH_SOURCE[0]}"`) + +function help() { + echo "usage: perf.sh [-h]" + echo + echo "Description: Runs Appwrapper performance test script(s) in subdirectories under $SCRIPT_DIR." + echo + echo "Preconditions: The Script assumes you've logged into your cluster already. If not, it will tell you to login." + echo + echo "Options:" + echo " -h Print this help message" + echo +} + +function check_kubectl_login_status() { + set +e + kubectl get ns default &> /dev/null + res="$?" + set -e + OCP="$res" + if [ $OCP == 1 ] + then + echo "You need to login to your Kubernetes Cluster" + exit 1 + else + echo + echo "Nice, looks like you're logged in" + echo + fi +} + +while getopts hf: option; do + case $option in + h) + help + exit 0 + ;; + *) + ;; + esac +done +shift $((OPTIND-1)) + +# Track whether we have a valid kubectl login +echo "Checking whether we have a valid oc login or not..." +check_kubectl_login_status + read -p "How many appwrapper jobs do you want?" jobs # Start the timer now @@ -21,23 +68,23 @@ do # Had to do this OSTYPE because sed acts differently on Linux versus Mac case "$OSTYPE" in linux-gnu*) - sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" preempt-exp.yaml ;; + sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;; darwin*) - sed -i '' "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" preempt-exp.yaml ;; + sed -i '' "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;; *) - sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" preempt-exp.yaml ;; + sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;; esac - oc apply -f preempt-exp.yaml + oc apply -f ${SCRIPT_DIR}/preempt-exp.yaml done # Let's reset the original preempt-exp.yaml file back to original value case "$OSTYPE" in linux-gnu*) - sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" preempt-exp.yaml ;; + sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;; darwin*) - sed -i '' "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" preempt-exp.yaml ;; + sed -i '' "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;; *) - sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" preempt-exp.yaml ;; + sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;; esac # Check for all jobs to report complete @@ -53,19 +100,19 @@ done echo " " export FINISHTIME=`date +"%T"` echo "All $jobstatus jobs finished: $FINISHTIME" |tee -a job-$STARTTIME.log -echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a job-$STARTTIME.log +echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/job-$STARTTIME.log echo " " -echo "Test results are stored in this file: job-$next_num-$STARTTIME.log" +echo "Test results are stored in this file: ${SCRIPT_DIR}/job-$next_num-$STARTTIME.log" # Rename the log to show the number of jobs used -mv job-$STARTTIME.log job-$next_num-$STARTTIME.log +mv ${SCRIPT_DIR}/job-$STARTTIME.log ${SCRIPT_DIR}/job-$next_num-$STARTTIME.log #Ask if you want to auto-cleanup the appwrapper jobs echo "Do you want to cleanup the most recently created appwrappers? [Y/n]" read DELETE if [[ $DELETE == "Y" || $DELETE == "y" ]]; then echo "OK, deleting" - ./cleanup.sh + ${SCRIPT_DIR}/cleanup.sh else echo "OK, you'll need to cleanup yourself later using ./cleanup.sh" fi diff --git a/test/perf-test/preempt-exp.yaml b/test/perf-test/preempt-exp.yaml index 6eaf61f07..d74c00c51 100644 --- a/test/perf-test/preempt-exp.yaml +++ b/test/perf-test/preempt-exp.yaml @@ -18,12 +18,12 @@ spec: custompodresources: - replicas: 1 requests: - cpu: 500m - memory: 512M + cpu: 10m + memory: 10M nvidia.com/gpu: 0 limits: cpu: 500m - memory: 512M + memory: 128M nvidia.com/gpu: 0 generictemplate: apiVersion: batch/v1 @@ -44,14 +44,14 @@ spec: spec: containers: - name: defaultaw-schd-spec-with-timeout-1 - image: ubuntu:latest + image: ubi8-minimal:latest command: [ "/bin/bash", "-c", "--" ] args: [ "sleep 10" ] resources: requests: - memory: "512Mi" - cpu: "500m" + memory: "10Mi" + cpu: "10m" limits: - memory: "512Mi" + memory: "128Mi" cpu: "500m" restartPolicy: Never From 29e19030df4ebd436d305b8593c64f25283d63e2 Mon Sep 17 00:00:00 2001 From: James Busche Date: Thu, 30 Mar 2023 13:34:02 -0700 Subject: [PATCH 3/5] Check for mcad install Signed-off-by: James Busche --- test/perf-test/perf.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/test/perf-test/perf.sh b/test/perf-test/perf.sh index e6183a80b..2c7a4a9e0 100755 --- a/test/perf-test/perf.sh +++ b/test/perf-test/perf.sh @@ -7,7 +7,9 @@ function help() { echo echo "Description: Runs Appwrapper performance test script(s) in subdirectories under $SCRIPT_DIR." echo - echo "Preconditions: The Script assumes you've logged into your cluster already. If not, it will tell you to login." + echo "Preconditions: " + echo " - The script assumes you've logged into your cluster already. If not, it will tell you to login." + echo " - The script checks that you have the mcad-controller installed, otherwise it'll tell you to install it first." echo echo "Options:" echo " -h Print this help message" @@ -27,10 +29,28 @@ function check_kubectl_login_status() { else echo echo "Nice, looks like you're logged in" - echo fi } +function check_mcad_installed_status() { + set +e + kubectl get pod -A |grep mcad-controller &> /dev/null + res2="$?" + kubectl get crd |grep appwrapper &> /dev/null + res3="$?" + set -e + MCAD="$res2" + CRD="$res3" + if [[ $MCAD == 1 ]] || [[ $CRD == 1 ]] + then + echo "You need Install MCAD Controller first before running this script" + exit 1 + else + echo "Nice, MCAD Controller is installed" + fi +} + + while getopts hf: option; do case $option in h) @@ -47,6 +67,12 @@ shift $((OPTIND-1)) echo "Checking whether we have a valid oc login or not..." check_kubectl_login_status +# Track whether you have the MCAD controller installed +echo "Checking MCAD Controller installation status" +echo +check_mcad_installed_status + +echo read -p "How many appwrapper jobs do you want?" jobs # Start the timer now From a1a4dbdb405592ea92c19bd16d5b398e1a70c11f Mon Sep 17 00:00:00 2001 From: James Busche Date: Thu, 6 Apr 2023 08:57:13 -0700 Subject: [PATCH 4/5] remove oc and formatting Signed-off-by: James Busche --- test/perf-test/perf.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/perf-test/perf.sh b/test/perf-test/perf.sh index 2c7a4a9e0..a8a04b72c 100755 --- a/test/perf-test/perf.sh +++ b/test/perf-test/perf.sh @@ -29,7 +29,7 @@ function check_kubectl_login_status() { else echo echo "Nice, looks like you're logged in" - fi + fi } function check_mcad_installed_status() { @@ -64,7 +64,7 @@ done shift $((OPTIND-1)) # Track whether we have a valid kubectl login -echo "Checking whether we have a valid oc login or not..." +echo "Checking whether we have a valid cluster login or not..." check_kubectl_login_status # Track whether you have the MCAD controller installed @@ -100,7 +100,7 @@ do *) sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;; esac - oc apply -f ${SCRIPT_DIR}/preempt-exp.yaml + kubectl apply -f ${SCRIPT_DIR}/preempt-exp.yaml done # Let's reset the original preempt-exp.yaml file back to original value @@ -114,13 +114,13 @@ done esac # Check for all jobs to report complete -jobstatus=`oc get jobs --no-headers --field-selector status.successful=1 |wc -l` +jobstatus=`kubectl get jobs --no-headers --field-selector status.successful=1 |wc -l` while [ $jobstatus -lt $jobs ] do echo "Number of completed jobs is: " $jobstatus " and the goal is: " $jobs sleep 10 - jobstatus=`oc get jobs --no-headers --field-selector status.successful=1 |wc -l` + jobstatus=`kubectl get jobs --no-headers --field-selector status.successful=1 |wc -l` done echo " " From f8f9cb44fdfa295b482b84ebbd11718ffee82910 Mon Sep 17 00:00:00 2001 From: James Busche Date: Thu, 6 Apr 2023 09:14:12 -0700 Subject: [PATCH 5/5] fix to run from any namespace Signed-off-by: James Busche --- test/perf-test/perf.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/perf-test/perf.sh b/test/perf-test/perf.sh index a8a04b72c..7450d7e25 100755 --- a/test/perf-test/perf.sh +++ b/test/perf-test/perf.sh @@ -114,13 +114,13 @@ done esac # Check for all jobs to report complete -jobstatus=`kubectl get jobs --no-headers --field-selector status.successful=1 |wc -l` +jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l` while [ $jobstatus -lt $jobs ] do echo "Number of completed jobs is: " $jobstatus " and the goal is: " $jobs sleep 10 - jobstatus=`kubectl get jobs --no-headers --field-selector status.successful=1 |wc -l` + jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l` done echo " "