Update Makefile

.github/pull_request_template.md

@@ -1,7 +1,7 @@
 
 ---
 Checklist:
-- [ ] Run automated tests (`./build/test.sh`)
+- [ ] Run automated tests (`make test`)
 - [ ] Test end to end manually (e.g. rebuild registry/operator and run `cx refresh` in an example folder)
 - [ ] Add license header to each new file
 - [ ] Update documentation

Makefile

@@ -12,47 +12,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-###############
-# Dev Helpers #
-###############
 SHELL := /bin/bash
-GOBIN ?= $$PWD/bin
-GOOS ?=
-define build
-    mkdir -p $(GOBIN) && \
-		GOOS=$(GOOS) \
-		GOARCH=amd64 \
-		CGO_ENABLED=0 \
-		GOBIN=$(GOBIN) \
-		GO111MODULE=on \
-		go install -v \
-		-gcflags='-e' \
-		$(1)
-endef
-
-
-############
-# Operator #
-############
-olocal:
-	@./dev/operator_local.sh
 
-ostop:
-	@kubectl -n=cortex delete --ignore-not-found=true deployment operator
+#######
+# Dev #
+#######
 
-oupdate:
-	@./cortex.sh -c=./dev/config/cortex.sh update operator
+# Operator
+
+olocal:
+	@./dev/operator_local.sh || true
 
 oinstall:
 	@./cortex.sh -c=./dev/config/cortex.sh install operator
 
+oupdate:
+	@./cortex.sh -c=./dev/config/cortex.sh update operator
+
 ouninstall:
 	@./cortex.sh -c=./dev/config/cortex.sh uninstall operator
 
+ostop:
+	@kubectl -n=cortex delete --ignore-not-found=true deployment operator
+
+# EKS
 
-#######
-# EKS #
-#######
 eks-up:
 	@./dev/eks.sh start
 	$(MAKE) oinstall
@@ -68,117 +52,102 @@ eks-down:
 eks-set:
 	@./dev/eks.sh set
 
-########
-# KOPS #
-########
+# KOPS
+
 kops-up:
 	@./dev/kops.sh start
 	$(MAKE) oinstall
-	
+
 kops-up-dev:
 	$(MAKE) kops-up
 	$(MAKE) ostop
-	
+
 kops-down:
 	@./dev/kops.sh stop
 
 kops-set:
 	@./dev/kops.sh set
 
-##############
-# Misc stuff #
-##############
-
-tools:
-	@go get -u -v github.com/VojtechVitek/rerun/cmd/rerun
-	@curl -fLSs https://circle.ci/cli | bash
-
-build-cli:
-	 @$(call build, ./cli)
-
-test:
-	@./build/test.sh
+# Docker images
 
 registry-all:
 	@./dev/registry.sh update
 
 registry-dev:
-	@./dev/registry.sh update dev 	
+	@./dev/registry.sh update dev
+
+registry-create:
+	@./dev/registry.sh create
+
+# Misc
+
+.PHONY: cli
+cli:
+	@mkdir -p ./bin
+	@GOARCH=amd64 CGO_ENABLED=0 go build -o ./bin/cortex ./cli
 
 find-missing-license:
 	@./build/find-missing-license.sh
 
 find-missing-version:
 	@./build/find-missing-version.sh
-
+
+aws-clear-bucket:
+	@./dev/aws.sh clear-bucket
+
+tools:
+	@go get -u -v github.com/VojtechVitek/rerun/cmd/rerun
+	@curl -fLSs https://circle.ci/cli | bash
+
+#########
+# Tests #
+#########
+
+test:
+	@./build/test.sh
+
+test-go:
+	@./build/test.sh go
+
+test-python:
+	@./build/test.sh python
 
 ###############
 # CI Commands #
 ###############
-build-spark-base:
-	@docker build . -f images/spark-base/Dockerfile -t cortexlabs/spark-base:latest
-
-build-tf-base:
-	@docker build . -f images/tf-base/Dockerfile -t cortexlabs/tf-base:latest
-	@docker build . -f images/tf-base-gpu/Dockerfile -t cortexlabs/tf-base-gpu:latest
-
-build-base: spark-base tf-base
 
-build-tf-dev:
+build-images:
+	@./build/build-image.sh images/spark-base spark-base
+	@./build/build-image.sh images/tf-base tf-base
+	@./build/build-image.sh images/tf-base-gpu tf-base-gpu
+	@./build/build-image.sh images/spark spark
+	@./build/build-image.sh images/spark-operator spark-operator
 	@./build/build-image.sh images/tf-train tf-train
-	@./build/build-image.sh images/tf-serve tf-serve
-	@./build/build-image.sh images/tf-api tf-api
 	@./build/build-image.sh images/tf-train-gpu tf-train-gpu
+	@./build/build-image.sh images/tf-serve tf-serve
 	@./build/build-image.sh images/tf-serve-gpu tf-serve-gpu
-
-build-spark-dev:
-	@./build/build-image.sh images/spark spark
-	@./build/build-image.sh images/spark-operator spark-operator
-
-build-tf-images: build-tf-base build-tf-dev
-build-spark-images: build-spark-base build-spark-dev
-
-build-argo-images:
-	@./build/build-image.sh images/argo-controller argo-controller
-	@./build/build-image.sh images/argo-executor argo-executor
-
-build-operator-images:
+	@./build/build-image.sh images/tf-api tf-api
 	@./build/build-image.sh images/operator operator
+	@./build/build-image.sh images/fluentd fluentd
 	@./build/build-image.sh images/nginx-controller nginx-controller
 	@./build/build-image.sh images/nginx-backend nginx-backend
-	@./build/build-image.sh images/fluentd fluentd
-
-build-images: build-tf-images build-spark-images build-argo-images build-operator-images
+	@./build/build-image.sh images/argo-controller argo-controller
+	@./build/build-image.sh images/argo-executor argo-executor
 
-push-tf-images:
+push-images:
+	@./build/push-image.sh spark
+	@./build/push-image.sh spark-operator
 	@./build/push-image.sh tf-train
-	@./build/push-image.sh tf-serve
-	@./build/push-image.sh tf-api
 	@./build/push-image.sh tf-train-gpu
+	@./build/push-image.sh tf-serve
 	@./build/push-image.sh tf-serve-gpu
-
-push-spark-images:
-	@./build/push-image.sh spark
-	@./build/push-image.sh spark-operator
-
-push-argo-images:
-	@./build/push-image.sh argo-controller
-	@./build/push-image.sh argo-executor
-
-push-operator-images:
+	@./build/push-image.sh tf-api
 	@./build/push-image.sh operator
+	@./build/push-image.sh fluentd
 	@./build/push-image.sh nginx-controller
 	@./build/push-image.sh nginx-backend
-	@./build/push-image.sh fluentd
-
-push-images: push-tf-images push-spark-images push-argo-images push-operator-images
+	@./build/push-image.sh argo-controller
+	@./build/push-image.sh argo-executor
 
 build-and-upload-cli:
 	@./build/cli.sh
-
-test-go:
-	@./build/test.sh go
-
-test-python:
-	@./build/test.sh python
-

build/cli.sh

@@ -23,7 +23,7 @@ function build_and_upload() {
   set -euo pipefail
 
   os=$1
-  GOOS=$os GOARCH=amd64 CGO_ENABLED=0 GO111MODULE=on go build -installsuffix cgo -o cortex github.com/cortexlabs/cortex/cli
+  GOOS=$os GOARCH=amd64 CGO_ENABLED=0 GO111MODULE=on go build -o cortex github.com/cortexlabs/cortex/cli
   aws s3 cp cortex s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex --only-show-errors
   rm cortex
   echo "Uploaded CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex"

build/find-missing-version.sh

@@ -14,10 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-output=$(grep --exclude="find-missing-version.sh" -R -A 50 -e "CORTEX_VERSION" . | grep -e "master")
+output=$(find . -type f \
+! -path "./vendor/*" \
+! -path "./bin/*" \
+! -path "./build/find-missing-version.sh" \
+! -path "./.git/*" \
+! -name ".*" \
+-exec grep -R -A 50 -e "CORTEX_VERSION" {} \+)
+
+output=$(echo "$output" | grep -e "master")
 
 if [[ $output ]]; then
   echo "$output"
   exit 1
 fi
-

dev/aws.sh

@@ -21,13 +21,11 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
 
 source $ROOT/dev/config/build.sh
 source $ROOT/dev/config/k8s.sh
+source $ROOT/dev/config/cortex.sh
 source $ROOT/dev/util.sh
 
-if [ "$1" = "delete-cache" ]; then
-  aws s3 rm --recursive --quiet s3://$K8S_BUCKET/apps
-  aws s3 rm --recursive --quiet s3://$K8S_BUCKET/contexts
-  aws s3 rm --recursive --quiet s3://$K8S_BUCKET/model_implementations
-  aws s3 rm --recursive --quiet s3://$K8S_BUCKET/transformations
+if [ "$1" = "clear-bucket" ]; then
+  aws s3 rm --recursive --quiet s3://$CORTEX_BUCKET
 else
   echo "Command $1 not found"
 fi