Improve observability of API statuses

cli/cluster/delete.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/json"
+	"github.com/cortexlabs/cortex/pkg/lib/pointer"
 	"github.com/cortexlabs/cortex/pkg/lib/prompt"
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
@@ -70,8 +71,7 @@ func getReadyRealtimeAPIReplicasOrNil(operatorConfig OperatorConfig, apiName str
 		return nil
 	}
 
-	totalReady := apiRes.Status.Updated.Ready + apiRes.Status.Stale.Ready
-	return &totalReady
+	return pointer.Int32(apiRes.Status.Ready)
 }
 
 func StopJob(operatorConfig OperatorConfig, kind userconfig.Kind, apiName string, jobID string) (schema.DeleteResponse, error) {

cli/cluster/get.go

@@ -51,6 +51,20 @@ func GetAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse
 	return apiRes, nil
 }
 
+func DescribeAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse, error) {
+	httpRes, err := HTTPGet(operatorConfig, "/describe/"+apiName)
+	if err != nil {
+		return nil, err
+	}
+
+	var apiRes []schema.APIResponse
+	if err = json.Unmarshal(httpRes, &apiRes); err != nil {
+		return nil, errors.Wrap(err, "/describe/"+apiName, string(httpRes))
+	}
+
+	return apiRes, nil
+}
+
 func GetAPIByID(operatorConfig OperatorConfig, apiName string, apiID string) ([]schema.APIResponse, error) {
 	httpRes, err := HTTPGet(operatorConfig, "/get/"+apiName+"/"+apiID)
 	if err != nil {

cli/cmd/describe.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2021 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/cortexlabs/cortex/cli/cluster"
+	"github.com/cortexlabs/cortex/cli/types/cliconfig"
+	"github.com/cortexlabs/cortex/pkg/lib/errors"
+	"github.com/cortexlabs/cortex/pkg/lib/exit"
+	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
+	"github.com/cortexlabs/cortex/pkg/types/userconfig"
+	"github.com/spf13/cobra"
+)
+
+const (
+	_titleReplicaStatus = "replica status"
+	_titleReplicaCount  = "replica count"
+)
+
+var (
+	_flagDescribeEnv   string
+	_flagDescribeWatch bool
+)
+
+func describeInit() {
+	_describeCmd.Flags().SortFlags = false
+	_describeCmd.Flags().StringVarP(&_flagDescribeEnv, "env", "e", "", "environment to use")
+	_describeCmd.Flags().BoolVarP(&_flagDescribeWatch, "watch", "w", false, "re-run the command every 2 seconds")
+}
+
+var _describeCmd = &cobra.Command{
+	Use:   "describe [API_NAME]",
+	Short: "describe an api",
+	Args:  cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		apiName := args[0]
+
+		var envName string
+		if wasFlagProvided(cmd, "env") {
+			envName = _flagDescribeEnv
+		} else {
+			var err error
+			envName, err = getEnvFromFlag("")
+			if err != nil {
+				telemetry.Event("cli.describe")
+				exit.Error(err)
+			}
+		}
+
+		env, err := ReadOrConfigureEnv(envName)
+		if err != nil {
+			telemetry.Event("cli.describe")
+			exit.Error(err)
+		}
+		telemetry.Event("cli.describe", map[string]interface{}{"env_name": env.Name})
+
+		rerun(_flagDescribeWatch, func() (string, error) {
+			env, err := ReadOrConfigureEnv(envName)
+			if err != nil {
+				exit.Error(err)
+			}
+
+			out, err := envStringIfNotSpecified(envName, cmd)
+			if err != nil {
+				return "", err
+			}
+			apiTable, err := describeAPI(env, apiName)
+			if err != nil {
+				return "", err
+			}
+
+			return out + apiTable, nil
+		})
+	},
+}
+
+func describeAPI(env cliconfig.Environment, apiName string) (string, error) {
+	apisRes, err := cluster.DescribeAPI(MustGetOperatorConfig(env.Name), apiName)
+	if err != nil {
+		return "", err
+	}
+
+	if len(apisRes) == 0 {
+		exit.Error(errors.ErrorUnexpected(fmt.Sprintf("unable to find api %s", apiName)))
+	}
+
+	apiRes := apisRes[0]
+
+	switch apiRes.Metadata.Kind {
+	case userconfig.RealtimeAPIKind:
+		return realtimeDescribeAPITable(apiRes, env)
+	case userconfig.AsyncAPIKind:
+		return asyncDescribeAPITable(apiRes, env)
+	default:
+		return "", errors.ErrorUnexpected(fmt.Sprintf("encountered unexpected kind %s for api %s", apiRes.Spec.Kind, apiRes.Spec.Name))
+	}
+}

cli/cmd/get.go

@@ -35,29 +35,28 @@ import (
 	libtime "github.com/cortexlabs/cortex/pkg/lib/time"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
+	"github.com/cortexlabs/yaml"
 	"github.com/spf13/cobra"
 )
 
 const (
 	_titleEnvironment = "env"
 	_titleRealtimeAPI = "realtime api"
-	_titleStatus      = "status"
+	_titleAsyncAPI    = "async api"
+	_titleLive        = "live"
 	_titleUpToDate    = "up-to-date"
-	_titleStale       = "stale"
-	_titleRequested   = "requested"
-	_titleFailed      = "failed"
-	_titleLastupdated = "last update"
+	_titleLastUpdated = "last update"
 )
 
 var (
-	_flagGetEnv string
-	_flagWatch  bool
+	_flagGetEnv   string
+	_flagGetWatch bool
 )
 
 func getInit() {
 	_getCmd.Flags().SortFlags = false
 	_getCmd.Flags().StringVarP(&_flagGetEnv, "env", "e", "", "environment to use")
-	_getCmd.Flags().BoolVarP(&_flagWatch, "watch", "w", false, "re-run the command every 2 seconds")
+	_getCmd.Flags().BoolVarP(&_flagGetWatch, "watch", "w", false, "re-run the command every 2 seconds")
 	_getCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|")))
 	addVerboseFlag(_getCmd)
 }
@@ -90,7 +89,7 @@ var _getCmd = &cobra.Command{
 			telemetry.Event("cli.get")
 		}
 
-		rerun(func() (string, error) {
+		rerun(_flagGetWatch, func() (string, error) {
 			if len(args) == 1 {
 				env, err := ReadOrConfigureEnv(envName)
 				if err != nil {
@@ -106,7 +105,7 @@ var _getCmd = &cobra.Command{
 					return "", err
 				}
 
-				if _flagOutput == flags.JSONOutputType {
+				if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
 					return apiTable, nil
 				}
 
@@ -136,7 +135,7 @@ var _getCmd = &cobra.Command{
 				if err != nil {
 					return "", err
 				}
-				if _flagOutput == flags.JSONOutputType {
+				if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
 					return jobTable, nil
 				}
 
@@ -166,7 +165,7 @@ var _getCmd = &cobra.Command{
 						return "", err
 					}
 
-					if _flagOutput == flags.JSONOutputType {
+					if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
 						return apiTable, nil
 					}
 
@@ -221,7 +220,7 @@ func getAPIsInAllEnvironments() (string, error) {
 
 		if err == nil {
 			for _, api := range apisRes {
-				switch api.Spec.Kind {
+				switch api.Metadata.Kind {
 				case userconfig.BatchAPIKind:
 					allBatchAPIEnvs = append(allBatchAPIEnvs, env.Name)
 					allBatchAPIs = append(allBatchAPIs, api)
@@ -247,12 +246,16 @@ func getAPIsInAllEnvironments() (string, error) {
 		allAPIsOutput = append(allAPIsOutput, apisOutput)
 	}
 
+	var bytes []byte
 	if _flagOutput == flags.JSONOutputType {
-		bytes, err := libjson.Marshal(allAPIsOutput)
-		if err != nil {
-			return "", err
-		}
-
+		bytes, err = libjson.Marshal(allAPIsOutput)
+	} else if _flagOutput == flags.YAMLOutputType {
+		bytes, err = yaml.Marshal(allAPIsOutput)
+	}
+	if err != nil {
+		return "", err
+	}
+	if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
 		return string(bytes), nil
 	}
 
@@ -337,11 +340,16 @@ func getAPIsByEnv(env cliconfig.Environment) (string, error) {
 		return "", err
 	}
 
+	var bytes []byte
 	if _flagOutput == flags.JSONOutputType {
-		bytes, err := libjson.Marshal(apisRes)
-		if err != nil {
-			return "", err
-		}
+		bytes, err = libjson.Marshal(apisRes)
+	} else if _flagOutput == flags.YAMLOutputType {
+		bytes, err = yaml.Marshal(apisRes)
+	}
+	if err != nil {
+		return "", err
+	}
+	if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
 		return string(bytes), nil
 	}
 
@@ -457,16 +465,21 @@ func getAPI(env cliconfig.Environment, apiName string) (string, error) {
 		return "", err
 	}
 
+	var bytes []byte
 	if _flagOutput == flags.JSONOutputType {
-		bytes, err := libjson.Marshal(apisRes)
-		if err != nil {
-			return "", err
-		}
+		bytes, err = libjson.Marshal(apisRes)
+	} else if _flagOutput == flags.YAMLOutputType {
+		bytes, err = yaml.Marshal(apisRes)
+	}
+	if err != nil {
+		return "", err
+	}
+	if _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {
 		return string(bytes), nil
 	}
 
 	if len(apisRes) == 0 {
-		exit.Error(errors.ErrorUnexpected(fmt.Sprintf("unable to find API %s", apiName)))
+		exit.Error(errors.ErrorUnexpected(fmt.Sprintf("unable to find api %s", apiName)))
 	}
 
 	apiRes := apisRes[0]

cli/cmd/lib_apis.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2021 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"github.com/cortexlabs/cortex/pkg/lib/table"
+	"github.com/cortexlabs/cortex/pkg/types/status"
+)
+
+func replicaCountTable(counts *status.ReplicaCounts) table.Table {
+	var rows [][]interface{}
+	for _, replicaCountType := range status.ReplicaCountTypes {
+		count := counts.GetCountBy(replicaCountType)
+		canBeHiddenIfZero := false
+		switch replicaCountType {
+		case status.ReplicaCountFailed:
+			canBeHiddenIfZero = true
+		case status.ReplicaCountKilled:
+			canBeHiddenIfZero = true
+		case status.ReplicaCountKilledOOM:
+			canBeHiddenIfZero = true
+		case status.ReplicaCountErrImagePull:
+			canBeHiddenIfZero = true
+		case status.ReplicaCountUnknown:
+			canBeHiddenIfZero = true
+		case status.ReplicaCountStalled:
+			canBeHiddenIfZero = true
+		}
+		if count == 0 && canBeHiddenIfZero {
+			continue
+		}
+		rows = append(rows, []interface{}{
+			replicaCountType,
+			count,
+		})
+	}
+
+	return table.Table{
+		Headers: []table.Header{
+			{Title: _titleReplicaStatus, MinWidth: 32, MaxWidth: 32},
+			{Title: _titleReplicaCount},
+		},
+		Rows: rows,
+	}
+}