More consistent uses of Maxcompute environment variables (#725)

* More consistent uses of Maxcompute environment variables

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

* Also change it in elasticdl template

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

Author: terrytangyuan

cmd/sqlflowserver/main_test.go

@@ -311,7 +311,7 @@ func TestEnd2EndMaxCompute(t *testing.T) {
 	if err != nil {
 		t.Fatalf("prepare test dataset failed: %v", err)
 	}
-	caseDB = "gomaxcompute_driver_w7u"
+	caseDB = os.Getenv("MAXCOMPUTE_PROJECT")
 	caseTrainTable = "sqlflow_test_iris_train"
 	caseTestTable = "sqlflow_test_iris_test"
 	casePredictTable = "sqlflow_test_iris_predict"

scripts/test_maxcompute.sh

@@ -14,6 +14,7 @@
 
 export SQLFLOW_TEST_DB=maxcompute
 export MAXCOMPUTE_ENDPOINT="service.cn.maxcompute.aliyun.com/api?curr_project=gomaxcompute_driver_w7u&scheme=https"
+export MAXCOMPUTE_PROJECT="gomaxcompute_driver_w7u"
 if [ "$MAXCOMPUTE_AK" = "" ] || [ "$MAXCOMPUTE_SK" == "" ]; then
   echo "skip maxcompute test because the env MAXCOMPUTE_AK or MAXCOMPUTE_SK is empty"
   exit 0

sql/sql_test.go

@@ -73,15 +73,16 @@ func testHiveDatabase() *DB {
 
 func testMaxcompute() *DB {
 	cfg := &gomaxcompute.Config{
-		AccessID:  os.Getenv("ODPS_ACCESS_ID"),
-		AccessKey: os.Getenv("ODPS_ACCESS_KEY"),
-		Project:   os.Getenv("ODPS_PROJECT"),
-		Endpoint:  os.Getenv("ODPS_ENDPOINT"),
+		AccessID:  os.Getenv("MAXCOMPUTE_AK"),
+		AccessKey: os.Getenv("MAXCOMPUTE_SK"),
+		Project:   os.Getenv("MAXCOMPUTE_PROJECT"),
+		Endpoint:  os.Getenv("MAXCOMPUTE_ENDPOINT"),
 	}
 
 	db, e := NewDB(fmt.Sprintf("maxcompute://%s", cfg.FormatDSN()))
 	assertNoErr(e)
-	// TODO(weiguo): Popularize
+	// Note: We do not popularize the test data here intentionally since
+	// it will take up quite some time on Maxcompute.
 	return db
 }
 

sql/template_elasticdl.go

@@ -16,7 +16,6 @@ package sql
 const elasticdlDataConversionTemplateText = `
 import os
 
-from elasticdl.python.common.constants import ODPSConfig
 from elasticdl.python.common.odps_io import ODPSReader
 from elasticdl.python.common.odps_recordio_conversion_utils import (
     write_recordio_shards_from_iterator,
@@ -26,10 +25,10 @@ from elasticdl.python.common.odps_recordio_conversion_utils import (
 COLUMN_NAMES = {{.FeaturesList}}
 
 reader = ODPSReader(
-    os.environ[ODPSConfig.PROJECT_NAME],
-    os.environ[ODPSConfig.ACCESS_ID],
-    os.environ[ODPSConfig.ACCESS_KEY],
-    os.environ[ODPSConfig.ENDPOINT],
+    os.environ["MAXCOMPUTE_PROJECT"],
+    os.environ["MAXCOMPUTE_AK"],
+    os.environ["MAXCOMPUTE_SK"],
+    os.environ["MAXCOMPUTE_ENDPOINT"],
     table = "{{.ODPSTableName}}",
     partition = None,
     num_processes = {{.NumProcesses}},
@@ -140,17 +139,17 @@ class PredictionOutputsProcessor(BasePredictionOutputsProcessor):
         if all(
             k in os.environ
             for k in (
-                ODPSConfig.PROJECT_NAME,
-                ODPSConfig.ACCESS_ID,
-                ODPSConfig.ACCESS_KEY,
-                ODPSConfig.ENDPOINT,
+                "MAXCOMPUTE_PROJECT",
+                "MAXCOMPUTE_AK",
+                "MAXCOMPUTE_SK",
+                "MAXCOMPUTE_ENDPOINT",
             )
         ):
             self.odps_writer = ODPSWriter(
-                os.environ[ODPSConfig.PROJECT_NAME],
-                os.environ[ODPSConfig.ACCESS_ID],
-                os.environ[ODPSConfig.ACCESS_KEY],
-                os.environ[ODPSConfig.ENDPOINT],
+                os.environ["MAXCOMPUTE_PROJECT"],
+                os.environ["MAXCOMPUTE_AK"],
+                os.environ["MAXCOMPUTE_SK"],
+                os.environ["MAXCOMPUTE_ENDPOINT"],
                 table = "{{.PredictOutputTable}}",
                 columns=["pred_" + str(i) for i in range({{.OutputShape}})],
                 column_types=["double" for _ in range({{.OutputShape}})],