add support for environment variables in device tests

Previously device tests included information such as access point SSID/password at compile time. This made it difficult to compile test binaries once and then send them to multiple test runners for execution.

This change adds a command to the test library to set environment variable on the target device: “setenv key value”. C library setenv/getenv facility is used to store variables.

Test runner, tests, and makefile are updated to use this functionality.

Author: igrr

tests/device/.gitignore

@@ -2,3 +2,4 @@
 .hardware
 test_report.xml
 test_report.html
+test_env.cfg

tests/device/Makefile

@@ -12,17 +12,10 @@ BS_DIR ?= libraries/BSTest
 DEBUG_LEVEL ?= DebugLevel=None____
 FQBN ?= esp8266com:esp8266:generic:CpuFrequency=80,FlashFreq=40,FlashMode=dio,UploadSpeed=115200,FlashSize=4M1M,LwIPVariant=v2mss536,ResetMethod=none,Debug=Serial,$(DEBUG_LEVEL)
 BUILD_TOOL := $(ARDUINO_IDE_PATH)/arduino-builder
-TEST_CONFIG := libraries/test_config/test_config.h
+TEST_CONFIG := test_env.cfg
 TEST_REPORT_XML := test_report.xml
 TEST_REPORT_HTML := test_report.html
 
-ifeq ("$(UPLOAD_PORT)","")
-$(error "Failed to detect upload port, please export UPLOAD_PORT manually")
-endif
-
-ifeq ("$(ARDUINO_IDE_PATH)","")
-$(error "Please export ARDUINO_IDE_PATH")
-endif
 
 ifneq ("$(V)","1")
 	SILENT = @
@@ -35,16 +28,17 @@ endif
 
 all: count tests test_report
 
-count:
-	@echo Running $(words $(TEST_LIST)) tests
+$(TEST_LIST): | virtualenv $(TEST_CONFIG) $(BUILD_DIR) $(HARDWARE_DIR)
 
-tests: $(BUILD_DIR) $(HARDWARE_DIR) virtualenv $(TEST_CONFIG) $(TEST_LIST)
+tests: $(TEST_LIST)
 
 $(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@)
 
 $(TEST_LIST):
+	@echo Running $(words $(TEST_LIST)) tests
 	$(SILENT)mkdir -p $(LOCAL_BUILD_DIR)
 ifneq ("$(NO_BUILD)","1")
+	@test -n "$(ARDUINO_IDE_PATH)" || (echo "Please export ARDUINO_IDE_PATH" && exit 1)
 	@echo Compiling $(notdir $@)
 	$(SILENT)$(BUILD_TOOL) -compile -logger=human \
 		-libraries "$(PWD)/libraries" \
@@ -59,6 +53,7 @@ ifneq ("$(NO_BUILD)","1")
 		$@
 endif
 ifneq ("$(NO_UPLOAD)","1")
+	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
 	@echo Uploading binary
 	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG)  \
 		-cp $(UPLOAD_PORT) \
@@ -67,6 +62,7 @@ ifneq ("$(NO_UPLOAD)","1")
 		-cf $(LOCAL_BUILD_DIR)/$(notdir $@).bin
 endif
 ifneq ("$(NO_RUN)","1")
+	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
 	@echo Running tests
 	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) -cp $(UPLOAD_PORT) -cd $(UPLOAD_BOARD) -cr
 	@source $(BS_DIR)/virtualenv/bin/activate && \
@@ -75,6 +71,7 @@ ifneq ("$(NO_RUN)","1")
 			-p $(UPLOAD_PORT) \
 			-n $(basename $(notdir $@)) \
 			-o $(LOCAL_BUILD_DIR)/test_result.xml \
+			--env-file $(TEST_CONFIG) \
 			`test -f $(addsuffix .py, $(basename $@)) && echo "-m $(addsuffix .py, $(basename $@))" || echo ""`
 endif
 
@@ -95,7 +92,7 @@ $(HARDWARE_DIR):
 	cd $(HARDWARE_DIR)/esp8266com && ln -s $(realpath $(ESP8266_CORE_PATH)) esp8266
 
 virtualenv:
-	make -C $(BS_DIR) virtualenv
+	@make -C $(BS_DIR) virtualenv
 
 clean:
 	rm -rf $(BUILD_DIR)
@@ -104,9 +101,9 @@ clean:
 
 $(TEST_CONFIG):
 	@echo "******    "
-	@echo "******    libraries/test_config/test_config.h does not exist"
-	@echo "******    Create one from libraries/test_config/test_config.h.template"
+	@echo "******    $(TEST_CONFIG) does not exist"
+	@echo "******    Create one from $(TEST_CONFIG).template"
 	@echo "******    "
-	false
+	@false
 
-.PHONY: tests all count virtualenv test_report $(BUILD_DIR) $(TEST_LIST)
+.PHONY: tests all count virtualenv test_report $(TEST_LIST)

tests/device/libraries/BSTest/runner.py

@@ -9,6 +9,11 @@
 import serial
 import subprocess
 import imp
+try:
+    from configparser import ConfigParser
+except:
+    from ConfigParser import ConfigParser
+import itertools
 from urlparse import urlparse
 from junit_xml import TestSuite, TestCase
 try:
@@ -34,12 +39,13 @@ class BSTestRunner(object):
     CRASH = 3
     BEGINTIMEOUT = 4
 
-    def __init__(self, spawn_obj, name, mocks):
+    def __init__(self, spawn_obj, name, mocks, env_vars):
         self.sp = spawn_obj
         self.tests = []
         self.reset_timeout = 2
         self.name = name
         self.mocks = mocks
+        self.env_vars = env_vars
 
     def get_test_list(self):
         self.sp.sendline('-1')
@@ -73,6 +79,7 @@ def get_test_list(self):
 
     def run_tests(self):
         test_cases = []
+        should_update_env = True
         for test in self.tests:
             desc = test['desc']
             name = test['name']
@@ -84,13 +91,20 @@ def run_tests(self):
             else:
                 test_output = StringIO()
                 self.sp.logfile = test_output
+                print('running test "{}"'.format(name))
+                if should_update_env:
+                    res = self.update_env()
+                    if res != BSTestRunner.SUCCESS:
+                        print('failed to set environment variables')
+                        break;
+                    should_update_env = False
                 if name in self.mocks:
-                    print('setting up mocks')
+                    debug_print('setting up mocks')
                     self.mocks[name]['setup']()
                 t_start = time.time()
                 result = self.run_test(index)
                 if name in self.mocks:
-                    print('tearing down mocks')
+                    debug_print('tearing down mocks')
                     self.mocks[name]['teardown']()
                 t_stop = time.time()
                 self.sp.logfile = None
@@ -103,6 +117,7 @@ def run_tests(self):
                 else:
                     print('test "{}" failed'.format(name))
                     test_case.add_failure_info('Test failed', output=test_output.getvalue())
+                    should_update_env = True
                 test_output.close()
             test_cases += [test_case];
         return TestSuite(self.name, test_cases)
@@ -152,6 +167,22 @@ def run_test(self, index):
         if timeout <= 0:
             return BSTestRunner.TIMEOUT
 
+    def update_env(self):
+        for env_kv in self.env_vars:
+            self.sp.sendline('setenv "{}" "{}"'.format(env_kv[0], env_kv[1]))
+            timeout = 10
+            while timeout > 0:
+                res = self.sp.expect(['>>>>>bs_test_setenv ok', EOF, TIMEOUT])
+                if res == 0:
+                    break
+                time.sleep(0.1)
+                timeout -= 0.1
+            if res == 0:
+                continue
+            else:
+                return BSTestRunner.TIMEOUT
+        return BSTestRunner.SUCCESS
+
 ser = None
 
 def spawn_port(port_name, baudrate=115200):
@@ -162,8 +193,8 @@ def spawn_port(port_name, baudrate=115200):
 def spawn_exec(name):
     return pexpect.spawn(name, timeout=0)
 
-def run_tests(spawn, name, mocks):
-    tw = BSTestRunner(spawn, name, mocks)
+def run_tests(spawn, name, mocks, env_vars):
+    tw = BSTestRunner(spawn, name, mocks, env_vars)
     tw.get_test_list()
     return tw.run_tests()
 
@@ -175,6 +206,7 @@ def parse_args():
     parser.add_argument('-n', '--name', help='Test run name')
     parser.add_argument('-o', '--output', help='Output JUnit format test report')
     parser.add_argument('-m', '--mock', help='Set python script to use for mocking purposes')
+    parser.add_argument('--env-file', help='File containing a list of environment variables to set', type=argparse.FileType('r'))
     return parser.parse_args()
 
 def main():
@@ -194,12 +226,19 @@ def main():
     if spawn_func is None:
         debug_print("Please specify port or executable", file=sys.stderr)
         return 1
+    env_vars = []
+    if args.env_file is not None:
+        cfg = ConfigParser()
+        cfg.optionxform = str
+        with args.env_file as fp:
+            cfg.readfp(fp)
+        env_vars = cfg.items('global')
     mocks = {}
     if args.mock is not None:
         mocks_mod = imp.load_source('mocks', args.mock)
         mocks = mock_decorators.env
     with spawn_func(spawn_arg) as sp:
-        ts = run_tests(sp, name, mocks)
+        ts = run_tests(sp, name, mocks, env_vars)
         if args.output:
             with open(args.output, "w") as f:
                 TestSuite.to_file(f, [ts])

tests/device/libraries/BSTest/src/BSArduino.h

@@ -35,11 +35,27 @@ class ArduinoIOHelper
         return len;
     }
 
-    bool read_int(int& result)
+    size_t read_line(char* dest, size_t dest_size)
     {
-        // TODO: fix this for 0 value
-        result = m_stream.parseInt();
-        return result != 0;
+        size_t len = 0;
+        // Can't use Stream::readBytesUntil here because it can't tell the
+        // difference between timing out and receiving the terminator.
+        while (len < dest_size - 1) {
+            int c = m_stream.read();
+            if (c < 0) {
+                delay(1);
+                continue;
+            }
+            if (c == '\r') {
+                continue;
+            }
+            if (c == '\n') {
+                dest[len] = 0;
+                break;
+            }
+            dest[len++] = c;
+        }
+        return len;
     }
 
 protected: