Skip to content

Commit 961b558

Browse files
earlephilhowerd-a-v
earlephilhower
authored and
d-a-v
committed
Fix device test environment variables (#6229)
* Fix device test environment variables Device tests were not connecting properly to WiFi because the environment variables were not set when WiFi.connect was called. This would result in tests sometimes working *if* the prior sketch run on the ESP saved WiFi connection information and auto-connect was enabled. But, in most cases, the tests would simply never connect to any WiFi and fail. getenv() works only after BS_RUN is called (because BS_RUN handles the actual parsing of environment variables sent from the host). Add a "pretest" function to all tests which is called by the host test controller only after all environment variables are set. Move all WiFi/etc. operations that were in each separate test's setup() into it. So the order of operations for tests now is: ESP: setup() -> Set serial baud -> Call BS_RUN() HOST: Send environment Send "do pretest" ESP: pretest() -> Set Wifi using env. ariables, etc. return "true" on success HOST: Send "run test 1" ESP: Run 1st test, return result HOST: Send "run test 2" ESP: Run 2nd test, return result <and so forth> If nothing is needed to be set up, just return true from the pretest function. All tests now run and at least connect to WiFi. There still seem to be some actual test errors, but not because of the WiFi/environment variables anymore. * Remove unneeded debug prints * Silence esptool.py output when not in V=1 mode Esptool-ck.exe had an option to be silent, but esptool.py doesn't so the output is very chatty and makes looking a the run logs hard (60 lines of esptool.py output, 3 lines of actual test reports). Redirect esptool.py STDOUT to /dev/null unless V=1 to clear this up. * Speed up builds massively by removing old JSON arduino-builder checks the build.options.json file and then goes off and pegs my CPU at 100% for over a minute on each test compile checking if files have been modified. Simply deleting any pre-existing options.json file causes this step to be skipped and a quick, clean recompile is done in siginificantly less time. * Enable compile warnings, fix any that show up Enable all GCC warnings when building the tests and fix any that came up (mostly signed/unsigned, unused, and deprecated ones). * Fix UMM_MALLOC printf crash, umm_test Printf can now handle PROGMEM addresses, so simplify and correct the debug printouts in umm_info and elsewhere.
1 parent 7d8782a commit 961b558

File tree

28 files changed

+205
-47
lines changed

28 files changed

+205
-47
lines changed

‎cores/esp8266/umm_malloc/umm_malloc.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -520,7 +520,7 @@ extern int umm_last_fail_alloc_size;
520520
#endif
521521

522522
// Macro to place constant strings into PROGMEM and print them properly
523-
#define printf(fmt, ...) do { static const char fstr[] PROGMEM = fmt; char rstr[sizeof(fmt)]; for (size_t i=0; i<sizeof(rstr); i++) rstr[i] = fstr[i]; printf(rstr, ##__VA_ARGS__); } while (0)
523+
#define printf(fmt, ...) printf(PSTR(fmt), ## __VA_ARGS__ )
524524

525525
/* -- dbglog {{{ */
526526

‎tests/device/Makefile

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ TEST_REPORT_HTML := test_report.html
2323

2424
ifneq ("$(V)","1")
2525
SILENT = @
26+
REDIR = >& /dev/null
2627
else
2728
BUILDER_DEBUG_FLAG = -verbose
2829
RUNNER_DEBUG_FLAG = -d
@@ -50,10 +51,11 @@ $(TEST_LIST):
5051
ifneq ("$(NO_BUILD)","1")
5152
@test -n "$(ARDUINO_IDE_PATH)" || (echo "Please export ARDUINO_IDE_PATH" && exit 1)
5253
@echo Compiling $(notdir $@)
54+
@rm -f $(LOCAL_BUILD_DIR)/build.options.json
5355
$(SILENT)$(BUILD_TOOL) -compile -logger=human \
5456
-libraries "$(PWD)/libraries" \
5557
-core-api-version="10608" \
56-
-warnings=none \
58+
-warnings=all \
5759
$(BUILDER_DEBUG_FLAG) \
5860
-build-path $(LOCAL_BUILD_DIR) \
5961
-tools $(ARDUINO_IDE_PATH)/tools-builder \
@@ -64,24 +66,25 @@ ifneq ("$(NO_BUILD)","1")
6466
endif
6567
ifneq ("$(NO_UPLOAD)","1")
6668
@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
67-
@test -e $(dir $@)/make_spiffs.py && (echo "Generating and uploading SPIFFS" && \
68-
(cd $(dir $@) && $(PYTHON) ./make_spiffs.py) && \
69+
@test -e $(dir $@)/make_spiffs.py && ( \
70+
echo "Generating and uploading SPIFFS" && \
71+
(cd $(dir $@) && $(PYTHON) ./make_spiffs.py $(REDIR) ) && \
6972
$(MKSPIFFS) --create $(dir $@)data/ --size 0xFB000 \
70-
--block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img && \
73+
--block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img $(REDIR) && \
7174
$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \
7275
--chip esp8266 \
7376
--port $(UPLOAD_PORT) \
7477
--baud $(UPLOAD_BAUD) \
7578
--after no_reset \
76-
write_flash 0x300000 $(LOCAL_BUILD_DIR)/spiffs.img ) \
79+
write_flash 0x300000 $(LOCAL_BUILD_DIR)/spiffs.img $(REDIR) ) \
7780
|| (echo "No SPIFFS to upload")
7881
@echo Uploading binary
7982
$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \
8083
--chip esp8266 \
8184
--port $(UPLOAD_PORT) \
8285
--baud $(UPLOAD_BAUD) \
8386
--after no_reset \
84-
write_flash 0x0 $(LOCAL_BUILD_DIR)/$(notdir $@).bin # no reset
87+
write_flash 0x0 $(LOCAL_BUILD_DIR)/$(notdir $@).bin $(REDIR) # no reset
8588
endif
8689
ifneq ("$(NO_RUN)","1")
8790
@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
@@ -90,7 +93,7 @@ ifneq ("$(NO_RUN)","1")
9093
--chip esp8266 \
9194
--port $(UPLOAD_PORT) \
9295
--baud $(UPLOAD_BAUD) \
93-
read_flash_status # reset
96+
read_flash_status $(REDIR) # reset
9497
$(SILENT)source $(BS_DIR)/virtualenv/bin/activate && \
9598
$(PYTHON) $(BS_DIR)/runner.py \
9699
$(RUNNER_DEBUG_FLAG) \

‎tests/device/libraries/BSTest/runner.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ def run_tests(self):
104104
if res != BSTestRunner.SUCCESS:
105105
print('failed to set environment variables')
106106
break;
107+
res = self.pretest()
108+
if res != BSTestRunner.SUCCESS:
109+
print('failed to run pretest init')
110+
break;
107111
should_update_env = False
108112
if name in self.mocks:
109113
debug_print('setting up mocks')
@@ -198,6 +202,21 @@ def update_env(self, env_to_set):
198202
return BSTestRunner.TIMEOUT
199203
return BSTestRunner.SUCCESS
200204

205+
def pretest(self):
206+
# Environment now set up, call the pretest init (wifi connect, etc.)
207+
self.sp.sendline('pretest');
208+
timeout = 10
209+
while timeout > 0:
210+
res = self.sp.expect(['>>>>>bs_test_pretest result=1', EOF, TIMEOUT]) # Only expect a pass, abort testing if failure
211+
if res == 0:
212+
break
213+
time.sleep(0.1)
214+
timeout -= 0.1
215+
if res != 0:
216+
return BSTestRunner.TIMEOUT
217+
else:
218+
return BSTestRunner.SUCCESS
219+
201220
def request_env(self, key):
202221
self.sp.sendline('getenv "{}"'.format(key))
203222
timeout = 10

‎tests/device/libraries/BSTest/src/BSArgs.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ inline size_t split_args(char *line, char **argv, size_t argv_size)
6969
const int ESCAPE = '\\';
7070
const int SPACE = ' ';
7171
split_state_t state = SS_SPACE;
72-
int argc = 0;
72+
size_t argc = 0;
7373
char *next_arg_start = line;
7474
char *out_ptr = line;
7575
for (char *in_ptr = line; argc < argv_size - 1; ++in_ptr) {
@@ -148,4 +148,4 @@ inline size_t split_args(char *line, char **argv, size_t argv_size)
148148

149149
} // namespace protocol
150150

151-
#endif //BS_ARGS_H
151+
#endif //BS_ARGS_H

‎tests/device/libraries/BSTest/src/BSProtocol.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
#define BS_LINE_PREFIX ">>>>>bs_test_"
77

8+
extern bool pretest();
9+
810
namespace bs
911
{
1012
namespace protocol
@@ -58,6 +60,12 @@ void output_getenv_result(IO& io, const char* key, const char* value)
5860
io.printf(BS_LINE_PREFIX "getenv value=\"%s\"\n", value);
5961
}
6062

63+
template<typename IO>
64+
void output_pretest_result(IO& io, bool res)
65+
{
66+
io.printf(BS_LINE_PREFIX "pretest result=%d\n", res?1:0);
67+
}
68+
6169
template<typename IO>
6270
bool input_handle(IO& io, char* line_buf, size_t line_buf_size, int& test_num)
6371
{
@@ -87,6 +95,14 @@ bool input_handle(IO& io, char* line_buf, size_t line_buf_size, int& test_num)
8795
output_getenv_result(io, argv[1], (value != NULL) ? value : "");
8896
return false;
8997
}
98+
if (strcmp(argv[0], "pretest") == 0) {
99+
if (argc != 1) {
100+
return false;
101+
}
102+
bool res = ::pretest();
103+
output_pretest_result(io, res);
104+
return false;
105+
}
90106
/* not one of the commands, try to parse as test number */
91107
char* endptr;
92108
test_num = (int) strtol(argv[0], &endptr, 10);

‎tests/device/test_BearSSL/test_BearSSL.ino

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,11 @@ void setup()
3333
{
3434
Serial.begin(115200);
3535
Serial.setDebugOutput(true);
36+
BS_RUN(Serial);
37+
}
38+
39+
bool pretest()
40+
{
3641
WiFi.persistent(false);
3742
WiFi.mode(WIFI_STA);
3843
WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
@@ -45,22 +50,11 @@ void setup()
4550
Serial.printf("Number of CA certs read: %d\n", numCerts);
4651
if (numCerts == 0) {
4752
Serial.printf("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?\n");
48-
REQUIRE(1==0);
53+
return false;
4954
}
50-
BS_RUN(Serial);
51-
}
52-
53-
static void stopAll()
54-
{
55-
WiFiClient::stopAll();
56-
}
57-
58-
static void disconnectWiFI()
59-
{
60-
wifi_station_disconnect();
55+
return true;
6156
}
6257

63-
6458
// Set time via NTP, as required for x.509 validation
6559
void setClock() {
6660
configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov");
@@ -143,7 +137,7 @@ int run(const char *str)
143137
return maxUsage;
144138
}
145139

146-
#define TC(x) TEST_CASE("BearSSL - Maximum stack usage < 5600 bytes @ "x".badssl.org", "[bearssl]") { REQUIRE(run(x) < 5600); }
140+
#define TC(x) TEST_CASE("BearSSL - Maximum stack usage < 5600 bytes @ " x ".badssl.org", "[bearssl]") { REQUIRE(run(x) < 5600); }
147141

148142
TC("expired")
149143
TC("wrong.host")

‎tests/device/test_ClientContext/test_ClientContext.ino

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,18 @@ void setup()
2323
{
2424
Serial.begin(115200);
2525
Serial.setDebugOutput(true);
26+
BS_RUN(Serial);
27+
}
28+
29+
bool pretest()
30+
{
2631
WiFi.persistent(false);
2732
WiFi.mode(WIFI_STA);
2833
WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
2934
while (WiFi.status() != WL_CONNECTED) {
3035
delay(500);
3136
}
32-
BS_RUN(Serial);
37+
return true;
3338
}
3439

3540
TEST_CASE("WiFi release ClientContext", "[clientcontext]")

‎tests/device/test_FS/test_FS.ino

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ void setup()
1010
BS_RUN(Serial);
1111
}
1212

13+
bool pretest()
14+
{
15+
return true;
16+
}
1317

1418

1519
TEST_CASE("read-write test","[fs]")

‎tests/device/test_Print_printf/test_Print_printf.ino

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,18 @@ void setup()
99
BS_RUN(Serial);
1010
}
1111

12+
bool pretest()
13+
{
14+
return true;
15+
}
16+
1217
TEST_CASE("Print::printf works for any reasonable output length", "[Print]")
1318
{
1419

1520
auto test_printf = [](size_t size) {
1621
StreamString str;
1722
auto buf = new char[size + 1];
18-
for (int i = 0; i < size; ++i) {
23+
for (size_t i = 0; i < size; ++i) {
1924
buf[i] = 'a';
2025
}
2126
buf[size] = 0;

‎tests/device/test_WiFiClient/test_WiFiClient.ino

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,18 @@ void setup()
1313
{
1414
Serial.begin(115200);
1515
Serial.setDebugOutput(true);
16+
BS_RUN(Serial);
17+
}
18+
19+
bool pretest()
20+
{
1621
WiFi.persistent(false);
1722
WiFi.mode(WIFI_STA);
1823
WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
1924
while (WiFi.status() != WL_CONNECTED) {
2025
delay(500);
2126
}
22-
BS_RUN(Serial);
27+
return true;
2328
}
2429

2530
static void stopAll()
@@ -53,4 +58,4 @@ TEST_CASE("WiFi disconnect during WiFiClient::connect", "[wificlient]")
5358

5459
void loop()
5560
{
56-
}
61+
}

‎tests/device/test_WiFiServer/test_WiFiServer.ino

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,18 @@ BS_ENV_DECLARE();
1010
void setup()
1111
{
1212
Serial.begin(115200);
13+
BS_RUN(Serial);
14+
}
15+
16+
bool pretest()
17+
{
1318
WiFi.persistent(false);
1419
WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
1520
while (WiFi.status() != WL_CONNECTED) {
1621
delay(500);
1722
}
1823
MDNS.begin("esp8266-wfs-test");
19-
BS_RUN(Serial);
24+
return true;
2025
}
2126

2227
TEST_CASE("Simple echo server", "[WiFiServer]")

‎tests/device/test_WiFi_events/test_WiFi_events.ino

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,14 @@ void setup()
1111
{
1212
Serial.begin(115200);
1313
Serial.setDebugOutput(false);
14+
BS_RUN(Serial);
15+
}
16+
17+
bool pretest()
18+
{
1419
WiFi.persistent(false);
1520
WiFi.mode(WIFI_OFF);
16-
BS_RUN(Serial);
21+
return true;
1722
}
1823

1924
static std::map<WiFiEvent_t, int> sEventsReceived;
@@ -29,10 +34,13 @@ TEST_CASE("WiFi.onEvent is called for specific events", "[wifi][events]")
2934
sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] = 0;
3035
sEventsReceived[WIFI_EVENT_STAMODE_GOT_IP] = 0;
3136

37+
#pragma GCC diagnostic push
38+
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
3239
WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_CONNECTED);
3340
WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_DISCONNECTED);
3441
WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_GOT_IP);
3542
WiFi.onEvent(onWiFiEvent, WIFI_EVENT_ANY);
43+
#pragma GCC diagnostic pop
3644

3745
WiFi.mode(WIFI_STA);
3846
WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
@@ -44,24 +52,30 @@ TEST_CASE("WiFi.onEvent is called for specific events", "[wifi][events]")
4452
WiFi.disconnect();
4553
delay(100);
4654
WiFi.mode(WIFI_OFF);
55+
#pragma GCC diagnostic push
56+
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
4757
REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_CONNECTED] == 2);
4858
REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] >= 2 && sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] % 2 == 0);
4959
REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_GOT_IP] == 2);
60+
#pragma GCC diagnostic pop
5061
}
5162

5263
TEST_CASE("STA mode events are called both when using DHCP and static config", "[wifi][events]")
5364
{
5465
String events;
5566

5667
auto handler1 = WiFi.onStationModeConnected([&](const WiFiEventStationModeConnected& evt){
68+
(void) evt;
5769
events += "connected,";
5870
});
5971
auto handler2 = WiFi.onStationModeDisconnected([&](const WiFiEventStationModeDisconnected& evt){
72+
(void) evt;
6073
if (events.length()) {
6174
events += "disconnected,";
6275
}
6376
});
6477
auto handler3 = WiFi.onStationModeGotIP([&](const WiFiEventStationModeGotIP& evt){
78+
(void) evt;
6579
events += "got_ip,";
6680
});
6781

@@ -104,12 +118,15 @@ TEST_CASE("Events are not called if handler is deleted", "[wifi][events]")
104118
String events;
105119

106120
WiFi.onStationModeConnected([&](const WiFiEventStationModeConnected& evt){
121+
(void) evt;
107122
events += "connected,";
108123
});
109124
WiFi.onStationModeDisconnected([&](const WiFiEventStationModeDisconnected& evt){
125+
(void) evt;
110126
events += "disconnected,";
111127
});
112128
WiFi.onStationModeGotIP([&](const WiFiEventStationModeGotIP& evt){
129+
(void) evt;
113130
events += "got_ip,";
114131
});
115132

0 commit comments

Comments
 (0)