Merge branch 'master' into NVS_Flash_CI_test

Author: P-R-O-C-H-Y

.github/scripts/sketch_utils.sh

@@ -109,18 +109,30 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
         exit 1
     fi
 
+    # The directory that will hold all the artifcats (the build directory) is
+    # provided through:
+    #  1. An env variable called ARDUINO_BUILD_DIR.
+    #  2. Created at the sketch level as "build" in the case of a single
+    #     configuration test.
+    #  3. Created at the sketch level as "buildX" where X is the number
+    #     of configuration built in case of a multiconfiguration test.
+
     ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
-    if [ -z "$ARDUINO_BUILD_DIR" ]; then
-        build_dir="$sketchdir/build"
-    else
+    if [ -n "$ARDUINO_BUILD_DIR" ]; then
         build_dir="$ARDUINO_BUILD_DIR"
+    elif [ $len -eq 1 ]; then
+        build_dir="$sketchdir/build"
     fi
 
     mkdir -p "$ARDUINO_CACHE_DIR"
     for i in `seq 0 $(($len - 1))`
     do
-        rm -rf "$build_dir$i"
-        mkdir -p "$build_dir$i"
+        if [ $len -ne 1 ]; then
+          build_dir="$sketchdir/build$i"
+        fi
+        rm -rf $build_dir
+        mkdir -p $build_dir
+
         currfqbn=`echo $fqbn | jq -r --argjson i $i '.[$i]'`
         sketchname=$(basename $sketchdir)
         echo "Building $sketchname with FQBN=$currfqbn"
@@ -134,7 +146,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
             -hardware "$user_path/hardware" \
             -libraries "$user_path/libraries" \
             -build-cache "$ARDUINO_CACHE_DIR" \
-            -build-path "$build_dir$i" \
+            -build-path "$build_dir" \
             $xtra_opts "${sketchdir}/${sketchname}.ino"
     done
 }

.github/scripts/tests_run.sh

@@ -14,14 +14,24 @@ function run_test() {
         len=1
     fi
 
+    if [ $len -eq 1 ]; then
+      build_dir="tests/$sketchname/build"
+      report_file="tests/$sketchname/$sketchname.xml"
+    fi
+
     for i in `seq 0 $(($len - 1))`
     do
         echo "Running test: $sketchname -- Config: $i"
         if [ $erase_flash -eq 1 ]; then
             esptool.py -c $target erase_flash
         fi
 
-        pytest tests --build-dir tests/$sketchname/build$i -k test_$sketchname --junit-xml=tests/$sketchname/$sketchname$i.xml
+        if [ $len -ne 1 ]; then
+            build_dir="tests/$sketchname/build$i"
+            report_file="tests/$sketchname/$sketchname$i.xml"
+        fi
+
+        pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file
         result=$?
         if [ $result -ne 0 ]; then
             return $result

boards.txt

@@ -491,6 +491,10 @@ void HardwareSerial::setRxInvert(bool invert)
 // negative Pin value will keep it unmodified
 void HardwareSerial::setPins(int8_t rxPin, int8_t txPin, int8_t ctsPin, int8_t rtsPin)
 {
+    if(_uart == NULL) {
+        log_e("setPins() shall be called after begin() - nothing done");
+        return;
+    }
     uartSetPins(_uart, rxPin, txPin, ctsPin, rtsPin);
 }
 

cores/esp32/HardwareSerial.cpp

@@ -72,6 +72,7 @@ esp_err_t i2cInit(uint8_t i2c_num, int8_t sda, int8_t scl, uint32_t frequency){
     } else if(frequency > 1000000UL){
         frequency = 1000000UL;
     }
+    log_i("Initialising I2C Master: sda=%d scl=%d freq=%d", sda, scl, frequency);
 
     i2c_config_t conf = { };
     conf.mode = I2C_MODE_MASTER;

cores/esp32/esp32-hal-i2c.c

@@ -89,7 +89,7 @@ void ledcWrite(uint8_t chan, uint32_t duty)
     //Fixing if all bits in resolution is set = LEDC FULL ON
     uint32_t max_duty = (1 << channels_resolution[chan]) - 1;
 
-    if(duty == max_duty){
+    if((duty == max_duty) && (max_duty != 1)){
         duty = max_duty + 1;
     }
 
@@ -226,3 +226,7 @@ void analogWrite(uint8_t pin, int value) {
     ledcWrite(pin_to_channel[pin] - 1, value);
   }
 }
+
+int8_t analogGetChannel(uint8_t pin) {
+    return pin_to_channel[pin] - 1;
+}

cores/esp32/esp32-hal-ledc.c

@@ -238,6 +238,10 @@ uint32_t uartAvailableForWrite(uart_t* uart)
     }
     UART_MUTEX_LOCK();
     uint32_t available =  uart_ll_get_txfifo_len(UART_LL_GET_HW(uart->num));  
+    size_t txRingBufferAvailable = 0;
+    if (ESP_OK == uart_get_tx_buffer_free_size(uart->num, &txRingBufferAvailable)) {
+        available += txRingBufferAvailable; 
+    }
     UART_MUTEX_UNLOCK();
     return available;
 }

cores/esp32/esp32-hal-uart.c

@@ -92,6 +92,7 @@ void yield(void);
 #include "esp32-hal-cpu.h"
 
 void analogWrite(uint8_t pin, int value);
+int8_t analogGetChannel(uint8_t pin);
 
 //returns chip temperature in Celsius
 float temperatureRead();

cores/esp32/esp32-hal.h

@@ -1,3 +1,13 @@
 ##########################
 Frequently Asked Questions
 ##########################
+
+How to modify an sdkconfig option in Arduino?
+---------------------------------------------
+
+Arduino-esp32 project is based on ESP-IDF. While ESP-IDF supports configuration of various compile-time options (known as "Kconfig options" or "sdkconfig options") via a "menuconfig" tool, this feature is not available in Arduino IDE.
+
+To use the arduino-esp32 core with a modified sdkconfig option, you need to use ESP-IDF to compile Arduino libraries. Please see :doc:`esp-idf_component` and :doc:`lib_builder` for the two solutions available.
+
+Note that modifying ``sdkconfig`` or ``sdkconfig.h`` files found in the arduino-esp32 project tree **does not** result in changes to these options. This is because ESP-IDF libraries are included into the arduino-esp32 project tree as pre-built libraries.
+

docs/source/faq.rst

@@ -196,6 +196,14 @@ Events Run On
 
 This function is also used to select the core that runs the Arduino events. This is only valid if the target SoC has 2 cores.
 
+Erase All Flash Before Sketch Upload
+************************************
+
+This option selects the flash memory region to be erased before uploading the new sketch.
+
+* **Disabled** - Upload the sketch without erasing all flash contents. (Default)
+* **Enabled** - Erase all flash contents before uploading the sketch.
+
 Port
 ****
 

docs/source/guides/tools_menu.rst

@@ -12,6 +12,7 @@
 #include <esp_gattc_api.h>
 
 #include <map>
+#include <vector>
 
 #include "BLEAddress.h"
 #include "BLEScan.h"

libraries/BLE/src/BLEAdvertisedDevice.h

@@ -0,0 +1,17 @@
+#######################################
+# Syntax Coloring Map
+#######################################
+
+#######################################
+# Datatypes (KEYWORD1)
+#######################################
+
+#######################################
+# Methods and Functions (KEYWORD2)
+#######################################
+
+#######################################
+# Constants (LITERAL1)
+#######################################
+
+RGB_BUILTIN	LITERAL1

libraries/ESP32/keywords.txt

@@ -28,6 +28,9 @@
 
 #define ENCRYPTED_BLOCK_SIZE 16
 
+#define SPI_SECTORS_PER_BLOCK   16      // usually large erase block is 32k/64k
+#define SPI_FLASH_BLOCK_SIZE    (SPI_SECTORS_PER_BLOCK*SPI_FLASH_SEC_SIZE)
+
 class UpdateClass {
   public:
     typedef std::function<void(size_t, size_t)> THandlerFunction_Progress;
@@ -166,6 +169,7 @@ class UpdateClass {
     bool _verifyHeader(uint8_t data);
     bool _verifyEnd();
     bool _enablePartition(const esp_partition_t* partition);
+    bool _chkDataInBlock(const uint8_t *data, size_t len) const;    // check if block contains any data or is empty
 
 
     uint8_t _error;

libraries/Update/src/Update.h

@@ -203,14 +203,23 @@ bool UpdateClass::_writeBuffer(){
     if (!_progress && _progress_callback) {
         _progress_callback(0, _size);
     }
-    if(!ESP.partitionEraseRange(_partition, _progress, SPI_FLASH_SEC_SIZE)){
-        _abort(UPDATE_ERROR_ERASE);
-        return false;
+    size_t offset = _partition->address + _progress;
+    bool block_erase = (_size - _progress >= SPI_FLASH_BLOCK_SIZE) && (offset % SPI_FLASH_BLOCK_SIZE == 0);             // if it's the block boundary, than erase the whole block from here
+    bool part_head_sectors = _partition->address % SPI_FLASH_BLOCK_SIZE && offset < (_partition->address / SPI_FLASH_BLOCK_SIZE + 1) * SPI_FLASH_BLOCK_SIZE;    // sector belong to unaligned partition heading block
+    bool part_tail_sectors = offset >= (_partition->address + _size) / SPI_FLASH_BLOCK_SIZE * SPI_FLASH_BLOCK_SIZE;     // sector belong to unaligned partition tailing block
+    if (block_erase || part_head_sectors || part_tail_sectors){
+        if(!ESP.partitionEraseRange(_partition, _progress, block_erase ? SPI_FLASH_BLOCK_SIZE : SPI_FLASH_SEC_SIZE)){
+            _abort(UPDATE_ERROR_ERASE);
+            return false;
+        }
     }
-    if (!ESP.partitionWrite(_partition, _progress + skip, (uint32_t*)_buffer + skip/sizeof(uint32_t), _bufferLen - skip)) {
+
+    // try to skip empty blocks on unecrypted partitions
+    if ((_partition->encrypted || _chkDataInBlock(_buffer + skip/sizeof(uint32_t), _bufferLen - skip)) && !ESP.partitionWrite(_partition, _progress + skip, (uint32_t*)_buffer + skip/sizeof(uint32_t), _bufferLen - skip)) {
         _abort(UPDATE_ERROR_WRITE);
         return false;
     }
+
     //restore magic or md5 will fail
     if(!_progress && _command == U_FLASH){
         _buffer[0] = ESP_IMAGE_HEADER_MAGIC;
@@ -389,4 +398,20 @@ const char * UpdateClass::errorString(){
     return _err2str(_error);
 }
 
+bool UpdateClass::_chkDataInBlock(const uint8_t *data, size_t len) const {
+    // check 32-bit aligned blocks only
+    if (!len || len % sizeof(uint32_t))
+        return true;
+
+    size_t dwl = len / sizeof(uint32_t);
+
+    do {
+        if (*(uint32_t*)data ^ 0xffffffff)      // for SPI NOR flash empty blocks are all one's, i.e. filled with 0xff byte
+            return true;
+
+        data += sizeof(uint32_t);
+    } while (--dwl);
+    return false;
+}
+
 UpdateClass Update;

libraries/Update/src/Updater.cpp

@@ -527,7 +527,7 @@ void WebServer::sendContent_P(PGM_P content, size_t size) {
 }
 
 
-void WebServer::_streamFileCore(const size_t fileSize, const String & fileName, const String & contentType)
+void WebServer::_streamFileCore(const size_t fileSize, const String & fileName, const String & contentType, const int code)
 {
   using namespace mime;
   setContentLength(fileSize);
@@ -536,7 +536,7 @@ void WebServer::_streamFileCore(const size_t fileSize, const String & fileName,
       contentType != String(FPSTR(mimeTable[none].mimeType))) {
     sendHeader(F("Content-Encoding"), F("gzip"));
   }
-  send(200, contentType, "");
+  send(code, contentType, "");
 }
 
 String WebServer::pathArg(unsigned int i) {

libraries/WebServer/src/WebServer.cpp

@@ -139,8 +139,8 @@ class WebServer
   static String urlDecode(const String& text);
 
   template<typename T>
-  size_t streamFile(T &file, const String& contentType) {
-    _streamFileCore(file.size(), file.name(), contentType);
+  size_t streamFile(T &file, const String& contentType, const int code = 200) {
+    _streamFileCore(file.size(), file.name(), contentType, code);
     return _currentClient.write(file);
   }
 
@@ -160,7 +160,7 @@ class WebServer
   void _prepareHeader(String& response, int code, const char* content_type, size_t contentLength);
   bool _collectHeader(const char* headerName, const char* headerValue);
 
-  void _streamFileCore(const size_t fileSize, const String & fileName, const String & contentType);
+  void _streamFileCore(const size_t fileSize, const String & fileName, const String & contentType, const int code = 200);
 
   String _getRandomHexString();
   // for extracting Auth parameters

libraries/WebServer/src/WebServer.h

@@ -38,7 +38,7 @@ class WiFiAPClass
 public:
 
     bool softAP(const char* ssid, const char* passphrase = NULL, int channel = 1, int ssid_hidden = 0, int max_connection = 4, bool ftm_responder = false);
-    bool softAPConfig(IPAddress local_ip, IPAddress gateway, IPAddress subnet, IPAddress dhcp_lease_start = INADDR_NONE);
+    bool softAPConfig(IPAddress local_ip, IPAddress gateway, IPAddress subnet, IPAddress dhcp_lease_start = (uint32_t) 0);
     bool softAPdisconnect(bool wifioff = false);
 
     uint8_t softAPgetStationNum();

libraries/WiFi/src/WiFiAP.h

@@ -303,9 +303,14 @@ int WiFiClient::connect(const char *host, uint16_t port, int32_t timeout)
 
 int WiFiClient::setSocketOption(int option, char* value, size_t len)
 {
-    int res = setsockopt(fd(), SOL_SOCKET, option, value, len);
+    return setSocketOption(SOL_SOCKET, option, (const void*)value, len);
+}
+
+int WiFiClient::setSocketOption(int level, int option, const void* value, size_t len)
+{
+    int res = setsockopt(fd(), level, option, value, len);
     if(res < 0) {
-        log_e("%X : %d", option, errno);
+        log_e("fail on %d, errno: %d, \"%s\"", fd(), errno, strerror(errno));
     }
     return res;
 }
@@ -330,11 +335,7 @@ int WiFiClient::setTimeout(uint32_t seconds)
 
 int WiFiClient::setOption(int option, int *value)
 {
-    int res = setsockopt(fd(), IPPROTO_TCP, option, (char *) value, sizeof(int));
-    if(res < 0) {
-        log_e("fail on fd %d, errno: %d, \"%s\"", fd(), errno, strerror(errno));
-    }
-    return res;
+    return setSocketOption(IPPROTO_TCP, option, (const void*)value, sizeof(int));
 }
 
 int WiFiClient::getOption(int option, int *value)

libraries/WiFi/src/WiFiClient.cpp

@@ -87,6 +87,7 @@ class WiFiClient : public ESPLwIPClient
     int fd() const;
 
     int setSocketOption(int option, char* value, size_t len);
+    int setSocketOption(int level, int option, const void* value, size_t len);
     int setOption(int option, int *value);
     int getOption(int option, int *value);
     int setTimeout(uint32_t seconds);

libraries/WiFi/src/WiFiClient.h

@@ -495,7 +495,7 @@ static void _arduino_event_cb(void* arg, esp_event_base_t event_base, int32_t ev
     	log_v("SC Found Channel");
     	arduino_event.event_id = ARDUINO_EVENT_SC_FOUND_CHANNEL;
     } else if (event_base == SC_EVENT && event_id == SC_EVENT_GOT_SSID_PSWD) {
-        #if ARDUHAL_LOG_LEVEL >= ARDUHAL_LOG_LEVEL_ERROR
+        #if ARDUHAL_LOG_LEVEL >= ARDUHAL_LOG_LEVEL_VERBOSE
             smartconfig_event_got_ssid_pswd_t *event = (smartconfig_event_got_ssid_pswd_t *)event_data;
             log_v("SC: SSID: %s, Password: %s", (const char *)event->ssid, (const char *)event->password);
         #endif
@@ -668,7 +668,7 @@ bool wifiLowLevelInit(bool persistent){
 	    cfg.static_tx_buf_num = 0;
             cfg.dynamic_tx_buf_num = 32;
 	    cfg.tx_buf_type = 1;
-            cfg.cache_tx_buf_num = 1;  // can't be zero!
+            cfg.cache_tx_buf_num = 4;  // can't be zero!
 	    cfg.static_rx_buf_num = 4;
             cfg.dynamic_rx_buf_num = 32;
         }

libraries/WiFi/src/WiFiGeneric.cpp

@@ -378,9 +378,14 @@ int WiFiClientSecure::setTimeout(uint32_t seconds)
 }
 int WiFiClientSecure::setSocketOption(int option, char* value, size_t len)
 {
-    int res = setsockopt(sslclient->socket, SOL_SOCKET, option, value, len);
+    return setSocketOption(SOL_SOCKET, option, (const void*)value, len);
+}
+
+int WiFiClientSecure::setSocketOption(int level, int option, const void* value, size_t len)
+{
+    int res = setsockopt(sslclient->socket, level, option, value, len);
     if(res < 0) {
-        log_e("%X : %d", option, errno);
+        log_e("fail on %d, errno: %d, \"%s\"", sslclient->socket, errno, strerror(errno));
     }
     return res;
 }

libraries/WiFiClientSecure/src/WiFiClientSecure.cpp

@@ -81,6 +81,7 @@ class WiFiClientSecure : public WiFiClient
     bool getFingerprintSHA256(uint8_t sha256_result[32]) { return get_peer_fingerprint(sslclient, sha256_result); };
     int setTimeout(uint32_t seconds);
     int setSocketOption(int option, char* value, size_t len);
+    int setSocketOption(int level, int option, const void* value, size_t len);
 
     operator bool()
     {

libraries/WiFiClientSecure/src/WiFiClientSecure.h

@@ -21,7 +21,7 @@
 #include "WiFi.h"
 
 #if !defined(MBEDTLS_KEY_EXCHANGE__SOME__PSK_ENABLED) && !defined(MBEDTLS_KEY_EXCHANGE_SOME_PSK_ENABLED)
-#  warning "Please configure IDF framework to include mbedTLS -> Enable pre-shared-key ciphersuites and activate at least one cipher"
+#  warning "Please configure IDF framework to include mbedTLS -> Enable PSK based ciphersuite modes (MBEDTLS_KEY_EXCHANGE_PSK) and activate at least one cipher"
 #else
 
 const char *pers = "esp32-tls";