Merge branch 'master' of https://github.com/esp8266/Arduino into gcc9.1

Author: earlephilhower

cores/esp8266/Crypto.cpp

@@ -23,10 +23,9 @@
     THE SOFTWARE.
 */
 
-#include <bearssl/bearssl.h>
 #include "Crypto.h"
 #include <TypeConversion.h>
-
+#include <bearssl/bearssl.h>
 #include <assert.h>
 
 namespace TypeCast = experimental::TypeConversion;

cores/esp8266/Crypto.h

@@ -27,6 +27,7 @@
 #define __ESP8266_ARDUINO_CRYPTO_H__
 
 #include <Arduino.h>
+#include <bearssl/bearssl_kdf.h>
 
 namespace experimental
 {
@@ -745,8 +746,7 @@ struct HKDF
 
 private:
 
-    // Use an opaque type to avoid #include <bearssl/bearssl.h> which drags the lib declarations into userland. The global scope prefix is required for compilation to succeed, it seems.
-    ::br_hkdf_context hkdfContext;
+    br_hkdf_context hkdfContext;
 };
 
 

cores/esp8266/spiffs/README.md

@@ -49,6 +49,9 @@ What spiffs does not:
  - Presently, it does not detect or handle bad blocks.
  - One configuration, one binary. There's no generic spiffs binary that handles all types of configurations.
 
+## NOTICE
+
+0.4.0 is under construction. This is a full rewrite and will change the underlying structure. Hence, it will not be compatible with earlier versions of the filesystem. The API is the same, with minor modifications. Some config flags will be removed (as they are mandatory in 0.4.0) and some features might fall away until 0.4.1. If you have any worries or questions, it can be discussed in issue [#179](https://github.com/pellepl/spiffs/issues/179)
 
 ## MORE INFO 
 

cores/esp8266/spiffs/spiffs_check.cpp

@@ -163,11 +163,17 @@ static s32_t spiffs_delete_obj_lazy(spiffs *fs, spiffs_obj_id obj_id) {
     return SPIFFS_OK;
   }
   SPIFFS_CHECK_RES(res);
-  u8_t flags = 0xff & ~SPIFFS_PH_FLAG_IXDELE;
+  u8_t flags = 0xff;
+#if SPIFFS_NO_BLIND_WRITES
+  res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_READ,
+      0, SPIFFS_PAGE_TO_PADDR(fs, objix_hdr_pix) + offsetof(spiffs_page_header, flags),
+      sizeof(flags), &flags);
+  SPIFFS_CHECK_RES(res);
+#endif
+  flags &= ~SPIFFS_PH_FLAG_IXDELE;
   res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_UPDT,
       0, SPIFFS_PAGE_TO_PADDR(fs, objix_hdr_pix) + offsetof(spiffs_page_header, flags),
-      sizeof(u8_t),
-      (u8_t *)&flags);
+      sizeof(flags), &flags);
   return res;
 }
 
@@ -425,10 +431,17 @@ static s32_t spiffs_lookup_check_validate(spiffs *fs, spiffs_obj_id lu_obj_id, s
           // just finalize
           SPIFFS_CHECK_DBG("LU: FIXUP: unfinalized page is referred, finalizing\n");
           CHECK_CB(fs, SPIFFS_CHECK_LOOKUP, SPIFFS_CHECK_FIX_LOOKUP, p_hdr->obj_id, p_hdr->span_ix);
-          u8_t flags = 0xff & ~SPIFFS_PH_FLAG_FINAL;
+          u8_t flags = 0xff;
+#if SPIFFS_NO_BLIND_WRITES
+          res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_READ,
+              0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix) + offsetof(spiffs_page_header, flags),
+              sizeof(flags), &flags);
+          SPIFFS_CHECK_RES(res);
+#endif
+          flags &= ~SPIFFS_PH_FLAG_FINAL;
           res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_UPDT,
               0, SPIFFS_PAGE_TO_PADDR(fs, cur_pix) + offsetof(spiffs_page_header, flags),
-              sizeof(u8_t), (u8_t*)&flags);
+              sizeof(flags), &flags);
         }
       }
     }

cores/esp8266/spiffs/spiffs_config.h

@@ -326,6 +326,17 @@ typedef uint8_t u8_t;
 #define SPIFFS_IX_MAP                         1
 #endif
 
+// By default SPIFFS in some cases relies on the property of NOR flash that bits
+// cannot be set from 0 to 1 by writing and that controllers will ignore such
+// bit changes. This results in fewer reads as SPIFFS can in some cases perform
+// blind writes, with all bits set to 1 and only those it needs reset set to 0.
+// Most of the chips and controllers allow this behavior, so the default is to
+// use this technique. If your controller is one of the rare ones that don't,
+// turn this option on and SPIFFS will perform a read-modify-write instead.
+#ifndef SPIFFS_NO_BLIND_WRITES
+#define SPIFFS_NO_BLIND_WRITES                0
+#endif
+
 // Set SPIFFS_TEST_VISUALISATION to non-zero to enable SPIFFS_vis function
 // in the api. This function will visualize all filesystem using given printf
 // function.
@@ -354,11 +365,20 @@ typedef uint8_t u8_t;
 #endif
 #endif
 
+#ifndef SPIFFS_SECURE_ERASE
+#define SPIFFS_SECURE_ERASE 0
+#endif
+
 // Types depending on configuration such as the amount of flash bytes
 // given to spiffs file system in total (spiffs_file_system_size),
 // the logical block size (log_block_size), and the logical page size
 // (log_page_size)
+//
+// Set SPIFFS_TYPES_OVERRIDE if you wish to have your own
+// definitions for these types (for example, if you want them
+// to be u32_t)
 
+#ifndef SPIFFS_TYPES_OVERRIDE
 // Block index type. Make sure the size of this type can hold
 // the highest number of all blocks - i.e. spiffs_file_system_size / log_block_size
 typedef u16_t spiffs_block_ix;
@@ -373,5 +393,6 @@ typedef u16_t spiffs_obj_id;
 // hold the largest possible span index on the system -
 // i.e. (spiffs_file_system_size / log_page_size) - 1
 typedef u16_t spiffs_span_ix;
+#endif
 
 #endif /* SPIFFS_CONFIG_H_ */

cores/esp8266/spiffs/spiffs_nucleus.cpp

@@ -879,8 +879,6 @@ s32_t spiffs_page_delete(
     spiffs *fs,
     spiffs_page_ix pix) {
   s32_t res;
-  spiffs_page_header hdr;
-  hdr.flags = 0xff & ~(SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_USED);
   // mark deleted entry in source object lookup
   spiffs_obj_id d_obj_id = SPIFFS_OBJ_ID_DELETED;
   res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_LU | SPIFFS_OP_C_DELE,
@@ -893,12 +891,29 @@ s32_t spiffs_page_delete(
   fs->stats_p_deleted++;
   fs->stats_p_allocated--;
 
+#if SPIFFS_SECURE_ERASE
+  // Secure erase
+  unsigned char data[SPIFFS_CFG_LOG_PAGE_SZ(fs) - sizeof(spiffs_page_header)];
+  bzero(data, sizeof(data));
+  res = _spiffs_wr(fs,  SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_DELE,
+      0,
+      SPIFFS_PAGE_TO_PADDR(fs, pix) + sizeof(spiffs_page_header), sizeof(data), data);
+  SPIFFS_CHECK_RES(res);
+#endif
+
   // mark deleted in source page
+  u8_t flags = 0xff;
+#if SPIFFS_NO_BLIND_WRITES
+  res = _spiffs_rd(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_READ,
+      0, SPIFFS_PAGE_TO_PADDR(fs, pix) + offsetof(spiffs_page_header, flags),
+      sizeof(flags), &flags);
+  SPIFFS_CHECK_RES(res);
+#endif
+  flags &= ~(SPIFFS_PH_FLAG_DELET | SPIFFS_PH_FLAG_USED);
   res = _spiffs_wr(fs, SPIFFS_OP_T_OBJ_DA | SPIFFS_OP_C_DELE,
       0,
       SPIFFS_PAGE_TO_PADDR(fs, pix) + offsetof(spiffs_page_header, flags),
-      sizeof(u8_t),
-      (u8_t *)&hdr.flags);
+      sizeof(flags), &flags);
 
   return res;
 }
@@ -2027,7 +2042,7 @@ s32_t spiffs_object_read(
     // remaining data in page
     len_to_read = MIN(len_to_read, SPIFFS_DATA_PAGE_SIZE(fs) - (cur_offset % SPIFFS_DATA_PAGE_SIZE(fs)));
     // remaining data in file
-    len_to_read = MIN(len_to_read, fd->size);
+    len_to_read = MIN(len_to_read, fd->size - cur_offset);
     SPIFFS_DBG("read: offset:" _SPIPRIi " rd:" _SPIPRIi " data spix:" _SPIPRIsp " is data_pix:" _SPIPRIpg " addr:" _SPIPRIad "\n", cur_offset, len_to_read, data_spix, data_pix,
         (u32_t)(SPIFFS_PAGE_TO_PADDR(fs, data_pix) + sizeof(spiffs_page_header) + (cur_offset % SPIFFS_DATA_PAGE_SIZE(fs))));
     if (len_to_read <= 0) {

cores/esp8266/spiffs/spiffs_nucleus.h

@@ -147,8 +147,8 @@ extern "C" {
 
 
 
-#if defined(__GNUC__) || defined(__clang__)
-    /* For GCC and clang */
+#if defined(__GNUC__) || defined(__clang__) || defined(__TI_COMPILER_VERSION__)
+    /* For GCC, clang and TI compilers */
 #define SPIFFS_PACKED __attribute__((packed))
 #elif defined(__ICCARM__) || defined(__CC_ARM)
     /* For IAR ARM and Keil MDK-ARM compilers */
@@ -266,8 +266,8 @@ extern "C" {
 #define SPIFFS_FH_OFFS(fs, fh)   ((fh) != 0 ? ((fh) + (fs)->cfg.fh_ix_offset) : 0)
 #define SPIFFS_FH_UNOFFS(fs, fh) ((fh) != 0 ? ((fh) - (fs)->cfg.fh_ix_offset) : 0)
 #else
-#define SPIFFS_FH_OFFS(fs, fh)   (fh)
-#define SPIFFS_FH_UNOFFS(fs, fh) (fh)
+#define SPIFFS_FH_OFFS(fs, fh)   ((spiffs_file)(fh))
+#define SPIFFS_FH_UNOFFS(fs, fh) ((spiffs_file)(fh))
 #endif
 
 

cores/esp8266/umm_malloc/umm_local.h

@@ -48,7 +48,8 @@ void ICACHE_FLASH_ATTR print_stats(int force);
 
 
 int ICACHE_FLASH_ATTR umm_info_safe_printf_P(const char *fmt, ...) __attribute__((format(printf, 1, 2)));
-#define UMM_INFO_PRINTF(fmt, ...) umm_info_safe_printf_P(PSTR(fmt), ##__VA_ARGS__)
+#define UMM_INFO_PRINTF(fmt, ...) umm_info_safe_printf_P(PSTR4(fmt), ##__VA_ARGS__)
+// use PSTR4() instead of PSTR() to ensure 4-bytes alignment in Flash, whatever the default alignment of PSTR_ALIGN
 
 
 #endif

libraries/ESP8266HTTPClient/examples/ReuseConnectionV2/ReuseConnectionV2.ino

@@ -0,0 +1,82 @@
+/**
+   reuseConnectionV2.ino
+
+    Created on: 22.11.2015
+
+   This example reuses the http connection and also restores the connection if the connection is lost
+*/
+
+
+#include <ESP8266WiFi.h>
+#include <ESP8266WiFiMulti.h>
+#include <ESP8266HTTPClient.h>
+
+#ifndef STASSID
+#define STASSID "your-ssid"
+#define STAPSK  "your-password"
+#endif
+
+ESP8266WiFiMulti WiFiMulti;
+
+HTTPClient http;
+WiFiClient client;
+
+void setup() {
+
+  Serial.begin(115200);
+  // Serial.setDebugOutput(true);
+
+  Serial.println();
+  Serial.println();
+  Serial.println("Connecting to WiFi...");
+
+  WiFi.mode(WIFI_STA);
+  WiFiMulti.addAP(STASSID, STAPSK);
+
+  // wait for WiFi connection
+  while ((WiFiMulti.run() != WL_CONNECTED)) {
+    Serial.write('.');
+    delay(500);
+  }
+  Serial.println(" connected to WiFi");
+
+  // allow reuse (if server supports it)
+  http.setReuse(true);
+
+
+  http.begin(client, "http://jigsaw.w3.org/HTTP/connection.html");
+  //http.begin(client, "jigsaw.w3.org", 80, "/HTTP/connection.html");
+}
+
+int pass = 0;
+
+void loop() {
+  // First 10 loop()s, retrieve the URL
+  if (pass < 10) {
+    pass++;
+    Serial.printf("Reuse connection example, GET url for the %d time\n", pass);
+    int httpCode = http.GET();
+    if (httpCode > 0) {
+      Serial.printf("[HTTP] GET... code: %d\n", httpCode);
+
+      // file found at server
+      if (httpCode == HTTP_CODE_OK) {
+        http.writeToStream(&Serial);
+      }
+    } else {
+      Serial.printf("[HTTP] GET... failed, error: %s\n", http.errorToString(httpCode).c_str());
+      // Something went wrong with the connection, try to reconnect
+      http.end();
+      http.begin(client, "http://jigsaw.w3.org/HTTP/connection.html");
+      //http.begin(client, "jigsaw.w3.org", 80, "/HTTP/connection.html");
+    }
+
+    if (pass == 10) {
+      http.end();
+      Serial.println("Done testing");
+    } else {
+      Serial.println("\n\n\nWait 5 second...\n");
+      delay(5000);
+    }
+  }
+}

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp

@@ -457,6 +457,10 @@ void HTTPClient::disconnect(bool preserveClient)
 #endif
         }
     } else {
+        if (!preserveClient && _client) { // Also destroy _client if not connected()
+            _client = nullptr;
+        }
+
         DEBUG_HTTPCLIENT("[HTTP-Client][end] tcp is closed\n");
     }
 }
@@ -970,7 +974,9 @@ int HTTPClient::writeToStream(Stream * stream)
         return returnError(HTTPC_ERROR_NO_STREAM);
     }
 
-    if(!connected()) {
+    // Only return error if not connected and no data available, because otherwise ::getString() will return an error instead of an empty
+    // string when the server returned a http code 204 (no content)
+    if(!connected() && _transferEncoding != HTTPC_TE_IDENTITY && _size > 0) {
         return returnError(HTTPC_ERROR_NOT_CONNECTED);
     }
 
@@ -979,11 +985,13 @@ int HTTPClient::writeToStream(Stream * stream)
     int ret = 0;
 
     if(_transferEncoding == HTTPC_TE_IDENTITY) {
-        ret = writeToStreamDataBlock(stream, len);
+        if(len > 0) {
+            ret = writeToStreamDataBlock(stream, len);
 
-        // have we an error?
-        if(ret < 0) {
-            return returnError(ret);
+            // have we an error?
+            if(ret < 0) {
+                return returnError(ret);
+            }
         }
     } else if(_transferEncoding == HTTPC_TE_CHUNKED) {
         int size = 0;
@@ -1198,12 +1206,8 @@ bool HTTPClient::hasHeader(const char* name)
  */
 bool HTTPClient::connect(void)
 {
-    if(connected()) {
-        if(_reuse) {
-            DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n");
-        } else {
-            DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n");
-        }
+    if(_reuse && _canReuse && connected()) {
+        DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n");
         while(_client->available() > 0) {
             _client->read();
         }
@@ -1334,22 +1338,21 @@ int HTTPClient::handleHeaderResponse()
     while(connected()) {
         size_t len = _client->available();
         if(len > 0) {
+            int headerSeparator = -1;
             String headerLine = _client->readStringUntil('\n');
 
             lastDataTime = millis();
 
             DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] RX: '%s'\n", headerLine.c_str());
 
             if (headerLine.startsWith(F("HTTP/1."))) {
-                if (_canReuse) {
-                    _canReuse = (headerLine[sizeof "HTTP/1." - 1] != '0');
-                }
-                _returnCode = headerLine.substring(9, headerLine.indexOf(' ', 9)).toInt();
-                continue;
-            }
 
-            int headerSeparator = headerLine.indexOf(':');
-            if (headerSeparator > 0) {
+                constexpr auto httpVersionIdx = sizeof "HTTP/1." - 1;
+                _canReuse = _canReuse && (headerLine[httpVersionIdx] != '0');
+                _returnCode = headerLine.substring(httpVersionIdx + 2, headerLine.indexOf(' ', httpVersionIdx + 2)).toInt();
+                _canReuse = _canReuse && (_returnCode > 0) && (_returnCode < 500);
+
+            } else if ((headerSeparator = headerLine.indexOf(':')) > 0) {
                 String headerName = headerLine.substring(0, headerSeparator);
                 String headerValue = headerLine.substring(headerSeparator + 1);
                 headerValue.trim();