From 69a869c82efa8580dffbd07780880106bc4c4260 Mon Sep 17 00:00:00 2001 From: Francesco Lavra Date: Thu, 2 Jan 2020 20:53:51 +0100 Subject: [PATCH 1/4] HTTPClient: add readChunkHeader() and readChunkTrailer() methods These new methods will be reused in the next commit when an implementation of the Stream class will be added to the HTTPClient class. HTTPClient::writeToStream() is being refactored to use the new methods. --- .../src/ESP8266HTTPClient.cpp | 78 +++++++++++++++---- .../ESP8266HTTPClient/src/ESP8266HTTPClient.h | 4 + 2 files changed, 67 insertions(+), 15 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 422dd3a0c0..87685176c5 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -139,6 +139,8 @@ void HTTPClient::clear() _size = -1; _headers.clear(); _location.clear(); + _chunkHeader.clear(); + _chunkLen = 0; _payload.reset(); } @@ -956,22 +958,15 @@ int HTTPClient::writeToStream(Stream * stream) if(!connected()) { return returnError(HTTPC_ERROR_CONNECTION_LOST); } - String chunkHeader = _client->readStringUntil('\n'); - - if(chunkHeader.length() <= 0) { + if(!readChunkHeader()) { return returnError(HTTPC_ERROR_READ_TIMEOUT); } - - chunkHeader.trim(); // remove \r - - // read size of chunk - len = (uint32_t) strtol((const char *) chunkHeader.c_str(), NULL, 16); - size += len; - DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", len); + size += _chunkLen; + DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", _chunkLen); // data left? - if(len > 0) { - int r = writeToStreamDataBlock(stream, len); + if(_chunkLen > 0) { + int r = writeToStreamDataBlock(stream, _chunkLen); if(r < 0) { // error in writeToStreamDataBlock return returnError(r); @@ -992,9 +987,7 @@ int HTTPClient::writeToStream(Stream * stream) } // read trailing \r\n at the end of the chunk - char buf[2]; - auto trailing_seq_len = _client->readBytes((uint8_t*)buf, 2); - if (trailing_seq_len != 2 || buf[0] != '\r' || buf[1] != '\n') { + if (!readChunkTrailer()) { return returnError(HTTPC_ERROR_READ_TIMEOUT); } @@ -1496,6 +1489,61 @@ int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) return bytesWritten; } +/** + * Read header of next chunk in HTTP response using chunked encoding + * @param blocking bool whether this method is allowed to block + * @return boolean value indicating whether a complete header could be read + */ +bool HTTPClient::readChunkHeader(bool blocking) +{ + if (blocking) { + _chunkHeader += _client->readStringUntil('\n'); + if (_chunkHeader.length() == 0) { + return false; + } + } else { + while (_client->available() && !_chunkHeader.endsWith("\n")) { + _chunkHeader += (char) _client->read(); + } + if (!_chunkHeader.endsWith("\n")) { + return false; + } + } + + // read size of chunk + _chunkLen = (uint32_t) strtol((const char *) _chunkHeader.c_str(), NULL, + 16); + + return true; +} + +/** + * Read trailer of current chunk in HTTP response using chunked encoding + * @param blocking bool whether this method is allowed to block + * @return boolean value indicating whether the complete trailer could be read + */ +bool HTTPClient::readChunkTrailer(bool blocking) +{ + uint8_t buf[2]; + + if (blocking || (_client->available() >= 2)) { + auto trailing_seq_len = _client->readBytes((uint8_t *) buf, 2); + + if (trailing_seq_len != 2 || buf[0] != '\r' || buf[1] != '\n') { + return false; + } else { + /* Clear _chunkHeader to indicate that the current chunk has been + * completely read, and reset _chunkLen to indicate that the next + * chunk header has not been read yet. */ + _chunkHeader.clear(); + _chunkLen = 0; + return true; + } + } else { + return false; + } +} + /** * called to handle error return, may disconnect the connection if still exists * @param error diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 1d7548f0f7..15d569ed95 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -225,6 +225,8 @@ class HTTPClient int handleHeaderResponse(); int writeToStreamDataBlock(Stream * stream, int len); + bool readChunkHeader(bool blocking = true); + bool readChunkTrailer(bool blocking = true); #if HTTPCLIENT_1_1_COMPATIBLE TransportTraitsPtr _transportTraits; @@ -257,6 +259,8 @@ class HTTPClient uint16_t _redirectLimit = 10; String _location; transferEncoding_t _transferEncoding = HTTPC_TE_IDENTITY; + String _chunkHeader; + uint32_t _chunkLen = 0; std::unique_ptr _payload; }; From eec286dbfab907cc0c154e593f5c718564fabed2 Mon Sep 17 00:00:00 2001 From: Francesco Lavra Date: Sat, 4 Jan 2020 17:00:50 +0100 Subject: [PATCH 2/4] HTTPClient: implement Stream class It is now possible to stream HTTP request data without knowing in advance the data size (using chunked transfer encoding), and to stream HTTP response data (correctly decoded according to the transfer encoding method used by the server) without requiring an output Stream implementation. (The existing getStream() method in the HTTPClient class returns the underlying WiFiClient stream which is not aware of the HTTP transfer encoding.) When using chunked encoding in an HTTP request, each call to write() results in the transmission of a separate chunk. The setting of _transferEncoding to HTTPC_TE_IDENTITY in the handleHeaderResponse() method has been removed because _transferEncoding is now used for both transmission and reception, and is reset to HTTPC_TE_IDENTITY in the clear() method. --- .../src/ESP8266HTTPClient.cpp | 165 +++++++++++++++++- .../ESP8266HTTPClient/src/ESP8266HTTPClient.h | 10 +- 2 files changed, 173 insertions(+), 2 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 87685176c5..b3b2afa355 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -141,6 +141,7 @@ void HTTPClient::clear() _location.clear(); _chunkHeader.clear(); _chunkLen = 0; + _transferEncoding = HTTPC_TE_IDENTITY; _payload.reset(); } @@ -1060,6 +1061,142 @@ String HTTPClient::errorToString(int error) } } +/** + * write a byte to HTTP connection stream + * @param b uint8_t byte to be written + * @return number of bytes written + */ +size_t HTTPClient::write(uint8_t b) +{ + return write(&b, sizeof(b)); +} + +/** + * write a byte array to HTTP connection stream + * @param buffer uint8_t * byte array to be written + * @param size size_t byte array size + * @return number of bytes written + */ +size_t HTTPClient::write(const uint8_t *buffer, size_t size) +{ + if (!connected()) { + return 0; + } + if (_transferEncoding == HTTPC_TE_CHUNKED) { + int written; + + if (size == 0) { + return size; + } + if ((_chunkLen != 0) && (_chunkOffset == _chunkLen)) { + if (_client->write("\r\n") == 2) { + _chunkLen = 0; + } else { + return 0; + } + } + if (_chunkLen == 0) { + String header = String(size, 16) + "\r\n"; + + if (_client->write(header.c_str()) != header.length()) { + return 0; + } + _chunkLen = size; + _chunkOffset = 0; + } + if (size > _chunkLen - _chunkOffset) { + size = _chunkLen - _chunkOffset; + } + written = _client->write(buffer, size); + _chunkOffset += written; + return written; + } + else { + return _client->write(buffer, size); + } +} + +/** + * retrieve number of bytes available to be read from HTTP connection stream + * @return number of available bytes + */ +int HTTPClient::available() +{ + if (!connected()) { + return 0; + } + if (_transferEncoding == HTTPC_TE_IDENTITY) { + return _client->available(); + } else if(_transferEncoding == HTTPC_TE_CHUNKED) { + if ((_chunkLen == 0) && !readChunkHeader(false)) { + return 0; + } + if (_chunkLen > 0) { + if (_chunkOffset < _chunkLen) { + unsigned int available = (unsigned int) _client->available(); + if (available < _chunkLen - _chunkOffset) { + return available; + } else { + return (_chunkLen - _chunkOffset); + } + } else { + readChunkTrailer(false); + } + } + return 0; + } else { + return 0; + } +} + +/** + * read a byte from HTTP connection stream + * @return byte read, or -1 if no bytes could be read + */ +int HTTPClient::read() +{ + if (!connected()) { + return -1; + } + if (_transferEncoding == HTTPC_TE_IDENTITY) { + return _client->read(); + } else if(_transferEncoding == HTTPC_TE_CHUNKED) { + while (true) { + if ((_chunkLen == 0) && !readChunkHeader()) { + return -1; + } + if (_chunkLen > 0) { + if (_chunkOffset < _chunkLen) { + int c = _client->read(); + + if (c >= 0) { + _chunkOffset++; + } + return c; + } else if (!readChunkTrailer()) { + return -1; + } + } else { + return -1; + } + } + } else { + return -1; + } +} + +/** + * retrieve next byte available to be read from HTTP connection stream + * @return byte available to be read, or -1 if no bytes can be read + */ +int HTTPClient::peek() +{ + if (!available()) { + return -1; + } + return _client->peek(); +} + /** * adds Header to the request * @param name @@ -1091,9 +1228,14 @@ void HTTPClient::addHeader(const String& name, const String& value, bool first, headerLine += "\r\n"; if (first) { _headers = headerLine + _headers; + _transferEncoding = HTTPC_TE_IDENTITY; } else { _headers += headerLine; } + if (name.equalsIgnoreCase(F("Transfer-Encoding")) && + value.equalsIgnoreCase(F("chunked"))) { + _transferEncoding = HTTPC_TE_CHUNKED; + } } } @@ -1269,6 +1411,27 @@ bool HTTPClient::sendHeader(const char * type) return (_client->write((const uint8_t *) header.c_str(), header.length()) == header.length()); } +/** + * sends HTTP request header + * @param type (GET, POST, ...) + * @return status + */ +bool HTTPClient::endRequest(void) +{ + if (!connected()) { + return false; + } + if (_transferEncoding == HTTPC_TE_CHUNKED) { + if ((_chunkLen != 0) && (_client->write("\r\n") != 2)) { + return false; + } + return (_client->write("0\r\n\r\n") == 5); + } + else { + return true; + } +} + /** * reads the response from the server * @return int http code @@ -1286,7 +1449,6 @@ int HTTPClient::handleHeaderResponse() String transferEncoding; - _transferEncoding = HTTPC_TE_IDENTITY; unsigned long lastDataTime = millis(); while(connected()) { @@ -1514,6 +1676,7 @@ bool HTTPClient::readChunkHeader(bool blocking) _chunkLen = (uint32_t) strtol((const char *) _chunkHeader.c_str(), NULL, 16); + _chunkOffset = 0; return true; } diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 15d569ed95..05073c1a38 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -137,7 +137,7 @@ typedef std::unique_ptr TransportTraitsPtr; class StreamString; -class HTTPClient +class HTTPClient : public Stream { public: HTTPClient(); @@ -210,6 +210,12 @@ class HTTPClient const String& getString(void); static String errorToString(int error); + size_t write(uint8_t b); + size_t write(const uint8_t *buffer, size_t size); + int available(); + int read(); + int peek(); + protected: struct RequestArgument { String key; @@ -222,6 +228,7 @@ class HTTPClient int returnError(int error); bool connect(void); bool sendHeader(const char * type); + bool endRequest(void); int handleHeaderResponse(); int writeToStreamDataBlock(Stream * stream, int len); @@ -261,6 +268,7 @@ class HTTPClient transferEncoding_t _transferEncoding = HTTPC_TE_IDENTITY; String _chunkHeader; uint32_t _chunkLen = 0; + uint32_t _chunkOffset = 0; std::unique_ptr _payload; }; From 89a5f5e721ed9193ad74bd2a37e37e82c508846f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 12 Jun 2022 22:44:14 +0200 Subject: [PATCH 3/4] fix merge by hand --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 810bbaafae..a7b1110c70 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -242,7 +242,8 @@ class HTTPClient : public Stream bool sendHeader(const char * type); bool endRequest(void); int handleHeaderResponse(); - int writeToStreamDataBlock(Stream * stream, int len); + bool readChunkHeader(bool blocking = true); + bool readChunkTrailer(bool blocking = true); // The common pattern to use the class is to // { From 53669c095b635bbf71438d9c342ee8484d04b621 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 12 Jun 2022 23:03:26 +0200 Subject: [PATCH 4/4] fix merge by hand #2 --- .../src/ESP8266HTTPClient.cpp | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 59225fdf4e..4fd336c4db 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -1258,3 +1258,60 @@ int HTTPClient::returnError(int error) } return error; } + + +/** + * Read header of next chunk in HTTP response using chunked encoding + * @param blocking bool whether this method is allowed to block + * @return boolean value indicating whether a complete header could be read + */ +bool HTTPClient::readChunkHeader(bool blocking) +{ + if (blocking) { + _chunkHeader += _client->readStringUntil('\n'); + if (_chunkHeader.length() == 0) { + return false; + } + } else { + while (_client->available() && !_chunkHeader.endsWith("\n")) { + _chunkHeader += (char) _client->read(); + } + if (!_chunkHeader.endsWith("\n")) { + return false; + } + } + + // read size of chunk + _chunkLen = (uint32_t) strtol((const char *) _chunkHeader.c_str(), NULL, + 16); + + _chunkOffset = 0; + return true; +} + +/** + * Read trailer of current chunk in HTTP response using chunked encoding + * @param blocking bool whether this method is allowed to block + * @return boolean value indicating whether the complete trailer could be read + */ +bool HTTPClient::readChunkTrailer(bool blocking) +{ + uint8_t buf[2]; + + if (blocking || (_client->available() >= 2)) { + auto trailing_seq_len = _client->readBytes((uint8_t *) buf, 2); + + if (trailing_seq_len != 2 || buf[0] != '\r' || buf[1] != '\n') { + return false; + } else { + /* Clear _chunkHeader to indicate that the current chunk has been + * completely read, and reset _chunkLen to indicate that the next + * chunk header has not been read yet. */ + _chunkHeader.clear(); + _chunkLen = 0; + return true; + } + } else { + return false; + } +}