Skip to content

HTTPClient: implement Stream class #6977

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

HTTPClient: implement Stream class #6977

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
69a869c
HTTPClient: add readChunkHeader() and readChunkTrailer() methods
francescolavra Jan 2, 2020
eec286d
HTTPClient: implement Stream class
francescolavra Jan 4, 2020
a2d1356
Merge branch 'master' into master
earlephilhower Aug 18, 2020
19bf46f
Merge branch 'master' into master
d-a-v Jun 11, 2022
89a5f5e
fix merge by hand
d-a-v Jun 12, 2022
53669c0
fix merge by hand #2
d-a-v Jun 12, 2022
136b942
Merge branch 'master' into master
d-a-v Jun 13, 2022
d3908b0
Merge branch 'master' into master
d-a-v Dec 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
240 changes: 226 additions & 14 deletions libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ void HTTPClient::clear()
_size = -1;
_headers.clear();
_location.clear();
_chunkHeader.clear();
_chunkLen = 0;
_transferEncoding = HTTPC_TE_IDENTITY;
_payload.reset();
}

Expand Down Expand Up @@ -674,18 +677,11 @@ int HTTPClient::writeToPrint(Print * print)
if(!connected()) {
return returnError(HTTPC_ERROR_CONNECTION_LOST);
}
String chunkHeader = _client->readStringUntil('\n');

if(chunkHeader.length() <= 0) {
if(!readChunkHeader()) {
return returnError(HTTPC_ERROR_READ_TIMEOUT);
}

chunkHeader.trim(); // remove \r

// read size of chunk
len = (uint32_t) strtol((const char *) chunkHeader.c_str(), NULL, 16);
size += len;
DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", len);
size += _chunkLen;
DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", _chunkLen);

// data left?
if(len > 0) {
Expand All @@ -710,9 +706,7 @@ int HTTPClient::writeToPrint(Print * print)
}

// read trailing \r\n at the end of the chunk
char buf[2];
auto trailing_seq_len = _client->readBytes((uint8_t*)buf, 2);
if (trailing_seq_len != 2 || buf[0] != '\r' || buf[1] != '\n') {
if (!readChunkTrailer()) {
return returnError(HTTPC_ERROR_READ_TIMEOUT);
}

Expand Down Expand Up @@ -785,6 +779,142 @@ String HTTPClient::errorToString(int error)
}
}

/**
* write a byte to HTTP connection stream
* @param b uint8_t byte to be written
* @return number of bytes written
*/
size_t HTTPClient::write(uint8_t b)
{
return write(&b, sizeof(b));
}

/**
* write a byte array to HTTP connection stream
* @param buffer uint8_t * byte array to be written
* @param size size_t byte array size
* @return number of bytes written
*/
size_t HTTPClient::write(const uint8_t *buffer, size_t size)
{
if (!connected()) {
return 0;
}
if (_transferEncoding == HTTPC_TE_CHUNKED) {
int written;

if (size == 0) {
return size;
}
if ((_chunkLen != 0) && (_chunkOffset == _chunkLen)) {
if (_client->write("\r\n") == 2) {
_chunkLen = 0;
} else {
return 0;
}
}
if (_chunkLen == 0) {
String header = String(size, 16) + "\r\n";

if (_client->write(header.c_str()) != header.length()) {
return 0;
}
_chunkLen = size;
_chunkOffset = 0;
}
if (size > _chunkLen - _chunkOffset) {
size = _chunkLen - _chunkOffset;
}
written = _client->write(buffer, size);
_chunkOffset += written;
return written;
}
else {
return _client->write(buffer, size);
}
}

/**
* retrieve number of bytes available to be read from HTTP connection stream
* @return number of available bytes
*/
int HTTPClient::available()
{
if (!connected()) {
return 0;
}
if (_transferEncoding == HTTPC_TE_IDENTITY) {
return _client->available();
} else if(_transferEncoding == HTTPC_TE_CHUNKED) {
if ((_chunkLen == 0) && !readChunkHeader(false)) {
return 0;
}
if (_chunkLen > 0) {
if (_chunkOffset < _chunkLen) {
unsigned int available = (unsigned int) _client->available();
if (available < _chunkLen - _chunkOffset) {
return available;
} else {
return (_chunkLen - _chunkOffset);
}
} else {
readChunkTrailer(false);
}
}
return 0;
} else {
return 0;
}
}

/**
* read a byte from HTTP connection stream
* @return byte read, or -1 if no bytes could be read
*/
int HTTPClient::read()
{
if (!connected()) {
return -1;
}
if (_transferEncoding == HTTPC_TE_IDENTITY) {
return _client->read();
} else if(_transferEncoding == HTTPC_TE_CHUNKED) {
while (true) {
if ((_chunkLen == 0) && !readChunkHeader()) {
return -1;
}
if (_chunkLen > 0) {
if (_chunkOffset < _chunkLen) {
int c = _client->read();

if (c >= 0) {
_chunkOffset++;
}
return c;
} else if (!readChunkTrailer()) {
return -1;
}
} else {
return -1;
}
}
} else {
return -1;
}
}

/**
* retrieve next byte available to be read from HTTP connection stream
* @return byte available to be read, or -1 if no bytes can be read
*/
int HTTPClient::peek()
{
if (!available()) {
return -1;
}
return _client->peek();
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

peek() is using available() to manage chunk-encoded streams.
Can read() do the same instead of duplicating code ?

Copy link
Author

@francescolavra francescolavra Aug 13, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think something like below would work for read():

while (!available()) {
    if (!connected()) {
        return -1;
    }
    yield();
}
if (_transferEncoding == HTTPC_TE_CHUNKED) {
    _chunkOffset++;
}
return _client->read();

But I'm afraid I don't have time to dig up my development environment and an ESP8266 to test this code.


/**
* adds Header to the request
* @param name
Expand Down Expand Up @@ -816,9 +946,14 @@ void HTTPClient::addHeader(const String& name, const String& value, bool first,
headerLine += "\r\n";
if (first) {
_headers = headerLine + _headers;
_transferEncoding = HTTPC_TE_IDENTITY;
} else {
_headers += headerLine;
}
if (name.equalsIgnoreCase(F("Transfer-Encoding")) &&
value.equalsIgnoreCase(F("chunked"))) {
_transferEncoding = HTTPC_TE_CHUNKED;
}
}
}

Expand Down Expand Up @@ -972,6 +1107,27 @@ bool HTTPClient::sendHeader(const char * type)
return StreamConstPtr(header).sendAll(_client.get()) == header.length();
}

/**
* sends HTTP request header
* @param type (GET, POST, ...)
* @return status
*/
bool HTTPClient::endRequest(void)
{
if (!connected()) {
return false;
}
if (_transferEncoding == HTTPC_TE_CHUNKED) {
if ((_chunkLen != 0) && (_client->write("\r\n") != 2)) {
return false;
}
return (_client->write("0\r\n\r\n") == 5);
}
else {
return true;
}
}

/**
* reads the response from the server
* @return int http code
Expand All @@ -989,7 +1145,6 @@ int HTTPClient::handleHeaderResponse()

String transferEncoding;

_transferEncoding = HTTPC_TE_IDENTITY;
unsigned long lastDataTime = millis();

while(connected()) {
Expand Down Expand Up @@ -1103,3 +1258,60 @@ int HTTPClient::returnError(int error)
}
return error;
}


/**
* Read header of next chunk in HTTP response using chunked encoding
* @param blocking bool whether this method is allowed to block
* @return boolean value indicating whether a complete header could be read
*/
bool HTTPClient::readChunkHeader(bool blocking)
{
if (blocking) {
_chunkHeader += _client->readStringUntil('\n');
if (_chunkHeader.length() == 0) {
return false;
}
} else {
while (_client->available() && !_chunkHeader.endsWith("\n")) {
_chunkHeader += (char) _client->read();
}
if (!_chunkHeader.endsWith("\n")) {
return false;
}
}

// read size of chunk
_chunkLen = (uint32_t) strtol((const char *) _chunkHeader.c_str(), NULL,
16);

_chunkOffset = 0;
return true;
}

/**
* Read trailer of current chunk in HTTP response using chunked encoding
* @param blocking bool whether this method is allowed to block
* @return boolean value indicating whether the complete trailer could be read
*/
bool HTTPClient::readChunkTrailer(bool blocking)
{
uint8_t buf[2];

if (blocking || (_client->available() >= 2)) {
auto trailing_seq_len = _client->readBytes((uint8_t *) buf, 2);

if (trailing_seq_len != 2 || buf[0] != '\r' || buf[1] != '\n') {
return false;
} else {
/* Clear _chunkHeader to indicate that the current chunk has been
* completely read, and reset _chunkLen to indicate that the next
* chunk header has not been read yet. */
_chunkHeader.clear();
_chunkLen = 0;
return true;
}
} else {
return false;
}
}
17 changes: 15 additions & 2 deletions libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,9 @@ typedef enum {
class TransportTraits;
typedef std::unique_ptr<TransportTraits> TransportTraitsPtr;

class HTTPClient
class StreamString;

class HTTPClient : public Stream
{
public:
HTTPClient() = default;
Expand Down Expand Up @@ -220,6 +222,12 @@ class HTTPClient
const String& getString(void);
static String errorToString(int error);

size_t write(uint8_t b);
size_t write(const uint8_t *buffer, size_t size);
int available();
int read();
int peek();

protected:
struct RequestArgument {
String key;
Expand All @@ -232,8 +240,10 @@ class HTTPClient
int returnError(int error);
bool connect(void);
bool sendHeader(const char * type);
bool endRequest(void);
int handleHeaderResponse();
int writeToStreamDataBlock(Stream * stream, int len);
bool readChunkHeader(bool blocking = true);
bool readChunkTrailer(bool blocking = true);

// The common pattern to use the class is to
// {
Expand Down Expand Up @@ -271,6 +281,9 @@ class HTTPClient
uint16_t _redirectLimit = 10;
String _location;
transferEncoding_t _transferEncoding = HTTPC_TE_IDENTITY;
String _chunkHeader;
uint32_t _chunkLen = 0;
uint32_t _chunkOffset = 0;
std::unique_ptr<StreamString> _payload;
};

Expand Down