我正在通过HTTP 1.1通过以太网连接通过HTTPS进行GET请求(这里不涉及WiFi)。
我的代码如下。
对于非常小的文件(25B)，文件下载成功。但是当文件大小增加(20K或2MB)时不下载，连接很快关闭。
那么，我的问题是:我怎样才能下载一个"large"文件吗?
我的代码在ESP32上运行。
#include <EthernetUdp.h>
#include <EthernetLarge.h>
#include <SSLClient.h>
// Choose the analog pin to get semi-random data from for SSL
// Pick a pin that's not connected or attached to a randomish voltage source
const int rand_pin = A5;
// Initialize the SSL client library
// We input an EthernetClient, our trust anchors, and the analog pin
EthernetClient base_client;
SSLClient client(base_client, TAs, (size_t)TAs_NUM, rand_pin);
void NanoEthernet::httpsGet(
char const *hostname, char const *path,
byte *buffer, size_t bufferCapacity,
size_t &bodyLength, bool &bufferOverflows,
void (*downloadChunkCallback)(byte * buffer, size_t bufferReadCount, size_t contentLength)
) {
log("Time is: %d %d %d %02d:%02d:%02d", day(), month(), year(), hour(), minute(), second());
char const * subModule = "httpsGet: ";
log("%sbegin %s%s…",subModule, hostname, path);
// === if you get a connection, report back via serial:
if (!client.connect(hostname, 443)) {
// if you didn't get a connection to the server:
error("%sconnection failed", subModule);
return false;
}
success("%sConnected!",subModule);
// Make a HTTP request:
client.printf("GET %s HTTP/1.1", path); client.println();
client.printf("Host: %s", hostname); client.println();
// client.println("Connection: keep-alive"); client.println();
// client.println("Keep-Alive: timeout=5, max=100"); client.println();
//client.println("Connection: close");
client.println();
//while(client.connected() && !client.available()) delay(1); //waits for data
// success("%sSent query headers (connected:%d)",
// subModule,
// client.connected()
// );
/// Body counter
int bodyCounter = 0;
/// Chunk counter
int chunkReadCount = 0;
/// Whether to read response header?
bool readResponseHeader = true;
byte *p = buffer;
int contentLength = 0;
const unsigned timeoutNoData = 10*1000; // 10 seconds
unsigned long long lastTimeNoData = millis();
unsigned waitForConnection = 10;
while (true) {
// log("%sResponse: connected: %d | readResponseHeader: %d",
// subModule,
// client.connected(),
// readResponseHeader
// );
if (!client.connected()) {
delay(1);
error("%sNo more connected, aborting", subModule);
if (waitForConnection-- <= 0)
continue;
else
break;
}
if (!client.available()) {
delay(1);
// == Still reading the header then continue, otherwise hanle timeout
if (!readResponseHeader) {
if (millis() - lastTimeNoData >= timeoutNoData) {
if (contentLength==0) {
log("%sNo more data are available after %d seconds timeout",
subModule,
timeoutNoData/1000
);
}
else {
error("%sNo more data are available timeout", subModule);
}
// == Stopping the connection
break;
}
}
//log(">.");
continue;
}
lastTimeNoData = millis();
// = Skip response header
if (readResponseHeader) {
const String header = client.readStringUntil('n'); // End of line is "rn"
log("%sResponse headers: %s", subModule, header.c_str());
const int nextData = client.peek();
// End of stream?
if (nextData==-1) {
error("%sEnd of stream (received data -1) while reading headers");
break;
}
const char nextChar = char(nextData);
//log("Response headers: => next char is %d 0x%02x (%c)", nextChar, nextChar, nextChar);
// == End of headers? (empty line)
if (nextChar == 'r') {
// Actually read the end of line
client.read(); // eat r
client.read(); // eat n
//log("End of response header, starting to read response body…");
readResponseHeader=false;
}
else {
char const *headerString = "Content-Length: ";
const int headerStringLength = 16;
if (header.startsWith(headerString)) {
const String sContentLength = header.substring(headerStringLength);
contentLength = header.substring(headerStringLength).toInt();
//log("Read Content-Length: string: '%s'| int: %d bytes", sContentLength.c_str(), byteCountToRead);
}
}
continue;
}
const int data = client.read(); //gets byte from ethernet buffer
// == End of stream?
if (data==-1) {
log("%sEnd of stream (data received: -1)",subModule);
break;
}
// == Keep the actual byte
const byte b = byte(data);
#if LOG_LEVEL >= LOG_LEVEL_DEBUG
//log(">%02x (%d)", b, bodyCounter);
log(">%02x ", b);
#endif
*p++ = b;
bodyCounter++;
chunkReadCount++;
// = When no chunk callback, overflow means stop reading
if (downloadChunkCallback==NULL) {
// == If buffer overflows, stop
if (bodyCounter >= bufferCapacity) {
bufferOverflows = true;
break;
}
}
// = But when chunk callback has been provided, pass it the
// chunk, rewind the buffer and continue
else {
// == If buffer overflows, stop
if (chunkReadCount >= bufferCapacity) {
downloadChunkCallback(buffer, chunkReadCount, contentLength);
// Now rewind the chunk to the satrt of the buffer prior to
// continue reading the reponsse body
chunkReadCount = 0;
p = buffer;
}
}
// == We read everything?
if (contentLength>0) {
if (bodyCounter>=contentLength) {
log("%sEnd of stream (received ontentLength bytes)", subModule);
break;
}
}
}
// = When chunk callback, pass it the potential remaining chunk
if(downloadChunkCallback!=NULL
&& chunkReadCount>0) {
downloadChunkCallback(buffer, chunkReadCount, contentLength);
}
// == Write the length we read
bodyLength = bodyCounter;
#if LOG_LEVEL >= LOG_LEVEL_DEBUG
log("");
log("%s: did the get and read %d bytes", subModule, bodyCounter);
#endif
client.stop();
return true;
}
我终于找到解决办法了。
关键是，当然，我没有2MB的空闲内存(实际上ESP32是一个320KB的RAM微控制器)，所以关键是要求服务器发送一系列非常小的块，我们处理以连续的方式将它们附加到闪存上的文件。
我想到的第一个候选是"块传输编码"，但是，唉，这取决于服务器选择这样的模式，因为远程服务器实际上是Azure存储，没有提出块模式。
这就是为什么我回调到Range模式的原因:您只需在标头中指定Range: bytes: <start>-<end>来接收相应的数据位。
因此，我最终打开了HTTPS连接，然后，使用keep alive连接，我制作了一系列GET，为此我要求滑动数据窗口。
代码片段如下:
/// @brief Writes the GET on [path] and write mandatory constant headers
/// @param host The host name
/// @param path The path of he document to get
static void httpsGetWriteMandatoryHeaders(char const *hostname, char const * path) {
client.printf("GET %s HTTP/1.1", path); client.println();
client.printf("Host: %s", hostname); client.println();
client.printf("Accept-encoding: gzip, deflate"); client.println();
client.printf("Cache-control: no-cache, no-store, must-revalidate"); client.println();
}
bool NanoEthernet :: httpsGet(
char const *hostname, char const *path,
byte *buffer, size_t bufferCapacity,
size_t &bodyLength, bool &bufferOverflows,
void (*downloadChunkCallback)(byte * buffer, size_t bufferReadCount, size_t contentLength),
bool verbose
) {
log("Time is: %d %d %d %02d:%02d:%02d", day(), month(), year(), hour(), minute(), second());
char const * subModule = "httpsGet: ";
if (verbose) log("%sbegin %s%s…",subModule, hostname, path);
// === if you get a connection, report back via serial:
if (!client.connect(hostname, 443)) {
// if you didn't get a connection to the server:
error("%sconnection failed", subModule);
return false;
}
unsigned chunkIndex = 0;
success("%sConnected!",subModule);
// Make a HTTP request:
httpsGetWriteMandatoryHeaders(hostname, path);
client.printf("Range: bytes=%d-%d", chunkIndex*bufferCapacity, (chunkIndex+1)*bufferCapacity); client.println();
#if COMPUTE_CHECKSUM
// Reset checksum variable.
checksum32 = 0;
#endif
// Counter for each chunck
int messageCounter = 0;
/// Body counter
int bodyCounter = 0;
/// Chunk counter
int chunkReadCount = 0;
/// Whether to read response header?
bool readResponseHeader = true;
byte *p = buffer;
// Length of single chunck
int contentLength = 0;
// // Length of full body (File to transfer)
// int bodyLength = 0;
const unsigned timeoutNoData = 10*1000; // 10 seconds
unsigned long long lastTimeNoData = millis();
unsigned waitForConnection = 10;
while (true) {
if (!client.connected()) {
delay(1);
error("%sNo more connected, aborting", subModule);
if (waitForConnection-- <= 0)
continue;
else
break;
}
if (!client.available()) {
delay(1);
// == Still reading the header then continue, otherwise hanle timeout
if (!readResponseHeader) {
if (millis() - lastTimeNoData >= timeoutNoData) {
if (contentLength==0) {
if (verbose)
log("%sNo more data are available after %d seconds timeout",
subModule,
timeoutNoData/1000
);
}
else {
error("%sNo more data are available timeout", subModule);
}
// == Stopping the connection
break;
}
}
//log(">.");
continue;
}
lastTimeNoData = millis();
// = Skip response header
if (readResponseHeader) {
const String header = client.readStringUntil('n'); // End of line is "rn"
#if LOG_LEVEL >= LOG_LEVEL_DEBUG
if((chunkIndex%20)==0){
if (verbose) log("%sResponse headers: %s", subModule, header.c_str());
}
#endif
const int nextData = client.peek();
// End of stream?
if (nextData==-1) {
error("%sEnd of stream (received data -1) while reading headers");
break;
}
const char nextChar = char(nextData);
//log("Response headers: => next char is %d 0x%02x (%c)", nextChar, nextChar, nextChar);
// == End of headers? (empty line)
if (nextChar == 'r') {
// Actually read the end of line
client.read(); // eat r
client.read(); // eat n
//log("End of response header, starting to read response body…");
readResponseHeader=false;
}
else {
// == Repsonse code
//HTTP/1.1 404 Not Found
char const *headerString = "HTTP/1.1 ";
size_t headerStringLength = 9;
if (header.startsWith(headerString)) {
const String sCode = header.substring(headerStringLength);
const unsigned code = header.substring(headerStringLength).toInt();
switch (code/100) {
case 2:
if (verbose) success("Response code: %d", code);
continue;
// == All others are considered as errors
case 3: warn("Response code: code %d is not handled specifically", code);
default:
error("Response code not 2xx: code %d, extiting", code);
client.stop();
return false;
}
continue;
}
// == Content length
headerString = "Content-Length: ";
headerStringLength = 16;
if (header.startsWith(headerString)) {
const String sContentLength = header.substring(headerStringLength);
contentLength = header.substring(headerStringLength).toInt();
//log("Read Content-Length: string: '%s'| int: %d bytes", sContentLength.c_str(), byteCountToRead);
}
else if (header.startsWith("Content-Range:") && bodyLength==0){
const String content_size = header.substring(header.indexOf("/")+1);
bodyLength = content_size.toInt();
if (verbose) log("body has a length of: %d bytes",bodyLength);
}
}
continue;
}
const int data = client.read(); //gets byte from ethernet buffer
// == End of stream?
if (data==-1) {
if (verbose) log("%sEnd of stream (data received: -1)",subModule);
break;
}
// == Keep the actual byte
const byte b = byte(data);
#if LOG_LEVEL >= LOG_LEVEL_DEBUG
log(">%02x ", b);
#endif
*p++ = b;
bodyCounter++;
chunkReadCount++;
messageCounter++;
#if COMPUTE_CHECKSUM
checkSum32(data); // Updating checksum 32
#endif
// = When no chunk callback, overflow means stop reading
if (downloadChunkCallback==NULL) {
// == If buffer overflows, stop
if (bodyCounter >= bufferCapacity) {
bufferOverflows = true;
break;
}
}
// = But when chunk callback has been provided, pass it the
// chunk, rewind the buffer and continue
else {
// == If buffer overflows, stop
if (chunkReadCount >= bufferCapacity) {
downloadChunkCallback(buffer, chunkReadCount, bodyLength);
// Now rewind the chunk to the satrt of the buffer prior to
// continue reading the reponsse body
chunkReadCount = 0;
p = buffer;
}
}
// == We read everything?
if (contentLength>0) {
if (messageCounter>=contentLength) {
#if LOG_LEVEL >= LOG_LEVEL_DEBUG
log("%sEnd of stream (received contentLength bytes)", subModule);
#endif
client.flush();
/// Ask for the new chunk
if(chunkIndex*bufferCapacity<bodyLength){
++chunkIndex;
if (verbose) log("Asking for new chunk #%d of %d bytes max", chunkIndex, bufferCapacity);
httpsGetWriteMandatoryHeaders(hostname, path);
if ((chunkIndex+1)*bufferCapacity<bodyLength){
client.printf("Range: bytes=%d-%d", chunkIndex*bufferCapacity+1, (chunkIndex+1)*bufferCapacity); client.println();
}
else{
client.printf("Range: bytes=%d-%d", chunkIndex*bufferCapacity+1, bodyLength); client.println();
}
client.println();
readResponseHeader=true;
messageCounter = 0;
while (!client.available()&&client.connected()) delay(1);
}
else{
if (verbose) success("All the data is well received");
break;
}
}
}
}
// = When chunk callback, pass it the potential remaining chunk
if(downloadChunkCallback!=NULL
&& chunkReadCount>0) {
downloadChunkCallback(buffer, chunkReadCount, bodyLength);
}
// == Write the length we read
bodyLength = bodyCounter;
// #if LOG_LEVEL >= LOG_LEVEL_DEBUG
log("");
log("%s: did the get and read %d bytes", subModule, bodyCounter);
// #endif
client.stop();
#if COMPUTE_CHECKSUM
if (verbose) log("------------- The checksum32 is %d -----------------", checksum32);
#endif
return true;
}