Description
Hardware:
Board: ESP32 DEVKIT V1
Core Installation version: 1.0.6
IDE name: PlatformIO
Peripherals: SD Card (SPI)
Partition Scheme: FAT32
Description:
Batch deletion of files is very slow due to having to rely on openNextFile().
Consider the script below which:
(1) populates the SD card with 500 2kb gibberish files when 'a' is pressed;
(2) removes 500 files when 'b' is pressed.
The result when pressing 'a', and then 'b' when that is finished, is:
...
496: Removed /TEST/724092.TXT 48172 ms
497: Removed /TEST/4778.TXT 48345 ms
498: Removed /TEST/669317.TXT 48518 ms
499: Removed /TEST/533569.TXT 48691 ms
Using openNextFile
Total time 78663 ms, 48691 ms of which were spent removing files.
Much of the time spent is not spent removing files but just opening them...
This was partly addressed around 2018-2019 in the discussion following issue #1394 (#1394). The patch provided by driftregion at the time (#1394 (comment)) significantly improves this process and it seems to have unfortunately fallen through the cracks, with none of these changes ever being implemented.
I have cloned the arduino-esp32 framework into /brunojoyal/arduino-esp32-sdfix in which I replaced the FS library with the one patched by driftregion. Putting
[env]
framework = arduino
platform_packages = framework-arduinoespressif32 @ https://www.github.com/brunojoyal/arduino-esp32-sdfix.git
in platformio.ini to use this framework, and uncommenting the #def SDFIX line in my test script, we get this result:
494: Removed /TEST/241878.TXT 48015 ms
495: Removed /TEST/332820.TXT 48189 ms
496: Removed /TEST/795255.TXT 48362 ms
497: Removed /TEST/348514.TXT 48536 ms
498: Removed /TEST/772893.TXT 48710 ms
499: Removed /TEST/188217.TXT 48884 ms
Using getNextFileName
Total time 49922 ms, 48884 ms of which were spent removing files.
Much better.
Question/Request: Would it be possible to include some or all of the changes made by driftregion to the FS library? Most importantly, the getNextFileName() method, or something similar.
Many thanks.
Sketch:
void loadGibberish(void *parameter);
void removeSomeGibberish(void *parameter);
#include <Arduino.h>
#include <SD.h>
#define SDFIX
void setup()
{
Serial.begin(115200);
while (!SD.begin())
{
Serial.println("Loading SD card...");
delay(200);
}
Serial.println("Card loaded.");
if (!SD.exists("/TEST"))
{
SD.mkdir("/TEST");
}
}
void loop()
{
if (Serial.available())
{
switch (Serial.read())
{
case 'a':
xTaskCreate(loadGibberish, "Load Gibberish", 10000, NULL, 1, NULL);
break;
case 'b':
xTaskCreate(removeSomeGibberish, "Remove Some Gibberish", 10000, NULL, 1, NULL);
break;
}
}
}
#define LEN 1000
int createCounter = 0;
char buf[100];
void loadGibberish(void *parameter) //create 513 2kb gibberish files.
{
while (true)
{
File newImportantFile;
for (int i = 0; i < 500; i++)
{
long timer = millis();
char filename[32];
sprintf(filename, "/TEST/%i.TXT", esp_random() % 999999); //random filename
newImportantFile = SD.open(filename, "w");
if (newImportantFile)
{
uint8_t gibberish[LEN];
for (int j = 0; j < 2; j++)
{
newImportantFile.write(gibberish, LEN);
}
newImportantFile.close();
long duration = millis() - timer;
sprintf(buf, "%i : %s %li ms", createCounter++, filename, duration);
Serial.println(buf);
}
else
vTaskDelete(NULL);
vTaskDelay(1);
}
vTaskDelete(NULL);
}
}
int deleteCounter = 0;
void removeSomeGibberish(void *parameter)
{
while (true)
{
long totaltime = millis();
long totalremovetime = 0;
File root = SD.open("/TEST");
for (int i = 0; i < 500; i++)
{
#ifdef SDFIX
char *filename = root.getNextFileName();
if (filename)
{
#else
File file = root.openNextFile();
if (file)
{
char filename[32];
memcpy(filename, file.name(), strlen(file.name()) + 1);
file.close();
#endif
long timer = millis();
if (!SD.remove(filename))
{
Serial.println("Problem. Aborting.");
}
sprintf(buf, "%i: Removed %s %li ms", deleteCounter++, filename, totalremovetime+=millis() - timer);
Serial.println(buf);
}
else
break;
vTaskDelay(1);
}
#ifdef SDFIX
Serial.println("Using getNextFileName");
#else
Serial.println("Using openNextFile");
#endif
sprintf(buf, "Total time %li ms, %li ms of which were spent removing files.", millis() - totaltime, totalremovetime);
Serial.println(buf);
root.close();
vTaskDelete(NULL);
}
}
void removeSomeGibberish2(void *parameter)
{
while (true)
{
for (int i = 0; i < 500; i++)
{
File root = SD.open("/TEST");
char *filename = root.getNextFileName();
if (filename)
{
long timer = millis();
if (!SD.remove(filename))
{
Serial.println("Problem. Aborting.");
}
sprintf(buf, "%i: Removed %s %li ms", deleteCounter++, filename, millis() - timer);
Serial.println(buf);
}
vTaskDelay(1);
}
vTaskDelete(NULL);
}
}