Skip to content

Commit

Permalink
Merge pull request #703 from MartinMueller2003/main
Browse files Browse the repository at this point in the history 
Fixed issue with uploading a long list of files
  • Loading branch information
@forkineye
forkineye authored Dec 8, 2023
2 parents 7b4418a + 5860330 commit dad96ac
Show file tree
Hide file tree
Showing 8 changed files with 85 additions and 37 deletions.
51 changes: 40 additions & 11 deletions ESPixelStick/src/FileMgr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -793,15 +793,19 @@ void c_FileMgr::DescribeSdCardToUser ()
} // DescribeSdCardToUser

//-----------------------------------------------------------------------------
void c_FileMgr::GetListOfSdFiles (String & Response)
void c_FileMgr::GetListOfSdFiles (String & Response, uint32_t FirstFileToSend)
{
// DEBUG_START;
#define EntryReservedSpace 75
#define EntryUsageFactor 3

DynamicJsonDocument ResponseJsonDoc (3 * 1024);
DynamicJsonDocument ResponseJsonDoc (1 * 1024);

do // once
{
if (0 == ResponseJsonDoc.capacity ())
uint32_t capacity = ResponseJsonDoc.capacity ();

if (0 == capacity)
{
logcon (String(CN_stars) + F ("ERROR: Failed to allocate memory for the GetListOfSdFiles web request response.") + CN_stars);
break;
Expand All @@ -820,6 +824,12 @@ void c_FileMgr::GetListOfSdFiles (String & Response)
ResponseJsonDoc[F ("totalBytes")] = ESP_SD.size64 ();
#endif
uint64_t usedBytes = 0;
uint32_t last = 0;

ResponseJsonDoc[F("usedBytes")] = uint32_t(-1);
ResponseJsonDoc[F("first")] = FirstFileToSend;
ResponseJsonDoc[F("last")] = uint32_t(-1);
ResponseJsonDoc[F("final")] = false;

File dir = ESP_SDFS.open ("/", CN_r);

Expand All @@ -829,44 +839,63 @@ void c_FileMgr::GetListOfSdFiles (String & Response)

if (!entry)
{
// no more files
// DEBUG_V("no more files to add to list");
ResponseJsonDoc[F("final")] = true;
break;
}

usedBytes += entry.size ();
++last;

// have we gotten to a file we have not sent yet?
if(FirstFileToSend)
{
--FirstFileToSend;
continue;
}

String EntryName = String (entry.name ());
EntryName = EntryName.substring ((('/' == EntryName[0]) ? 1 : 0));
// DEBUG_V ("EntryName: " + EntryName);
// DEBUG_V ("EntryName.length(): " + String(EntryName.length ()));

if ((0 != EntryName.length ()) &&
(EntryName != String (F ("System Volume Information"))) &&
if ((!EntryName.isEmpty ()) &&
(!EntryName.equals(String (F ("System Volume Information")))) &&
(0 != entry.size ())
)
{
// DEBUG_V ("Adding File: '" + EntryName + "'");
uint32_t neededMemory = EntryReservedSpace + (EntryName.length() * EntryUsageFactor);
uint32_t usedSpace = ResponseJsonDoc.memoryUsage();
uint32_t availableSpace = capacity - usedSpace;
if(neededMemory >= availableSpace)
{
// DEBUG_V("No more file names will fit in the list");
entry.close ();
break;
}

// DEBUG_V ("Adding File: '" + EntryName + "'");
JsonObject CurrentFile = FileArray.createNestedObject ();
CurrentFile[CN_name] = EntryName;
CurrentFile[F ("date")] = entry.getLastWrite ();
CurrentFile[F ("length")] = entry.size ();
}

entry.close ();
}
} // end while true

dir.close();

ResponseJsonDoc[F("usedBytes")] = usedBytes;
ResponseJsonDoc[F("last")] = last;

if(ResponseJsonDoc.overflowed())
{
logcon (String(CN_stars) + F ("ERROR: JSON Doc too small to hold the list of SD files") + CN_stars);
break;
}

} while (false);
} while (false); // once

// DEBUG_V(String("ResponseJsonDoc.size(): ") + String(ResponseJsonDoc.size()));
Response.reserve(1024);
Expand Down Expand Up @@ -909,8 +938,8 @@ void c_FileMgr::GetListOfSdFiles (std::vector<String> & Response)
// DEBUG_V ("EntryName: '" + EntryName + "'");
// DEBUG_V ("EntryName.length(): " + String(EntryName.length ()));

if ((0 != EntryName.length ()) &&
(EntryName != String (F ("System Volume Information"))) &&
if ((!EntryName.isEmpty ()) &&
(!EntryName.equals(String (F ("System Volume Information")))) &&
(0 != entry.size ())
)
{
Expand Down
2 changes: 1 addition & 1 deletion ESPixelStick/src/FileMgr.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class c_FileMgr
size_t WriteSdFile (const FileId & FileHandle, byte * FileData, size_t NumBytesToWrite);
size_t WriteSdFile (const FileId & FileHandle, byte * FileData, size_t NumBytesToWrite, size_t StartingPosition);
void CloseSdFile (const FileId & FileHandle);
void GetListOfSdFiles (String & Response);
void GetListOfSdFiles (String & Response, uint32_t FirstFileToSend);
void GetListOfSdFiles (std::vector<String> & Response);
size_t GetSdFileSize (const String & FileName);
size_t GetSdFileSize (const FileId & FileHandle);
Expand Down
6 changes: 5 additions & 1 deletion ESPixelStick/src/WebMgr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,8 +250,12 @@ void c_WebMgr::init ()
}
else
{
// DEBUG_V(String("URL: ") + request->url ());
uint32_t StartingFileIndex = request->url ().substring(String(F("/files/")).length()).toInt();
// DEBUG_V(String("StartingFileIndex: ") + StartingFileIndex);

String Response;
FileMgr.GetListOfSdFiles(Response);
FileMgr.GetListOfSdFiles(Response, StartingFileIndex);
request->send (200, CN_applicationSLASHjson, Response);
// DEBUG_V(String("Files: ") + Response);
}
Expand Down
2 changes: 1 addition & 1 deletion ESPixelStick/src/input/InputFPPRemote.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ void c_InputFPPRemote::StartPlaying (String& FileName)
do // once
{
// DEBUG_V (String ("FileName: '") + FileName + "'");
if ((0 == FileName.length ()) ||
if ((FileName.isEmpty ()) ||
(FileName.equals("null")))
{
// DEBUG_V ("No file to play");
Expand Down
6 changes: 3 additions & 3 deletions ESPixelStick/src/input/InputFPPRemotePlayFileFsm.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@ bool fsm_PlayFile_state_PlayingFile::Sync (String& FileName, float ElapsedSecond
do // once
{
// are we on the correct file?
if (FileName != p_Parent->GetFileName ())
if (!FileName.equals(p_Parent->GetFileName ()))
{
// DEBUG_V ("Sync: Filename change");
p_Parent->Stop ();
Expand Down Expand Up @@ -620,7 +620,7 @@ void fsm_PlayFile_state_Error::Start (String& FileName, float ElapsedSeconds, ui
// DEBUG_START;
// DEBUG_V("fsm_PlayFile_state_Error::Start");

if (FileName != p_Parent->GetFileName ())
if (!FileName.equals(p_Parent->GetFileName ()))
{
p_Parent->fsm_PlayFile_state_Idle_imp.Start (FileName, ElapsedSeconds, PlayCount);
}
Expand All @@ -647,7 +647,7 @@ bool fsm_PlayFile_state_Error::Sync (String& FileName, float ElapsedSeconds)
// DEBUG_START;
// DEBUG_V("State:Error");

if (FileName != p_Parent->GetFileName ())
if (!FileName.equals(p_Parent->GetFileName ()))
{
p_Parent->fsm_PlayFile_state_Idle_imp.Start (FileName, ElapsedSeconds, 1);
}
Expand Down
2 changes: 1 addition & 1 deletion ESPixelStick/src/platformDefinitions/GPIO_Defs_ESP32_QUINLED_Dig-Octa.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
// Not Finished - #define SUPPORT_OutputType_TLS3001
// #define SUPPORT_OutputType_APA102 // SPI
// #define SUPPORT_OutputType_DMX // UART / RMT
// #define SUPPORT_OutputType_GECE // UART / RMT
#define SUPPORT_OutputType_GECE // UART / RMT
#define SUPPORT_OutputType_GS8208 // UART / RMT
// #define SUPPORT_OutputType_Renard // UART / RMT
// #define SUPPORT_OutputType_Serial // UART / RMT
Expand Down
2 changes: 1 addition & 1 deletion ESPixelStick/src/service/FPPDiscovery.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1112,7 +1112,7 @@ void c_FPPDiscovery::StartPlaying (String & FileName, float SecondsElapsed)
}
// DEBUG_V ("");

if (0 == FileName.length())
if (FileName.isEmpty())
{
// DEBUG_V("Do not have a file to start");
break;
Expand Down
51 changes: 33 additions & 18 deletions html/script.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
var StatusRequestTimer = null;
var FseqFileListRequestTimer = null;
let ExpectedStartingFileIndex = 0;
var DiagTimer = null;

// global data
Expand Down Expand Up @@ -243,7 +244,7 @@ $(function () {
// console.log(file);
// console.log(resp);
Dropzone.forElement('#filemanagementupload').removeAllFiles(true)
RequestListOfFiles();
RequestListOfFiles(0);
});

this.on('complete', function (file, resp) {
Expand Down Expand Up @@ -387,14 +388,14 @@ function ProcessWindowChange(NextWindow) {
}

else if (NextWindow === "#config") {
RequestListOfFiles();
RequestListOfFiles(0);
RcfResponse = RequestConfigFile("config.json");
RcfResponse = RequestConfigFile("output_config.json");
RcfResponse = RequestConfigFile("input_config.json");
}

else if (NextWindow === "#filemanagement") {
RequestListOfFiles();
RequestListOfFiles(0);
}

UpdateAdvancedOptionsMode();
Expand Down Expand Up @@ -507,22 +508,24 @@ function RequestStatusUpdate()

} // RequestStatusUpdate

function RequestListOfFiles() {
function RequestListOfFiles(StartingFileIndex) {
ExpectedStartingFileIndex = StartingFileIndex;

// is the timer running?
if (null === FseqFileListRequestTimer) {
// timer runs until we get a response
FseqFileListRequestTimer = setTimeout(function () {
clearTimeout(FseqFileListRequestTimer);
FseqFileListRequestTimer = null;

RequestListOfFiles();
RequestListOfFiles(StartingFileIndex);

}, 1000);
} // end timer was not running

// ask for a file list from the server

return fetch("HTTP://" + target + "/files", {
return fetch("HTTP://" + target + "/files/" + StartingFileIndex, {
method: 'GET',
mode: "cors", // no-cors, *cors, same-origin
headers: { 'Content-Type': 'application/json' },
Expand Down Expand Up @@ -586,20 +589,27 @@ function ProcessGetFileListResponse(JsonConfigData) {
clearTimeout(FseqFileListRequestTimer);
FseqFileListRequestTimer = null;

// console.info("$('#FileManagementTable > tr').length " + $('#FileManagementTable > tr').length);
// are we starting a fresh list (first chunk)
if(JsonConfigData.first === 0)
{
// console.info("$('#FileManagementTable > tr').length " + $('#FileManagementTable > tr').length);

while (1 < $('#FileManagementTable > tr').length) {
// console.info("Deleting $('#FileManagementTable tr').length " + $('#FileManagementTable tr').length);
$('#FileManagementTable tr').last().remove();
// console.log("After Delete: $('#FileManagementTable tr').length " + $('#FileManagementTable tr').length);
while (1 < $('#FileManagementTable > tr').length) {
// console.info("Deleting $('#FileManagementTable tr').length " + $('#FileManagementTable tr').length);
$('#FileManagementTable tr').last().remove();
// console.log("After Delete: $('#FileManagementTable tr').length " + $('#FileManagementTable tr').length);
}
}

let CurrentRowId = 0;
console.info("Expected File Index: " + ExpectedStartingFileIndex);
console.info("Received File index: " + JsonConfigData.first);

JsonConfigData.files.forEach(function (file) {
let CurrentRowId = $('#FileManagementTable > tr').length;
let SelectedPattern = '<td><input type="checkbox" id="FileSelected_' + (CurrentRowId) + '"></td>';
let NamePattern = '<td><output type="text" id="FileName_' + (CurrentRowId) + '"></td>';
let DatePattern = '<td><output type="text" id="FileDate_' + (CurrentRowId) + '"></td>';
let SizePattern = '<td><output type="text" id="FileSize_' + (CurrentRowId) + '"></td>';
let NamePattern = '<td><output type="text" id="FileName_' + (CurrentRowId) + '"></td>';
let DatePattern = '<td><output type="text" id="FileDate_' + (CurrentRowId) + '"></td>';
let SizePattern = '<td><output type="text" id="FileSize_' + (CurrentRowId) + '"></td>';

let rowPattern = '<tr>' + SelectedPattern + NamePattern + DatePattern + SizePattern + '</tr>';
$('#FileManagementTable tr:last').after(rowPattern);
Expand All @@ -615,9 +625,14 @@ function ProcessGetFileListResponse(JsonConfigData) {
$('#FileDate_' + (CurrentRowId)).val(new Date(0).toISOString());
$('#FileSize_' + (CurrentRowId)).val(0);
}
}); // end foreach

CurrentRowId++;
});
// was this the last chunk?
if(false === JsonConfigData.final)
{
// not last. Ask for the next chunk
RequestListOfFiles(JsonConfigData.last);
}
} // ProcessGetFileListResponse

function RequestFileDeletion() {
Expand All @@ -631,7 +646,7 @@ function RequestFileDeletion() {
}
});

RequestListOfFiles();
RequestListOfFiles(0);

} // RequestFileDeletion

Expand Down

0 comments on commit dad96ac

Please sign in to comment.