Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

realtime reading of binary files #33

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

realtime reading of binary files #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
251ddf8
TRestRawMultiCoBoAsAdToSignalProcess: real time reading for the binar…
Jun 29, 2021
92bf7d9
TRestRawToSignalProcess::FRead() waits for partial reading
nkx111 Jun 29, 2021
0a015ac
TRestRawMultiCoBoAsAdToSignalProcess::CloseFile(): more stable
nkx111 Jun 29, 2021
80ff35e
TRestRawToSignalProcess: update sleep method
nkx111 Jun 30, 2021
1c299a6
TRestRawToSignalProcess: default waiting time at eof is now 0 to retu…
nkx111 Jun 30, 2021
dfb0bb3
fixed TRestRawSignal::IsADCSaturation
Sep 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion inc/TRestRawMultiCoBoAsAdToSignalProcess.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,9 @@ class TRestRawMultiCoBoAsAdToSignalProcess : public TRestRawToSignalProcess {

bool ReadFrameHeader(CoBoHeaderFrame& Frame);

bool ReadFrameDataP(FILE* f, CoBoHeaderFrame& hdr);
bool ReadFrameDataP(int fileid, CoBoHeaderFrame& hdr);
bool ReadFrameDataF(CoBoHeaderFrame& hdr);
void CloseFile(int fileid);

Bool_t EndReading();

Expand Down
2 changes: 1 addition & 1 deletion inc/TRestRawSignal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ class TRestRawSignal : public TObject {

Int_t GetMinPeakBin();

Bool_t IsACDSaturation(int Nflat = 3, int OverThres = 3600);
Bool_t IsADCSaturation(int Nflat = 3);

void GetDifferentialSignal(TRestRawSignal* diffSgnl, Int_t smearPoints);

Expand Down
2 changes: 2 additions & 0 deletions inc/TRestRawToSignalProcess.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class TRestRawToSignalProcess : public TRestEventProcess {
Double_t tStart;
Long64_t totalBytesReaded;
Long64_t totalBytes;
Int_t fMaxWaitTimeEOF; // wait for xx seconds at eof before really closing the binary file

TRestRawSignalEvent* fSignalEvent = 0; //!
#ifndef __CINT__
Expand Down Expand Up @@ -70,6 +71,7 @@ class TRestRawToSignalProcess : public TRestEventProcess {

void SetRunOrigin(Int_t run_origin) { fRunOrigin = run_origin; }
void SetSubRunOrigin(Int_t sub_run_origin) { fSubRunOrigin = sub_run_origin; }
bool FRead(void* ptr, size_t size, size_t n, FILE* file);

void LoadConfig(std::string cfgFilename, std::string name = "");

Expand Down
78 changes: 30 additions & 48 deletions src/TRestRawMultiCoBoAsAdToSignalProcess.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,6 @@
/// <hr>
///

// int counter = 0;

#include "TRestRawMultiCoBoAsAdToSignalProcess.h"

#include "TRestDataBase.h"
Expand Down Expand Up @@ -153,23 +151,18 @@ Bool_t TRestRawMultiCoBoAsAdToSignalProcess::AddInputFile(string file) {
fileerrors.push_back(0);

int i = fHeaderFrame.size() - 1;
if (fread(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i]) != 1 || feof(fInputFiles[i])) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
if (!FRead(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i])) {
CloseFile(i);
return kFALSE;
}
totalBytesReaded += 256;
if (!ReadFrameHeader(fHeaderFrame[i])) {
cout << "error when reading frame header in file " << i << " \"" << fInputFileNames[i] << "\""
<< endl;
cout << "event id " << fCurrentEvent + 1 << ". The file will be closed" << endl;
fHeaderFrame[i].Show();
cout << endl;
GetChar();
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
CloseFile(i);
return false;
}

Expand All @@ -182,7 +175,7 @@ TRestEvent* TRestRawMultiCoBoAsAdToSignalProcess::ProcessEvent(TRestEvent* evInp
fSignalEvent->Initialize();

if (EndReading()) {
return NULL;
return nullptr;
}
if (!fillbuffer()) {
fSignalEvent->SetOK(false);
Expand Down Expand Up @@ -238,7 +231,7 @@ TRestEvent* TRestRawMultiCoBoAsAdToSignalProcess::ProcessEvent(TRestEvent* evInp
fSignalEvent->SetSubRunOrigin(0);

// cout << fSignalEvent->GetNumberOfSignals() << endl;
// if( fSignalEvent->GetNumberOfSignals( ) == 0 ) return NULL;
// if( fSignalEvent->GetNumberOfSignals( ) == 0 ) return nullptr;

return fSignalEvent;
}
Expand All @@ -259,24 +252,19 @@ void TRestRawMultiCoBoAsAdToSignalProcess::EndProcess() {
bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {
// if the file is opened but not read, read header frame
for (int i = 0; i < fInputFiles.size(); i++) {
if (fInputFiles[i] != NULL && ftell(fInputFiles[i]) == 0) {
if (fread(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i]) != 1 || feof(fInputFiles[i])) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
if (fInputFiles[i] != nullptr && ftell(fInputFiles[i]) == 0) {
if (!FRead(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i])) {
CloseFile(i);
return kFALSE;
}
totalBytesReaded += 256;
if (!ReadFrameHeader(fHeaderFrame[i])) {
cout << "error when reading frame header in file " << i << " \"" << fInputFileNames[i] << "\""
<< endl;
cout << "event id " << fCurrentEvent + 1 << ". The file will be closed" << endl;
fHeaderFrame[i].Show();
cout << endl;
GetChar();
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
CloseFile(i);
return false;
}
}
Expand All @@ -292,7 +280,7 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {

// loop for each file
for (int i = 0; i < fHeaderFrame.size(); i++) {
if (fInputFiles[i] == NULL) {
if (fInputFiles[i] == nullptr) {
continue;
}

Expand All @@ -313,32 +301,24 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {
unsigned int type = fHeaderFrame[i].frameType;
if (fHeaderFrame[i].frameHeader[0] == 0x08 && type == 1) // partial readout
{
ReadFrameDataP(fInputFiles[i], fHeaderFrame[i]);
ReadFrameDataP(i, fHeaderFrame[i]);
} else if (fHeaderFrame[i].frameHeader[0] == 0x08 && type == 2) // full readout
{
if (fread(frameDataF, 2048, 136, fInputFiles[i]) != 136 || feof(fInputFiles[i])) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
if (!FRead(frameDataF, 2048, 136, fInputFiles[i])) {
CloseFile(i);
break;
}
totalBytesReaded += 278528;
ReadFrameDataF(fHeaderFrame[i]);
} else {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
CloseFile(i);
return false;
}

// reading next header
if (fread(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i]) != 1 || feof(fInputFiles[i])) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295; // maximum of unsigned int
if (!FRead(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i])) {
CloseFile(i);
break;
}
totalBytesReaded += 256;
if (!ReadFrameHeader(fHeaderFrame[i])) {
warning << "Event " << fCurrentEvent << " : error when reading next frame header" << endl;
warning << "in file " << i << " \"" << fInputFileNames[i] << "\"" << endl;
Expand All @@ -350,11 +330,9 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {
fVerboseLevel = REST_Silent;
for (int k = 0; k < 1088; k++) // fullreadoutsize(278528)/headersize(256)=1088
{
if (fread(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i]) != 1 ||
feof(fInputFiles[i])) {
if (!FRead(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i])) {
break;
}
totalBytesReaded += 256;
if (ReadFrameHeader(fHeaderFrame[i])) {
fVerboseLevel = tmp;
warning << "Successfully found next header (EventId : " << fHeaderFrame[i].eventIdx
Expand All @@ -368,9 +346,7 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {
}
}
if (!found) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295; // maximum of unsigned int
CloseFile(i);
}
}
}
Expand Down Expand Up @@ -493,7 +469,7 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameHeader(CoBoHeaderFrame& HdrF
return true;
}

bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataP(FILE* f, CoBoHeaderFrame& hdr) {
bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataP(int fileid, CoBoHeaderFrame& hdr) {
unsigned int i;
int j;
unsigned int agetIdx, chanIdx, buckIdx, sample, chTmp;
Expand All @@ -510,12 +486,10 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataP(FILE* f, CoBoHeaderFra
if (size > 256) {
unsigned int NBuckTotal = (size - 256) / 4;
for (i = 0; i < NBuckTotal; i++) {
if ((fread(frameDataP, 4, 1, f)) != 1 || feof(f)) {
fclose(f);
f = NULL;
if (!FRead(frameDataP, 4, 1, fInputFiles[fileid])) {
CloseFile(fileid);
return kFALSE;
}
totalBytesReaded += 4;
// total: 4bytes, 32 bits
// 11 111111|1 1111111|11 11 1111|11111111
// agetIdx chanIdx buckIdx unused samplepoint
Expand Down Expand Up @@ -546,6 +520,14 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataP(FILE* f, CoBoHeaderFra
return true;
}

void TRestRawMultiCoBoAsAdToSignalProcess::CloseFile(int i) {
if(fInputFiles[i] != nullptr) {
fclose(fInputFiles[i]);
fInputFiles[i] = nullptr;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
}
}

bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataF(CoBoHeaderFrame& hdr) {
int i;
int j;
Expand Down Expand Up @@ -644,7 +626,7 @@ Bool_t TRestRawMultiCoBoAsAdToSignalProcess::EndReading() {
}

for (int n = 0; n < nFiles; n++) {
if (fInputFiles[n] != NULL) {
if (fInputFiles[n] != nullptr) {
return false;
}
}
Expand Down
14 changes: 9 additions & 5 deletions src/TRestRawSignal.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -528,17 +528,21 @@ Int_t TRestRawSignal::GetMinPeakBin() {
///////////////////////////////////////////////
/// \brief It returns whether the signal has ADC saturation
///
Bool_t TRestRawSignal::IsACDSaturation(int Nflat, int OverThres) {
Bool_t TRestRawSignal::IsADCSaturation(int Nflat) {
if (Nflat <= 0) return false;
// GetMaxPeakBin() will always find the first max peak bin if mulitple
// bins are in same max value.
int index = GetMaxPeakBin();
Short_t value = fSignalData[index];

bool sat = true;
bool sat = false;
if (index + Nflat <= fSignalData.size()) {
for (int i = index; i < index + Nflat; i++) {
if (fSignalData[index + i] > OverThres && fSignalData[index] == fSignalData[index + i]) {
} else {
sat = false;
if (fSignalData[i] != value) {
break;
}
if (i == index + Nflat - 1) {
sat = true;
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/TRestRawSignalAnalysisProcess.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ TRestEvent* TRestRawSignalAnalysisProcess::ProcessEvent(TRestEvent* evInput) {
risetime[sgnl->GetID()] = sgnl->GetRiseTime();
peak_time[sgnl->GetID()] = sgnl->GetMaxPeakBin();
npointsot[sgnl->GetID()] = sgnl->GetPointsOverThreshold().size();
if (sgnl->IsACDSaturation()) saturatedchnId.push_back(sgnl->GetID());
if (sgnl->IsADCSaturation()) saturatedchnId.push_back(sgnl->GetID());
}

SetObservableValue("pointsoverthres_map", npointsot);
Expand Down
32 changes: 32 additions & 0 deletions src/TRestRawToSignalProcess.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ void TRestRawToSignalProcess::InitFromConfigFile() {
fElectronicsType = GetParameter("electronics");
fShowSamples = StringToInteger(GetParameter("showSamples", "10"));
fMinPoints = StringToInteger(GetParameter("minPoints", "512"));
fMaxWaitTimeEOF = StringToInteger(GetParameter("maxWaitTimeEOF", "0"));

PrintMetadata();

Expand Down Expand Up @@ -240,3 +241,34 @@ Bool_t TRestRawToSignalProcess::GoToNextFile() {
}
return false;
}

// custom fread method which has retry times for the file to be written when reading
bool TRestRawToSignalProcess::FRead(void* ptr, size_t size, size_t n, FILE* file) {
if (file == nullptr || ptr == nullptr) return false;
if (size == 0 || n == 0) return false;
int nwaits = 0;
size_t chunksReaded = 0;
size_t chunksRemaining = n;
while (1) {
int pos = ftell(file);
int reads = fread((char*)ptr + chunksReaded * size, size, chunksRemaining, file);
totalBytesReaded += reads * size;
if (reads != chunksRemaining || feof(file)) {
if (reads == 0) {
nwaits++;
} else {
// In case it reads something partially
nwaits = 1;
chunksReaded += reads;
chunksRemaining -= reads;
}
if (nwaits > fMaxWaitTimeEOF) return false;

std::this_thread::sleep_for(std::chrono::seconds(1));
fseek(file, ftell(file), 0);
} else {
return true;
}
}
return false;
}