Skip to content

realtime reading of binary files #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nkx111 wants to merge 6 commits into
base: master
Choose a base branch
from
Open

realtime reading of binary files #33

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
251ddf8
TRestRawMultiCoBoAsAdToSignalProcess: real time reading for the binar…
Jun 29, 2021
92bf7d9
TRestRawToSignalProcess::FRead() waits for partial reading
nkx111 Jun 29, 2021
0a015ac
TRestRawMultiCoBoAsAdToSignalProcess::CloseFile(): more stable
nkx111 Jun 29, 2021
80ff35e
TRestRawToSignalProcess: update sleep method
nkx111 Jun 30, 2021
1c299a6
TRestRawToSignalProcess: default waiting time at eof is now 0 to retu…
nkx111 Jun 30, 2021
dfb0bb3
fixed TRestRawSignal::IsADCSaturation
Sep 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion inc/TRestRawMultiCoBoAsAdToSignalProcess.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,9 @@ class TRestRawMultiCoBoAsAdToSignalProcess : public TRestRawToSignalProcess {

bool ReadFrameHeader(CoBoHeaderFrame& Frame);

bool ReadFrameDataP(FILE* f, CoBoHeaderFrame& hdr);
bool ReadFrameDataP(int fileid, CoBoHeaderFrame& hdr);
bool ReadFrameDataF(CoBoHeaderFrame& hdr);
void CloseFile(int fileid);

Bool_t EndReading();

Expand Down
2 changes: 2 additions & 0 deletions inc/TRestRawToSignalProcess.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class TRestRawToSignalProcess : public TRestEventProcess {
Double_t tStart;
Long64_t totalBytesReaded;
Long64_t totalBytes;
Int_t fMaxWaitTimeEOF; // wait for xx seconds at eof before really closing the binary file

TRestRawSignalEvent* fSignalEvent = 0; //!
#ifndef __CINT__
Expand Down Expand Up @@ -70,6 +71,7 @@ class TRestRawToSignalProcess : public TRestEventProcess {

void SetRunOrigin(Int_t run_origin) { fRunOrigin = run_origin; }
void SetSubRunOrigin(Int_t sub_run_origin) { fSubRunOrigin = sub_run_origin; }
bool FRead(void* ptr, size_t size, size_t n, FILE* file);

void LoadConfig(std::string cfgFilename, std::string name = "");

Expand Down
66 changes: 23 additions & 43 deletions src/TRestRawMultiCoBoAsAdToSignalProcess.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,6 @@
/// <hr>
///

// int counter = 0;

#include "TRestRawMultiCoBoAsAdToSignalProcess.h"

#include "TRestDataBase.h"
Expand Down Expand Up @@ -153,23 +151,18 @@ Bool_t TRestRawMultiCoBoAsAdToSignalProcess::AddInputFile(string file) {
fileerrors.push_back(0);

int i = fHeaderFrame.size() - 1;
if (fread(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i]) != 1 || feof(fInputFiles[i])) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
if (!FRead(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i])) {
CloseFile(i);
return kFALSE;
}
totalBytesReaded += 256;
if (!ReadFrameHeader(fHeaderFrame[i])) {
cout << "error when reading frame header in file " << i << " \"" << fInputFileNames[i] << "\""
<< endl;
cout << "event id " << fCurrentEvent + 1 << ". The file will be closed" << endl;
fHeaderFrame[i].Show();
cout << endl;
GetChar();
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
CloseFile(i);
return false;
}

Expand Down Expand Up @@ -260,23 +253,18 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {
// if the file is opened but not read, read header frame
for (int i = 0; i < fInputFiles.size(); i++) {
if (fInputFiles[i] != NULL && ftell(fInputFiles[i]) == 0) {
if (fread(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i]) != 1 || feof(fInputFiles[i])) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
if (!FRead(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i])) {
CloseFile(i);
return kFALSE;
}
totalBytesReaded += 256;
if (!ReadFrameHeader(fHeaderFrame[i])) {
cout << "error when reading frame header in file " << i << " \"" << fInputFileNames[i] << "\""
<< endl;
cout << "event id " << fCurrentEvent + 1 << ". The file will be closed" << endl;
fHeaderFrame[i].Show();
cout << endl;
GetChar();
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
CloseFile(i);
return false;
}
}
Expand Down Expand Up @@ -313,32 +301,24 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {
unsigned int type = fHeaderFrame[i].frameType;
if (fHeaderFrame[i].frameHeader[0] == 0x08 && type == 1) // partial readout
{
ReadFrameDataP(fInputFiles[i], fHeaderFrame[i]);
ReadFrameDataP(i, fHeaderFrame[i]);
} else if (fHeaderFrame[i].frameHeader[0] == 0x08 && type == 2) // full readout
{
if (fread(frameDataF, 2048, 136, fInputFiles[i]) != 136 || feof(fInputFiles[i])) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
if (!FRead(frameDataF, 2048, 136, fInputFiles[i])) {
CloseFile(i);
break;
}
totalBytesReaded += 278528;
ReadFrameDataF(fHeaderFrame[i]);
} else {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
CloseFile(i);
return false;
}

// reading next header
if (fread(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i]) != 1 || feof(fInputFiles[i])) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295; // maximum of unsigned int
if (!FRead(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i])) {
CloseFile(i);
break;
}
totalBytesReaded += 256;
if (!ReadFrameHeader(fHeaderFrame[i])) {
warning << "Event " << fCurrentEvent << " : error when reading next frame header" << endl;
warning << "in file " << i << " \"" << fInputFileNames[i] << "\"" << endl;
Expand All @@ -350,11 +330,9 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {
fVerboseLevel = REST_Silent;
for (int k = 0; k < 1088; k++) // fullreadoutsize(278528)/headersize(256)=1088
{
if (fread(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i]) != 1 ||
feof(fInputFiles[i])) {
if (!FRead(fHeaderFrame[i].frameHeader, 256, 1, fInputFiles[i])) {
break;
}
totalBytesReaded += 256;
if (ReadFrameHeader(fHeaderFrame[i])) {
fVerboseLevel = tmp;
warning << "Successfully found next header (EventId : " << fHeaderFrame[i].eventIdx
Expand All @@ -368,9 +346,7 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::fillbuffer() {
}
}
if (!found) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295; // maximum of unsigned int
CloseFile(i);
}
}
}
Expand Down Expand Up @@ -493,7 +469,7 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameHeader(CoBoHeaderFrame& HdrF
return true;
}

bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataP(FILE* f, CoBoHeaderFrame& hdr) {
bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataP(int fileid, CoBoHeaderFrame& hdr) {
unsigned int i;
int j;
unsigned int agetIdx, chanIdx, buckIdx, sample, chTmp;
Expand All @@ -510,12 +486,10 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataP(FILE* f, CoBoHeaderFra
if (size > 256) {
unsigned int NBuckTotal = (size - 256) / 4;
for (i = 0; i < NBuckTotal; i++) {
if ((fread(frameDataP, 4, 1, f)) != 1 || feof(f)) {
fclose(f);
f = NULL;
if (!FRead(frameDataP, 4, 1, fInputFiles[fileid])) {
CloseFile(fileid);
return kFALSE;
}
totalBytesReaded += 4;
// total: 4bytes, 32 bits
// 11 111111|1 1111111|11 11 1111|11111111
// agetIdx chanIdx buckIdx unused samplepoint
Expand Down Expand Up @@ -546,6 +520,12 @@ bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataP(FILE* f, CoBoHeaderFra
return true;
}

void TRestRawMultiCoBoAsAdToSignalProcess::CloseFile(int i) {
fclose(fInputFiles[i]);
fInputFiles[i] = NULL;
fHeaderFrame[i].eventIdx = (unsigned int)4294967295;
}

bool TRestRawMultiCoBoAsAdToSignalProcess::ReadFrameDataF(CoBoHeaderFrame& hdr) {
int i;
int j;
Expand Down
19 changes: 19 additions & 0 deletions src/TRestRawToSignalProcess.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ void TRestRawToSignalProcess::InitFromConfigFile() {
fElectronicsType = GetParameter("electronics");
fShowSamples = StringToInteger(GetParameter("showSamples", "10"));
fMinPoints = StringToInteger(GetParameter("minPoints", "512"));
fMaxWaitTimeEOF = StringToInteger(GetParameter("maxWaitTimeEOF", "1"));

PrintMetadata();

Expand Down Expand Up @@ -240,3 +241,21 @@ Bool_t TRestRawToSignalProcess::GoToNextFile() {
}
return false;
}

// custom fread method which has retry times for the file to be written when reading
bool TRestRawToSignalProcess::FRead(void* ptr, size_t size, size_t n, FILE* file) {
int nwaits = 0;
while (1) {
int reads = fread(ptr, size, n, file);
totalBytesReaded += reads * size;
if (reads != n || feof(file)) {
nwaits++;
if (nwaits > fMaxWaitTimeEOF) return false;
sleep(1);
Comment thread
nkx111 marked this conversation as resolved.
Outdated
fseek(file, ftell(file), 0);
} else {
return true;
}
}
return false;
}