Merge pull request #50 from AxFoundation/readout_escalation

v2.2 rc

DAQController.cc

@@ -107,7 +107,6 @@ int DAQController::Arm(std::shared_ptr<Options>& options){
       digi->AcquisitionStop();
     }
   }
-  fCounter = 0;
   if (OpenThreads()) {
     fLog->Entry(MongoLog::Warning, "Error opening threads");
     fStatus = DAXHelpers::Idle;
@@ -162,7 +161,7 @@ int DAQController::Stop(){
     if (one_still_running) std::this_thread::sleep_for(std::chrono::milliseconds(100));
   }while(one_still_running && counter++ < 10);
   if (counter >= 10) fLog->Entry(MongoLog::Local, "Boards taking a while to clear");
-  std::cout<<"Deactivating boards"<<std::endl;
+  fLog->Entry(MongoLog::Local, "Stopping boards");
   for( auto const& link : fDigitizers ){
     for(auto digi : link.second){
       digi->AcquisitionStop(true);
@@ -191,7 +190,7 @@ int DAQController::Stop(){
   fPLL = 0;
   fLog->SetRunId(-1);
   fOptions.reset();
-  std::cout<<"Finished end"<<std::endl;
+  fLog->Entry(MongoLog::Local, "Finished end sequence");
   fStatus = DAXHelpers::Idle;
   return 0;
 }
@@ -209,6 +208,7 @@ void DAQController::ReadData(int link){
   std::vector<int> mutex_wait_times;
   mutex_wait_times.reserve(1<<20);
   int words = 0;
+  unsigned transfer_batch = fOptions->GetInt("transfer_batch", 8);
   int bytes_this_loop(0);
   fRunning[link] = true;
   std::chrono::microseconds sleep_time(fOptions->GetInt("us_between_reads", 10));
@@ -217,9 +217,8 @@ void DAQController::ReadData(int link){
   while(fReadLoop){
     for(auto& digi : fDigitizers[link]) {
 
-      // Every 1k reads check board status
-      if(readcycler%10000==0){
-        readcycler=0;
+      // periodically report board status
+      if(readcycler == 0){
         board_status = digi->GetAcquisitionStatus();
         fLog->Entry(MongoLog::Local, "Board %i has status 0x%04x",
             digi->bid(), board_status);
@@ -235,8 +234,7 @@ void DAQController::ReadData(int link){
             fLog->Entry(MongoLog::Local, "Board %i has PLL unlock", digi->bid());
             fPLL++;
           }
-          if (err_val & 0x2) fLog->Entry(MongoLog::Local, "Board %i has VME bus error",
-                                         digi->bid());
+          if (err_val & 0x2) fLog->Entry(MongoLog::Local, "Board %i has VME bus error", digi->bid());
         }
       }
       if((words = digi->Read(dp))<0){
@@ -249,7 +247,7 @@ void DAQController::ReadData(int link){
         bytes_this_loop += words*sizeof(char32_t);
       }
     } // for digi in digitizers
-    if (local_buffer.size() > 0) {
+    if (local_buffer.size() && (readcycler % transfer_batch == 0)) {
       fDataRate += bytes_this_loop;
       auto t_start = std::chrono::high_resolution_clock::now();
       while (fFormatters[(++c)%num_threads]->ReceiveDatapackets(local_buffer, bytes_this_loop)) {}
@@ -258,7 +256,7 @@ void DAQController::ReadData(int link){
             t_end-t_start).count());
       bytes_this_loop = 0;
     }
-    readcycler++;
+    if (++readcycler > 10000) readcycler = 0;
     std::this_thread::sleep_for(sleep_time);
   } // while run
   if (mutex_wait_times.size() > 0) {
@@ -333,10 +331,13 @@ void DAQController::StatusUpdate(mongocxx::collection* collection) {
       buf.second += x.second;
     }
   }
+  int rate_alt = std::accumulate(retmap.begin(), retmap.end(), 0,
+      [&](int tot, const std::pair<int, int>& p) {return std::move(tot) + p.second;});
   auto doc = document{} <<
     "host" << fHostname <<
     "time" << bsoncxx::types::b_date(std::chrono::system_clock::now())<<
-    "rate" << rate/1e6 <<
+    "rate_old" << rate*1e-6 <<
+    "rate" << rate_alt*1e-6 <<
     "status" << fStatus <<
     "buffer_size" << (buf.first + buf.second)/1e6 <<
     "mode" << (fOptions ? fOptions->GetString("name", "none") : "none") <<
@@ -348,7 +349,7 @@ void DAQController::StatusUpdate(mongocxx::collection* collection) {
         doc << std::to_string(pair.first) << short(pair.second>>10); // KB not MB
       } << close_document << 
     finalize;
-  collection->insert_one(std::move(doc)); // opts is const&
+  collection->insert_one(std::move(doc));
   return;
 }
 
@@ -361,7 +362,7 @@ void DAQController::InitLink(std::vector<std::shared_ptr<V1724>>& digis,
       fLog->Entry(MongoLog::Warning, "Errors during baseline fitting");
       return;
     } else if (ret > 0) {
-      fLog->Entry(MongoLog::Message, "Baselines didn't converge so we'll use Plan B");
+      fLog->Entry(MongoLog::Debug, "Baselines didn't converge so we'll use Plan B");
     }
   }
 
@@ -602,6 +603,10 @@ int DAQController::FitBaselines(std::vector<std::shared_ptr<V1724>> &digis,
               // iterate
               if (channel_finished[bid][ch] >= convergence_threshold) {
                 done &= true;
+                if (channel_finished[bid][ch]++ == convergence_threshold) {
+                  fLog->Entry(MongoLog::Local, "%i.%i converged after %i steps: %.1f", bid, ch,
+                      step, bl_per_channel[bid][ch][step]);
+                }
                 continue;
               }
 

DAQController.hh

@@ -56,8 +56,7 @@ private:
   int fNProcessingThreads;
 
   // For reporting to frontend
-  std::atomic_int fDataRate;
-  std::atomic_long fCounter;
+  volatile std::atomic_long fDataRate;
   std::atomic_int fPLL;
 };
 

DDC10.cc

@@ -16,6 +16,15 @@ DDC10::DDC10(){
 }
 
 DDC10::~DDC10(){
+   fHopts.signal_threshold = fHopts.sign = fHopts.rise_time_cut = fHopts.dynamic_veto_limit = fHopts.static_veto_duration = 0;
+   fHopts.integration_threshold = fHopts.rho_0 = fHopts.rho_1 = fHopts.rho_2 = fHopts.rho_3 = 0;
+   fHopts.window = fHopts.prescaling = fHopts.component_status = fHopts.width_cut = fHopts.delay = 0;
+   try{
+     if (fHopts.address != "")
+       Initialize(fHopts);
+   }catch(std::exception& e) {
+     std::cout<<"HEV cleanup threw: " << e.what() << std::endl;
+   }
 }
 
 int DDC10::Initialize(HEVOptions d_opts)

Makefile

@@ -2,7 +2,7 @@ SHELL	= /bin/bash -O extglob -c
 CC	= g++
 CXX	= g++
 BUILD_COMMIT = "$(shell git log -n 1 --pretty=oneline | awk '{print $$1}')"
-CFLAGS	= -Wall -Wextra -pedantic -pedantic-errors -g -DLINUX -DREDAX_BUILD_COMMIT='$(BUILD_COMMIT)' -std=c++17 -pthread $(shell pkg-config --cflags libmongocxx)
+CFLAGS	= -Wall -Wextra -pedantic -pedantic-errors -g -O2 -DLINUX -DREDAX_BUILD_COMMIT='$(BUILD_COMMIT)' -std=c++17 -pthread $(shell pkg-config --cflags libmongocxx)
 CPPFLAGS := $(CFLAGS)
 IS_READER0 := false
 ifeq "$(shell hostname)" "reader0"

MongoLog.cc

@@ -51,19 +51,17 @@ void MongoLog::Flusher() {
     std::unique_lock<std::mutex> lk(fMutex);
     fCV.wait(lk, [&]{return fQueue.size() > 0 || fFlush == false;});
     if (fQueue.size()) {
-      auto [today, ms, priority, message] = std::move(fQueue.front());
+      auto [today, priority, message] = std::move(fQueue.front());
       fQueue.pop_front();
       lk.unlock();
-      std::stringstream msg;
-      msg<<FormatTime(&today, ms)<<" ["<<fPriorities[priority+1] <<"]: "<<message<<std::endl;
-      if (Today(&today) != fToday) RotateLogFile();
-      std::cout << msg.str();
-      fOutfile << msg.str();
+      if (today != fToday) RotateLogFile();
+      std::cout << message << std::endl;
+      fOutfile << message << std::endl;
       if(priority >= fLogLevel){
         try{
           auto d = bsoncxx::builder::stream::document{} <<
             "user" << fHostname <<
-            "message" << std::move(message) <<
+            "message" << message.substr(45) << // 45 overhead chars
             "priority" << priority <<
             "runid" << fRunId <<
             bsoncxx::builder::stream::finalize;
@@ -84,11 +82,15 @@ void MongoLog::Flusher() {
   } // while
 }
 
-std::string MongoLog::FormatTime(struct tm* date, int ms) {
-  std::string out("YYYY-MM-DD HH:mm:SS.SSS");
+std::string MongoLog::FormatTime(struct tm* date, int ms, char* writeto) {
+  std::string out;
+  if (writeto == nullptr) {
+    out = "YYYY-MM-DD HH:mm:SS.SSS | fRunId |";
+    writeto = out.data();
+  }
   // this is kinda awkward but we can't use c++20's time-formatting gubbins so :(
-  sprintf(out.data(), "%04i-%02i-%02i %02i:%02i:%02i.%03i", date->tm_year+1900,
-      date->tm_mon+1, date->tm_mday, date->tm_hour, date->tm_min, date->tm_sec, ms);
+  std::sprintf(writeto, "%04i-%02i-%02i %02i:%02i:%02i.%03i | %6i |", date->tm_year+1900,
+      date->tm_mon+1, date->tm_mday, date->tm_hour, date->tm_min, date->tm_sec, ms, fRunId);
   return out;
 }
 
@@ -114,7 +116,7 @@ int MongoLog::RotateLogFile() {
   auto filename = LogFilePath(&today);
   std::cout<<"Logging to " << filename << std::endl;
   auto pp = filename.parent_path();
-  if (!fs::exists(pp) && !fs::create_directories(pp)) {
+  if (!pp.empty() && !fs::exists(pp) && !fs::create_directories(pp)) {
     std::cout << "Could not create output directories for logging!" << std::endl;
     return -1;
   }
@@ -123,7 +125,7 @@ int MongoLog::RotateLogFile() {
     std::cout << "Could not rotate logfile!\n";
     return -1;
   }
-  fOutfile << FormatTime(&today, ms) << " [INIT]: logfile initialized: commit " << REDAX_BUILD_COMMIT << "\n";
+  fOutfile << FormatTime(&today, ms) << " INIT | logfile initialized, commit " << REDAX_BUILD_COMMIT << "\n";
   fToday = Today(&today);
   if (fDeleteAfterDays == 0) return 0;
   std::vector<int> days_per_month = {31,28,31,30,31,30,31,31,30,31,30,31};
@@ -139,36 +141,39 @@ int MongoLog::RotateLogFile() {
     last_week.tm_mday += days_per_month[last_week.tm_mon]; // off by one error???
   }
   auto p = LogFileName(&last_week);
-  if (std::experimental::filesystem::exists(p)) {
-    fOutfile << FormatTime(&today, ms) << " [INIT]: Deleting " << p << '\n';
-    std::experimental::filesystem::remove(p);
+  if (fs::exists(p)) {
+    fOutfile << FormatTime(&today, ms) << " INIT | Deleting " << p << '\n';
+    fs::remove(p);
   } else {
-    fOutfile << FormatTime(&today, ms) << " [INIT]: No older logfile to delete :(\n";
+    fOutfile << FormatTime(&today, ms) << " INIT | No older logfile to delete :(\n";
   }
   return 0;
 }
 
 int MongoLog::Entry(int priority, const std::string& message, ...){
   auto [today, ms] = Now();
 
-  // Thanks Martin
-  // http://www.martinbroadhurst.com/string-formatting-in-c.html
+  // if we had c++20 this would look much cleaner
   va_list args;
   va_start (args, message);
   // First pass just gets what the length will be
   size_t len = std::vsnprintf(NULL, 0, message.c_str(), args);
   va_end (args);
   // Declare with proper length
-  std::string msg(len + 1, 0);
-  va_start (args, message);
+  int length_overhead = 50;
+  std::string full_message(length_overhead + len + 1, '\0');
   // Fill the new string we just made
-  std::vsnprintf(msg.data(), len+1, message.c_str(), args);
+  FormatTime(&today, ms, full_message.data());
+  int end_i = std::strlen("YYYY-MM-DD HH:MM:SS.SSS | fRunID |");
+  end_i += std::sprintf(full_message.data() + end_i, " %7s | ", fPriorities[priority+1].c_str());
+  va_start (args, message);
+  std::vsnprintf(full_message.data() + end_i, len+1, message.c_str(), args);
   va_end (args);
-  // strip the trailing \0
-  msg.pop_back();
+  // strip trailing \0
+  while (full_message.back() == '\0') full_message.pop_back();
   {
-    std::unique_lock<std::mutex> lg(fMutex);
-    fQueue.emplace_back(std::make_tuple(std::move(today), ms, priority, std::move(msg)));
+    std::lock_guard<std::mutex> lg(fMutex);
+    fQueue.emplace_back(std::make_tuple(Today(&today), priority, std::move(full_message)));
   }
   fCV.notify_one();
   return 0;

MongoLog.hh

@@ -77,7 +77,7 @@ protected:
   std::tuple<struct tm, int> Now();
   void Flusher();
   int RotateLogFile();
-  virtual std::string FormatTime(struct tm*, int);
+  virtual std::string FormatTime(struct tm*, int, char* = nullptr);
   virtual int Today(struct tm*);
   virtual std::string LogFileName(struct tm*);
   virtual std::experimental::filesystem::path OutputDirectory(struct tm*);
@@ -96,7 +96,7 @@ protected:
   int fDeleteAfterDays;
   int fToday;
   std::mutex fMutex;
-  std::list<std::tuple<struct tm, int, int, std::string>> fQueue;
+  std::list<std::tuple<int, int, std::string>> fQueue;
   std::condition_variable fCV;
   std::experimental::filesystem::path fOutputDir;
   std::thread fFlushThread;

Options.cc

@@ -233,6 +233,19 @@ std::vector<u_int16_t> Options::GetThresholds(int board) {
   }
 }
 
+std::vector<int> Options::GetBLTalloc() {
+  int default_value = 23;
+  std::vector<int> ret;
+  try{
+    for (auto& val : bson_options["blt_alloc"].get_array().value)
+      ret.push_back(val.get_int32().value);
+  } catch(std::exception& e) {
+    fLog->Entry(MongoLog::Local, "Using default BLT allocs");
+    ret.push_back(default_value);
+  }
+  return ret;
+}
+
 int Options::GetV1495Opts(std::map<std::string, int>& ret) {
   if (bson_options.find("V1495") == bson_options.end())
     return 1;

Options.hh

@@ -83,6 +83,7 @@ public:
   std::vector<BoardType> GetBoards(std::string);
   std::vector<RegisterType> GetRegisters(int, bool=false);
   std::vector<uint16_t> GetDAC(int, int, uint16_t);
+  std::vector<int> GetBLTalloc();
   int GetV1495Opts(std::map<std::string, int>&);
   int GetCrateOpt(CrateOptions &ret);
   int GetHEVOpt(HEVOptions &ret);

StraxFormatter.cc

@@ -329,11 +329,11 @@ void StraxFormatter::WriteOutChunk(int chunk_i){
   struct timespec comp_start, comp_end;
   clock_gettime(CLOCK_THREAD_CPUTIME_ID, &comp_start);
 
-  std::vector<std::list<std::string>*> buffers{{&fChunks[chunk_i], &fOverlaps[chunk_i]}};
-  std::vector<long> uncompressed_size(3, 0);
+  std::list<std::string>* buffers[2] = {&fChunks[chunk_i], &fOverlaps[chunk_i]};
+  long uncompressed_size[3] = {0L, 0L, 0L};
   std::string uncompressed;
-  std::vector<std::shared_ptr<std::string>> out_buffer(3);
-  std::vector<int> wsize(3);
+  std::shared_ptr<std::string> out_buffer[3];
+  int wsize[3];
   long max_compressed_size = 0;
 
   for (int i = 0; i < 2; i++) {

V1495.hh

@@ -4,19 +4,6 @@
 #include <memory>
 #include <map>
 
-//Register address definitions taken from XENON1T m_veto class in kodiaq
-//https://github.com/coderdj/kodiaq and XENON1T DAQ m_veto config files
-
-/*
-#define V1495_ModuleReset               0x800A
-#define V1495_MaskInA                   0x1020
-#define V1495_MaskInB                   0x1024
-#define V1495_MaskInD                   0x1028
-#define V1495_MajorityThreshold         0x1014
-#define V1495_CoincidenceWidth          0x1010
-#define V1495_CTRL			0x1018
-*/
-
 class MongoLog;
 class Options;