diff --git a/otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.cc b/otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.cc index 5789dc15..623f11ed 100644 --- a/otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.cc +++ b/otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.cc @@ -19,14 +19,21 @@ #include #include #include +#include #include +#include "otsdaq/ConfigurationInterface/ConfigurationInterface.h" +#include "otsdaq/Macros/StringMacros.h" +#include "otsdaq/TableCore/TableBase.h" + #define OUT_ON_ERR_SIZE 2000 //tail size of output to include on error using namespace ots; XDAQ_INSTANTIATOR_IMPL(ARTDAQSupervisor) +const std::string ARTDAQSupervisor::ARTDAQ_SYSVAR_NAMESPACE = "artdaq"; + #define FAKE_CONFIG_NAME "ots_config" #define DAQINTERFACE_PORT \ std::atoi(__ENV__("ARTDAQ_BASE_PORT")) + \ @@ -540,6 +547,9 @@ void ARTDAQSupervisor::init(void) // } } start_runner_(); + + initArtdaqSystemVariables(); + __SUP_COUT__ << "Initialized." << __E__; } // end init() @@ -548,6 +558,8 @@ void ARTDAQSupervisor::transitionConfiguring(toolbox::Event::Reference /*event*/ { __SUP_COUTT__ << "transitionConfiguring" << __E__; + loadArtdaqSystemVariables(); + // activate the configuration tree (the first iteration) if(RunControlStateMachine::getIterationIndex() == 0 && RunControlStateMachine::getSubIterationIndex() == 0) @@ -2141,3 +2153,145 @@ void ots::ARTDAQSupervisor::start_runner_() runner_thread_ = std::make_unique(&ots::ARTDAQSupervisor::daqinterfaceRunner_, this); } // end start_runner_() + +//============================================================================== +std::string ARTDAQSupervisor::getServiceDataFilePath() const +{ + return StringMacros::getPersistentSystemVariablesFilePath(); +} // end getServiceDataFilePath() + +//============================================================================== +void ARTDAQSupervisor::initArtdaqSystemVariables() +{ + loadArtdaqSystemVariables(); + + auto& ns = StringMacros::systemVariables_[ARTDAQ_SYSVAR_NAMESPACE]; + __SUP_COUT__ << "Artdaq system variables initialized: " + << StringMacros::mapToString(ns) << __E__; +} // end initArtdaqSystemVariables() + +//============================================================================== +void ARTDAQSupervisor::loadArtdaqSystemVariables() +{ + if(StringMacros::loadPersistentSystemVariables()) + __SUP_COUT__ << "Loaded artdaq system variables from " << getServiceDataFilePath() + << __E__; + else + __SUP_COUT__ << "No persisted artdaq system variables file found at " + << getServiceDataFilePath() << __E__; +} // end loadArtdaqSystemVariables() + +//============================================================================== +void ARTDAQSupervisor::saveArtdaqSystemVariables() +{ + std::string filePath = getServiceDataFilePath(); + std::ofstream file(filePath); + if(!file.is_open()) + { + __SUP_SS__ << "Failed to open file for writing artdaq system variables: " + << filePath << __E__; + __SUP_SS_THROW__; + } + + for(auto& [key, value] : StringMacros::systemVariables_[ARTDAQ_SYSVAR_NAMESPACE]) + file << key << "=" << value << "\n"; + + __SUP_COUT__ << "Saved artdaq system variables to " << filePath << __E__; +} // end saveArtdaqSystemVariables() + +//============================================================================== +void ARTDAQSupervisor::forceSupervisorPropertyValues(void) +{ + CorePropertySupervisorBase::addSupervisorProperty( + CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.AutomatedRequestTypes, + "getSystemVariables | getJsonDocuments"); +} // end forceSupervisorPropertyValues() + +//============================================================================== +void ARTDAQSupervisor::request(const std::string& requestType, + cgicc::Cgicc& cgiIn, + HttpXmlDocument& xmlOut, + const WebUsers::RequestUserInfo& /*userInfo*/) +try +{ + __SUP_COUT__ << "ARTDAQSupervisor request: " << requestType << __E__; + + if(requestType == "getSystemVariables") + { + for(auto& [key, value] : StringMacros::systemVariables_[ARTDAQ_SYSVAR_NAMESPACE]) + xmlOut.addTextElementToData("artdaq_" + key, value); + } + else if(requestType == "setSystemVariable") + { + std::string key = CgiDataUtilities::postData(cgiIn, "key"); + std::string value = CgiDataUtilities::postData(cgiIn, "value"); + + if(key.empty()) + { + xmlOut.addTextElementToData("Error", "Variable key must not be empty."); + return; + } + for(char c : key) + if(!std::isalnum(c) && c != '_') + { + xmlOut.addTextElementToData( + "Error", + "Variable key must contain only alphanumeric characters and " + "underscores."); + return; + } + + StringMacros::systemVariables_[ARTDAQ_SYSVAR_NAMESPACE][key] = value; + saveArtdaqSystemVariables(); + + __SUP_COUT__ << "Set artdaq system variable " << key << " = " << value << __E__; + xmlOut.addTextElementToData("Success", "Variable '" + key + "' set."); + } + else if(requestType == "getJsonDocuments") + { + auto* ifc = ConfigurationInterface::getInstance(); + + std::set allTableNames = ifc->getAllTableNames(); + + for(const auto& tableName : allTableNames) + { + if(tableName.find(TableBase::JSON_DOC_PREPEND) != 0) + continue; + + std::string docName = tableName.substr(TableBase::JSON_DOC_PREPEND.size()); + + TableBase tmpTable(true, tableName); + std::set versions = ifc->getVersions(&tmpTable); + + std::string versionList; + for(const auto& v : versions) + { + if(!versionList.empty()) + versionList += ","; + versionList += v.toString(); + } + + xmlOut.addTextElementToData("jsonDoc_name", docName); + xmlOut.addTextElementToData("jsonDoc_versions", versionList); + } + } + else + { + __SUP_SS__ << "Unknown request type '" << requestType << "' for ARTDAQSupervisor." + << __E__; + __SUP_COUT__ << ss.str(); + xmlOut.addTextElementToData("Error", ss.str()); + } +} +catch(const std::runtime_error& e) +{ + __SUP_SS__ << "Error handling request '" << requestType << "': " << e.what() << __E__; + __SUP_COUT_ERR__ << ss.str(); + xmlOut.addTextElementToData("Error", ss.str()); +} +catch(...) +{ + __SUP_SS__ << "Unknown error handling request '" << requestType << "'." << __E__; + __SUP_COUT_ERR__ << ss.str(); + xmlOut.addTextElementToData("Error", ss.str()); +} // end request() diff --git a/otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.hh b/otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.hh index 6c8d24cb..6768ce55 100644 --- a/otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.hh +++ b/otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.hh @@ -52,6 +52,11 @@ class ARTDAQSupervisor : public CoreSupervisorBase virtual void transitionStopping(toolbox::Event::Reference event) override; virtual void enteringError(toolbox::Event::Reference event) override; + void request(const std::string& requestType, + cgicc::Cgicc& cgiIn, + HttpXmlDocument& xmlOut, + const WebUsers::RequestUserInfo& userInfo) override; + virtual std::vector getSubappInfo(void) override; virtual std::string getStatusProgressDetail(void) override { @@ -80,6 +85,14 @@ class ARTDAQSupervisor : public CoreSupervisorBase static std::list tokenize_(std::string const& input); private: + void forceSupervisorPropertyValues(void) override; + void initArtdaqSystemVariables(); + void saveArtdaqSystemVariables(); + void loadArtdaqSystemVariables(); + std::string getServiceDataFilePath() const; + + static const std::string ARTDAQ_SYSVAR_NAMESPACE; + void configuringThread(void); void startingThread(void); diff --git a/otsdaq/CodeEditor/CodeEditor.cc b/otsdaq/CodeEditor/CodeEditor.cc index 5dbaf663..68794d45 100644 --- a/otsdaq/CodeEditor/CodeEditor.cc +++ b/otsdaq/CodeEditor/CodeEditor.cc @@ -7,6 +7,7 @@ #include //for std::toupper #include //for std::map #include //for std::regex +#include //for std::stringstream #include //for std::thread using namespace ots; @@ -69,6 +70,7 @@ try // // getDirectoryContent // getFileContent + // getFhiclFileContent // saveFileContent // cleanBuild // incrementalBuild @@ -84,6 +86,10 @@ try { getFileContent(cgiIn, xmlOut); } + else if(option == "getFhiclFileContent") + { + getFhiclFileContent(cgiIn, xmlOut); + } else if(!readOnlyMode && option == "saveFileContent") { saveFileContent(cgiIn, xmlOut, username); @@ -479,6 +485,74 @@ void CodeEditor::getFileContent(cgicc::Cgicc& cgiIn, HttpXmlDocument* xmlOut) } // end getFileContent() +//============================================================================== +/// getFhiclFileContent +/// Locates a fcl file by its path relative to a $FHICL_FILE_PATH entry +/// (e.g. "mu2e-trig-config/core/trigSequences.fcl"), searching each +/// colon-separated directory in $FHICL_FILE_PATH in order, the same +/// convention any fcl-consuming tool uses to resolve #include and +/// @sequence:: search paths. This lets read-only viewers reach fcl files +/// that live outside the areas normally accessible to the Code Editor +/// ($USER_DATA, $OTSDAQ_WEB_PATH, $OTSDAQ_DATA, srcs/), such as files +/// installed by a UPS/spack product. +void CodeEditor::getFhiclFileContent(cgicc::Cgicc& cgiIn, HttpXmlDocument* xmlOut) +{ + std::string relativePath = CgiDataUtilities::getData(cgiIn, "path"); + relativePath = safePathString(StringMacros::decodeURIComponent(relativePath)); + // leading slashes are not meaningful for a $FHICL_FILE_PATH-relative lookup + while(relativePath.size() && relativePath[0] == '/') + relativePath = relativePath.substr(1); + xmlOut->addTextElementToData("path", relativePath); + + if(relativePath.find("..") != std::string::npos) + { + __SS__ << "Illegal '..' found in requested fcl path '" << relativePath << ".'" + << __E__; + __SS_THROW__; + } + + std::string fhiclFilePath; + { + const char* envVal = getenv("FHICL_FILE_PATH"); + if(envVal) + fhiclFilePath = envVal; + } + + std::string contents; + bool found = false; + std::string lastError; + std::stringstream searchPaths(fhiclFilePath); + std::string dir; + while(std::getline(searchPaths, dir, ':')) + { + if(dir.empty()) + continue; + try + { + CodeEditor::readFile(dir, relativePath, contents); + found = true; + break; + } + catch(const std::runtime_error& e) + { + lastError = e.what(); + } + } + + if(!found) + { + __SS__ << "Could not find '" << relativePath + << "' in any directory of $FHICL_FILE_PATH."; + if(lastError.size()) + ss << " Last error: " << lastError; + ss << __E__; + __SS_THROW__; + } + + xmlOut->addTextElementToData("content", contents); + +} // end getFhiclFileContent() + //============================================================================== /// getFileGitURL void CodeEditor::getFileGitURL(cgicc::Cgicc& cgiIn, HttpXmlDocument* xmlOut) diff --git a/otsdaq/CodeEditor/CodeEditor.h b/otsdaq/CodeEditor/CodeEditor.h index 3e830926..1ff40e25 100644 --- a/otsdaq/CodeEditor/CodeEditor.h +++ b/otsdaq/CodeEditor/CodeEditor.h @@ -39,6 +39,7 @@ class CodeEditor const std::string& path, HttpXmlDocument* xmlOut); void getFileContent(cgicc::Cgicc& cgiIn, HttpXmlDocument* xmlOut); + void getFhiclFileContent(cgicc::Cgicc& cgiIn, HttpXmlDocument* xmlOut); void getFileGitURL(cgicc::Cgicc& cgiIn, HttpXmlDocument* xmlOut); void saveFileContent(cgicc::Cgicc& cgiIn, HttpXmlDocument* xmlOut, diff --git a/otsdaq/ConfigurationInterface/ConfigurationManager.cc b/otsdaq/ConfigurationInterface/ConfigurationManager.cc index abfe1546..65ebe979 100644 --- a/otsdaq/ConfigurationInterface/ConfigurationManager.cc +++ b/otsdaq/ConfigurationInterface/ConfigurationManager.cc @@ -5775,6 +5775,11 @@ try { __COUTT__ << "Initializing prerequisites for artdaq!" << __E__; + // refresh persisted 'artdaq' system variables (set via web GUIs, e.g. the + // Trigger Menu Editor) so table plugins resolve current ${OTS.artdaq.*} + // values in this process at configure time + StringMacros::loadPersistentSystemVariables(); + auto activeTables = getActiveVersions(); for(auto& tablePair : activeTables) { diff --git a/otsdaq/CoreSupervisors/CorePropertySupervisorBase.cc b/otsdaq/CoreSupervisors/CorePropertySupervisorBase.cc index cd07b4c6..71096056 100644 --- a/otsdaq/CoreSupervisors/CorePropertySupervisorBase.cc +++ b/otsdaq/CoreSupervisors/CorePropertySupervisorBase.cc @@ -61,6 +61,11 @@ CorePropertySupervisorBase::CorePropertySupervisorBase(xdaq::Application* applic StringMacros::systemVariables_["System"]["totalMemoryMB"] = "unknown"; } } + + // load 'artdaq' namespace system variables persisted by the ARTDAQ Supervisor + // (e.g. set via the Trigger Menu Editor web GUI) so ${OTS.artdaq.*} + // references resolve in every Supervisor process + StringMacros::loadPersistentSystemVariables(); } // end init StringMacros::systemVariables_ __SUP_COUTV__(StringMacros::mapToString(StringMacros::systemVariables_)); diff --git a/otsdaq/FiniteStateMachine/RunInfoVInterface.h b/otsdaq/FiniteStateMachine/RunInfoVInterface.h index 8ff0fb00..e9ba48bf 100644 --- a/otsdaq/FiniteStateMachine/RunInfoVInterface.h +++ b/otsdaq/FiniteStateMachine/RunInfoVInterface.h @@ -44,7 +44,9 @@ class RunInfoVInterface ///< : public Configurable ERROR, PAUSE, RESUME, - START + START, + STOP_COMPLETE, + HALT_COMPLETE }; RunInfoVInterface (const std::string& runInfoPluginClassName, @@ -76,7 +78,6 @@ class RunInfoVInterface ///< : public Configurable const std::string& /* comment */) { __SS__ << "updateRunInfo() Not implemented by the Run Info Plugin (" << mfSubject_ << ")!!"; __SS_THROW__; }; - /// Get functions ---- const std::string& getActiveStateMachineName (void) const { return activeStateMachineName_; } diff --git a/otsdaq/GatewaySupervisor/GatewaySupervisor.cc b/otsdaq/GatewaySupervisor/GatewaySupervisor.cc index a006f521..458f82b0 100644 --- a/otsdaq/GatewaySupervisor/GatewaySupervisor.cc +++ b/otsdaq/GatewaySupervisor/GatewaySupervisor.cc @@ -5002,7 +5002,8 @@ void GatewaySupervisor::StateChangerWorkLoop(GatewaySupervisor* theSupervisor) /// escape entryText to make it html/xml safe!! //// reserved: ", ', &, <, >, \n, double-space void GatewaySupervisor::makeSystemLogEntry(const std::string& entryText, - const std::string& subjectText /* = "" */) + const std::string& subjectText /* = "" */, + bool skipFooter /* = false */) { __COUT__ << "Making System Logbook Entry: " << entryText << __E__; if(subjectText.size()) @@ -5025,6 +5026,7 @@ void GatewaySupervisor::makeSystemLogEntry(const std::string& entryText, SOAPParameters parameters("EntryText", StringMacros::encodeURIComponent(entryText)); parameters.addParameter("SubjectText", StringMacros::encodeURIComponent(subjectText)); + parameters.addParameter("SkipFooter", skipFooter ? "1" : "0"); for(auto& logbookInfo : logbookInfoMap) { @@ -5354,6 +5356,10 @@ void GatewaySupervisor::stateMachineXgiHandler(xgi::Input* in, xgi::Output* out) std::string logEntry = StringMacros::decodeURIComponent(CgiDataUtilities::postData(cgiIn, "logEntry")); + if(command == "Stop") + activeStateMachineWriteToEcl_ = + (CgiDataUtilities::postData(cgiIn, "writeToEcl") == "1"); + attemptStateMachineTransition(&xmlOut, out, command, @@ -5490,6 +5496,14 @@ try if(logEntry != "") { + if(command == RunControlStateMachine::START_TRANSITION_NAME) + { + activeStateMachineRawStartComment_ = logEntry; + activeStateMachineRawStopComment_.clear(); + } + else if(command == RunControlStateMachine::STOP_TRANSITION_NAME) + activeStateMachineRawStopComment_ = logEntry; + logEntry += " (" + StringMacros::getTimestampString(time(0)) + ")"; if(command == RunControlStateMachine::START_TRANSITION_NAME && @@ -5534,10 +5548,9 @@ try activeStateMachineSystemDumpOnRunFilename_ = ""; //clear (and set if enabled during configure transition) - activeStateMachineRequireUserLogOnRun_ = false, - activeStateMachineRequireUserLogOnConfigure_ = - false; //clear (and set if enabled during configure transition) - activeStateMachineRunInfoPluginType_ = TableViewColumnInfo:: + activeStateMachineRequireUserLogOnRun_ = false, + activeStateMachineRequireUserLogOnConfigure_ = false; + activeStateMachineRunInfoPluginType_ = TableViewColumnInfo:: DATATYPE_STRING_DEFAULT; //clear (and set if enabled during configure transition) if(currentState != RunControlStateMachine::HALTED_STATE_NAME && @@ -5864,7 +5877,7 @@ try // Claim the next run number from the Run Info plugin (pre-start transition). runNumber = runInfoInterface->claimNextRunNumber( activeStateMachineConfigureConditionID_, - getLastLogEntry(RunControlStateMachine::START_TRANSITION_NAME)); + activeStateMachineRawStartComment_); } // end Run Info Plugin handling @@ -6195,78 +6208,17 @@ void GatewaySupervisor::stateHalted(toolbox::fsm::FiniteStateMachine& /*fsm*/) __SUP_COUTV__( SOAPUtilities::translate(theStateMachine_.getCurrentMessage()).getCommand()); - // if coming from Running or Paused, update Run Info w/HALT + // if coming from Running or Paused (i.e. Abort), record HALT and HALT_COMPLETE if(theStateMachine_.getProvenanceStateName() == RunControlStateMachine::RUNNING_STATE_NAME || theStateMachine_.getProvenanceStateName() == RunControlStateMachine::PAUSED_STATE_NAME) { - try - { - ConfigurationTree configLinkNode = - CorePropertySupervisorBase::theConfigurationManager_ - ->getSupervisorTableNode(supervisorContextUID_, - supervisorApplicationUID_); - if(!configLinkNode.isDisconnected()) - { - ConfigurationTree fsmLinkNode = - configLinkNode.getNode("LinkToStateMachineTable") - .getNode(activeStateMachineName_); - std::string runInfoPluginType = - fsmLinkNode.getNode("RunInfoPluginType").getValue(); - __SUP_COUTV__(runInfoPluginType); - if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT && - runInfoPluginType != - TableViewColumnInfo::DATATYPE_STRING_ALT_DEFAULT && - runInfoPluginType != "No Run Info Plugin") - { - std::unique_ptr runInfoInterface = nullptr; - try - { - runInfoInterface.reset( - makeRunInfo(runInfoPluginType, activeStateMachineName_)); - } - catch(...) - { - } - - if(runInfoInterface == nullptr) - { - __SS__ << "Run Info interface plugin construction failed of type " - << runInfoPluginType << __E__; - __SS_THROW__; - } - - runInfoInterface->updateRunInfo( - activeStateMachineRunConditionID_, - RunInfoVInterface::RunTransitionType::HALT, - getLastLogEntry(RunControlStateMachine::HALT_TRANSITION_NAME)); - } - } - } - catch(const std::runtime_error& e) - { - __SS__ << "RUN INFO HALT TRANSITION UPDATE INTO DATABASE FAILED!!! " - << e.what() << __E__; - __SS_THROW__; - } - catch(...) - { - __SS__ << "RUN INFO HALT TRANSITION UPDATE INTO DATABASE FAILED!!! " << __E__; - try - { - throw; - } //one more try to printout extra info - catch(const std::exception& e) - { - ss << "Exception message: " << e.what(); - } - catch(...) - { - } - __SS_THROW__; - } // End write run info into db - } // end update Run Info handling + writeRunInfoTransition( + RunInfoVInterface::RunTransitionType::HALT, + getLastLogEntry(RunControlStateMachine::HALT_TRANSITION_NAME)); + writeRunInfoTransition(RunInfoVInterface::RunTransitionType::HALT_COMPLETE, ""); + } activeStateMachineWindowName_ = ""; //clear window name to indicate that no window (including Iterator) is in control, which allows GUIs to change cleanup strategy @@ -6285,89 +6237,84 @@ void GatewaySupervisor::stateConfigured(toolbox::fsm::FiniteStateMachine& /*fsm* __COUTV__( SOAPUtilities::translate(theStateMachine_.getCurrentMessage()).getCommand()); - // if coming from Running or Paused, update Run Info w/STOP + // if coming from Running or Paused, record STOP_COMPLETE + // (the STOP record was already written at the start of transitionStopping) if(theStateMachine_.getProvenanceStateName() == RunControlStateMachine::RUNNING_STATE_NAME || theStateMachine_.getProvenanceStateName() == RunControlStateMachine::PAUSED_STATE_NAME) { + writeRunInfoTransition(RunInfoVInterface::RunTransitionType::STOP_COMPLETE, ""); + + // Write consolidated end-of-run summary to ECL if enabled via env var and user didn't opt out + bool doLogConsolidated = true; //default to logging consolidated run summary try { - ConfigurationTree configLinkNode = - CorePropertySupervisorBase::theConfigurationManager_ - ->getSupervisorTableNode(supervisorContextUID_, - supervisorApplicationUID_); - if(!configLinkNode.isDisconnected()) + doLogConsolidated = __ENV__("OTS_LOG_CONSOLIDATED_RUN") == std::string("1"); + } + catch(...) + { /* ignore errors */ + ; + } + if(doLogConsolidated && activeStateMachineWriteToEcl_) + { + try { - __COUTV__(activeStateMachineName_); - ConfigurationTree fsmLinkNode = - configLinkNode.getNode("LinkToStateMachineTable") - .getNode(activeStateMachineName_); - std::string runInfoPluginType = - fsmLinkNode.getNode("RunInfoPluginType").getValue(); - __COUTV__(runInfoPluginType); - if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT && - runInfoPluginType != - TableViewColumnInfo::DATATYPE_STRING_ALT_DEFAULT && - runInfoPluginType != "No Run Info Plugin") + std::stringstream eclSs; + if(!activeStateMachineRawStartComment_.empty()) + eclSs << "Start: " << activeStateMachineRawStartComment_ << "\n"; + if(!activeStateMachineRawStopComment_.empty()) + eclSs << "Stop: " << activeStateMachineRawStopComment_ << "\n"; + + eclSs << "\nRun Number: " << activeStateMachineRunNumber_ << "\n"; + eclSs << "Run Type: " << activeStateMachineName_ << "/" + << activeStateMachineRunAlias_ << "\n"; + + eclSs << "\nStart Time: " + << StringMacros::getTimestampString( + activeStateMachineRunWallClockStartTime_) + << "\n"; + time_t endTime = time(0); + eclSs << "End Time: " << StringMacros::getTimestampString(endTime) + << "\n"; { - std::unique_ptr runInfoInterface = nullptr; - try - { - runInfoInterface.reset( - makeRunInfo(runInfoPluginType, activeStateMachineName_)); - } - catch(...) - { - } + int dur = activeStateMachineRunDuration_ms; + int dur_s = dur / 1000; + dur = dur % 1000; + int dur_m = dur_s / 60; + dur_s = dur_s % 60; + int dur_h = dur_m / 60; + dur_m = dur_m % 60; + eclSs << "Duration: " << std::setw(2) << std::setfill('0') << dur_h + << ":" << std::setw(2) << std::setfill('0') << dur_m << ":" + << std::setw(2) << std::setfill('0') << dur_s << "\n"; + } - if(runInfoInterface == nullptr) + eclSs << "\nConfiguration: " << activeStateMachineConfigurationAlias_ + << " [" << theConfigurationTableGroup_.first << "(" + << theConfigurationTableGroup_.second.str() << ")]\n"; + { + std::lock_guard lock(remoteGatewayAppsMutex_); + for(const auto& remote : remoteGatewayApps_) { - __SS__ << "Run Info interface plugin construction failed of type " - << runInfoPluginType << __E__; - __SS_THROW__; + if(!remote.fsm_included) + continue; + eclSs << " " << remote.appInfo.name << ": " + << remote.selected_config_alias << "\n"; } - - runInfoInterface->updateRunInfo( - activeStateMachineRunConditionID_, - RunInfoVInterface::RunTransitionType::STOP, - getLastLogEntry(RunControlStateMachine::STOP_TRANSITION_NAME)); } - } - else - __COUT__ << "Gateway Supervisor configuration record not found at '" - << ConfigurationManager::XDAQ_CONTEXT_TABLE_NAME << "/" - << supervisorContextUID_ << "/" << supervisorApplicationUID_ - << "' - consider adding one to control configuration dumps " - "and state machine properties." - << __E__; - } - catch(const std::runtime_error& e) - { - __SS__ - << "RUN INFO CONFIGURED STATE INSERT OR UPDATE INTO DATABASE FAILED!!! " - << e.what() << __E__; - __SS_THROW__; - } - catch(...) - { - __SS__ - << "RUN INFO CONFIGURED STATE INSERT OR UPDATE INTO DATABASE FAILED!!! " - << __E__; - try - { - throw; - } //one more try to printout extra info - catch(const std::exception& e) - { - ss << "Exception message: " << e.what(); + + makeSystemLogEntry( + eclSs.str(), + activeStateMachineRunAlias_ + " " + activeStateMachineRunNumber_, + true /* skipFooter */); } catch(...) { + __COUT_WARN__ << "Failed to write end-of-run ECL entry." << __E__; } - __SS_THROW__; - } // End write run info into db - } // end update Run Info handling + } + } } // end stateConfigured() @@ -8083,8 +8030,9 @@ try } // end make logbook entry RunControlStateMachine::theProgressBar_.step(); - activeStateMachineRunStartTime = std::chrono::steady_clock::now(); - activeStateMachineRunDuration_ms = 0; + activeStateMachineRunStartTime = std::chrono::steady_clock::now(); + activeStateMachineRunWallClockStartTime_ = time(0); + activeStateMachineRunDuration_ms = 0; broadcastMessage( theStateMachine_ .getCurrentMessage()); // ---------------------------------- broadcast! @@ -8561,6 +8509,19 @@ try std::chrono::steady_clock::now() - activeStateMachineRunStartTime) .count(); + // Write STOP to DB before the broadcast so the record exists even if the transition fails. + // A STOP_COMPLETE record is written at the end of the transition in stateConfigured(). + try + { + writeRunInfoTransition(RunInfoVInterface::RunTransitionType::STOP, + activeStateMachineRawStopComment_); + } + catch(...) + { + __COUT_WARN__ << "STOP transition DB write failed — will not prevent transition." + << __E__; + } + RunControlStateMachine::theProgressBar_.step(); bool doLog = false; @@ -14155,6 +14116,72 @@ void GatewaySupervisor::setNextRunNumber(unsigned int runNumber, runNumberFile.close(); } // end setNextRunNumber() +//============================================================================== +void GatewaySupervisor::writeRunInfoTransition( + RunInfoVInterface::RunTransitionType transitionType, const std::string& comment) +{ + try + { + ConfigurationTree configLinkNode = + CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode( + supervisorContextUID_, supervisorApplicationUID_); + if(!configLinkNode.isDisconnected()) + { + ConfigurationTree fsmLinkNode = + configLinkNode.getNode("LinkToStateMachineTable") + .getNode(activeStateMachineName_); + std::string runInfoPluginType = + fsmLinkNode.getNode("RunInfoPluginType").getValue(); + if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT && + runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_ALT_DEFAULT && + runInfoPluginType != "No Run Info Plugin") + { + std::unique_ptr runInfoInterface = nullptr; + try + { + runInfoInterface.reset( + makeRunInfo(runInfoPluginType, activeStateMachineName_)); + } + catch(...) + { + } + + if(runInfoInterface == nullptr) + { + __SS__ << "Run Info interface plugin construction failed of type " + << runInfoPluginType << __E__; + __SS_THROW__; + } + + runInfoInterface->updateRunInfo( + activeStateMachineRunConditionID_, transitionType, comment); + } + } + } + catch(const std::runtime_error& e) + { + __SS__ << "RUN INFO TRANSITION UPDATE INTO DATABASE FAILED!!! " << e.what() + << __E__; + __SS_THROW__; + } + catch(...) + { + __SS__ << "RUN INFO TRANSITION UPDATE INTO DATABASE FAILED!!! " << __E__; + try + { + throw; + } + catch(const std::exception& e) + { + ss << "Exception message: " << e.what(); + } + catch(...) + { + } + __SS_THROW__; + } +} // end writeRunInfoTransition() + //============================================================================== /// getLastLogEntry /// diff --git a/otsdaq/GatewaySupervisor/GatewaySupervisor.h b/otsdaq/GatewaySupervisor/GatewaySupervisor.h index 818f7db7..f0b17d34 100644 --- a/otsdaq/GatewaySupervisor/GatewaySupervisor.h +++ b/otsdaq/GatewaySupervisor/GatewaySupervisor.h @@ -7,6 +7,7 @@ #include "otsdaq/CoreSupervisors/ConfigurationSupervisorBase.h" #include "otsdaq/CoreSupervisors/CorePropertySupervisorBase.h" #include "otsdaq/FiniteStateMachine/RunControlStateMachine.h" +#include "otsdaq/FiniteStateMachine/RunInfoVInterface.h" #include "otsdaq/GatewaySupervisor/Iterator.h" #include "otsdaq/SOAPUtilities/SOAPMessenger.h" #include "otsdaq/SupervisorInfo/AllSupervisorInfo.h" @@ -149,7 +150,7 @@ class WorkLoopManager; void transitionStartingUp(toolbox::Event::Reference e) override; void enteringError(toolbox::Event::Reference e) override; - void makeSystemLogEntry(const std::string& entryText, const std::string& subjectText = ""); + void makeSystemLogEntry(const std::string& entryText, const std::string& subjectText = "", bool skipFooter = false); static void addSystemMessage(std::string toUserCSV, std::string message); void checkForAsyncError(void); @@ -164,6 +165,7 @@ class WorkLoopManager; void setNextRunNumber (unsigned int runNumber, const std::string& fsmName = ""); std::string getLastLogEntry (const std::string& logType, const std::string& fsmName = ""); void setLastLogEntry (const std::string& logType, const std::string& logEntry, const std::string& fsmName = ""); + void writeRunInfoTransition (RunInfoVInterface::RunTransitionType transitionType, const std::string& comment); static xoap::MessageReference lastTableGroupRequestHandler (const SOAPParameters& parameters); @@ -343,11 +345,14 @@ class WorkLoopManager; std::string activeStateMachineRunInfoPluginType_; /// stateMachineConfigureLogEntry_, stateMachineStartLogEntry_, stateMachineStopLogEntry_; + std::string activeStateMachineRawStartComment_, activeStateMachineRawStopComment_; std::string activeStateMachineRunNumber_, activeStateMachineRunAlias_, activeStateMachineConfigurationAlias_; bool activeStateMachineRollOverLogOnConfigure_, activeStateMachineRollOverLogOnStart_; std::chrono::steady_clock::time_point activeStateMachineRunStartTime; + time_t activeStateMachineRunWallClockStartTime_ = 0; int activeStateMachineRunDuration_ms; ///< For paused runs, don't count time spent in pause state + bool activeStateMachineWriteToEcl_ = true; unsigned int activeStateMachineConfigureConditionID_, activeStateMachineRunConditionID_; std::string activeStateMachineSubsystemCommonList_, activeStateMachineSubsystemCommonOverrideList_; /// // for find_if #include #include // for uintptr_t +#include // for loadPersistentSystemVariables +#include // for loadPersistentSystemVariables using namespace ots; @@ -30,6 +32,49 @@ unsigned int StringMacros::getConcurrencyCount(void) return hw; } //end getConcurrencyCount() +//============================================================================== +// getPersistentSystemVariablesFilePath +// Path of the file where 'artdaq' namespace system variables are persisted +// (written by the ARTDAQ Supervisor, e.g. from web GUI setSystemVariable requests). +std::string StringMacros::getPersistentSystemVariablesFilePath(void) +{ + return std::string(__ENV__("USER_DATA")) + "/ServiceData/ArtdaqSystemVariables.dat"; +} // end getPersistentSystemVariablesFilePath() + +//============================================================================== +// loadPersistentSystemVariables +// Load persisted 'artdaq' namespace system variables so that +// ${OTS.artdaq.} references resolve in every process, not only in +// the ARTDAQ Supervisor that saved them. Returns false if no file was found. +bool StringMacros::loadPersistentSystemVariables(void) +try +{ + // serialize concurrent callers (e.g. the parallel table-init threads calling + // ConfigurationManager::initPrereqsForARTDAQ() at configure time) - unguarded + // concurrent insertion into the static systemVariables_ map corrupts the heap + static std::mutex loadMutex; + std::lock_guard lock(loadMutex); + + std::ifstream file(getPersistentSystemVariablesFilePath()); + if(!file.is_open()) + return false; + + auto& ns = systemVariables_["artdaq"]; + std::string line; + while(std::getline(file, line)) + { + size_t eqPos = line.find('='); + if(eqPos == std::string::npos) + continue; + ns[line.substr(0, eqPos)] = line.substr(eqPos + 1); + } + return true; +} // end loadPersistentSystemVariables() +catch(...) +{ + return false; // e.g. USER_DATA not defined in this process +} + #define TLVL_EscapeString 30 // = TLVL_DEBUG + 30 #define TLVL_EnvMath 49 // = TLVL_DEBUG + 49 #define TLVL_EnvSub 50 // = TLVL_DEBUG + 50 diff --git a/otsdaq/Macros/StringMacros.h b/otsdaq/Macros/StringMacros.h index 67828b89..5b6bc852 100644 --- a/otsdaq/Macros/StringMacros.h +++ b/otsdaq/Macros/StringMacros.h @@ -91,6 +91,8 @@ struct StringMacros std::string /* value */>> systemVariables_; static const std::string TBD; //for to-be-defined system variables (so there is a value in wiz mode, before configuration, etc.) static unsigned int getConcurrencyCount (void); + static std::string getPersistentSystemVariablesFilePath(void); + static bool loadPersistentSystemVariables(void); ///< loads persisted 'artdaq' namespace systemVariables_; returns false if no file found static bool isNumber (const std::string& stringToCheck); ///< Note: before call consider use of stringToCheck = StringMacros::convertEnvironmentVariables(stringToCheck) static std::string getNumberType (const std::string& stringToCheck); ///< Note: before call consider use of stringToCheck = StringMacros::convertEnvironmentVariables(stringToCheck)