diff --git a/otsdaq/FiniteStateMachine/RunInfoVInterface.h b/otsdaq/FiniteStateMachine/RunInfoVInterface.h index 8ff0fb00..e9ba48bf 100644 --- a/otsdaq/FiniteStateMachine/RunInfoVInterface.h +++ b/otsdaq/FiniteStateMachine/RunInfoVInterface.h @@ -44,7 +44,9 @@ class RunInfoVInterface ///< : public Configurable ERROR, PAUSE, RESUME, - START + START, + STOP_COMPLETE, + HALT_COMPLETE }; RunInfoVInterface (const std::string& runInfoPluginClassName, @@ -76,7 +78,6 @@ class RunInfoVInterface ///< : public Configurable const std::string& /* comment */) { __SS__ << "updateRunInfo() Not implemented by the Run Info Plugin (" << mfSubject_ << ")!!"; __SS_THROW__; }; - /// Get functions ---- const std::string& getActiveStateMachineName (void) const { return activeStateMachineName_; } diff --git a/otsdaq/GatewaySupervisor/GatewaySupervisor.cc b/otsdaq/GatewaySupervisor/GatewaySupervisor.cc index 9ef540a3..e51e4639 100644 --- a/otsdaq/GatewaySupervisor/GatewaySupervisor.cc +++ b/otsdaq/GatewaySupervisor/GatewaySupervisor.cc @@ -4606,7 +4606,8 @@ void GatewaySupervisor::StateChangerWorkLoop(GatewaySupervisor* theSupervisor) /// escape entryText to make it html/xml safe!! //// reserved: ", ', &, <, >, \n, double-space void GatewaySupervisor::makeSystemLogEntry(const std::string& entryText, - const std::string& subjectText /* = "" */) + const std::string& subjectText /* = "" */, + bool skipFooter /* = false */) { __COUT__ << "Making System Logbook Entry: " << entryText << __E__; if(subjectText.size()) @@ -4629,6 +4630,7 @@ void GatewaySupervisor::makeSystemLogEntry(const std::string& entryText, SOAPParameters parameters("EntryText", StringMacros::encodeURIComponent(entryText)); parameters.addParameter("SubjectText", StringMacros::encodeURIComponent(subjectText)); + parameters.addParameter("SkipFooter", skipFooter ? "1" : "0"); for(auto& logbookInfo : logbookInfoMap) { @@ -4958,6 +4960,10 @@ void GatewaySupervisor::stateMachineXgiHandler(xgi::Input* in, xgi::Output* out) std::string logEntry = StringMacros::decodeURIComponent(CgiDataUtilities::postData(cgiIn, "logEntry")); + if(command == "Stop") + activeStateMachineWriteToEcl_ = + (CgiDataUtilities::postData(cgiIn, "writeToEcl") == "1"); + attemptStateMachineTransition(&xmlOut, out, command, @@ -5094,6 +5100,14 @@ try if(logEntry != "") { + if(command == RunControlStateMachine::START_TRANSITION_NAME) + { + activeStateMachineRawStartComment_ = logEntry; + activeStateMachineRawStopComment_.clear(); + } + else if(command == RunControlStateMachine::STOP_TRANSITION_NAME) + activeStateMachineRawStopComment_ = logEntry; + logEntry += " (" + StringMacros::getTimestampString(time(0)) + ")"; if(command == RunControlStateMachine::START_TRANSITION_NAME && @@ -5138,10 +5152,9 @@ try activeStateMachineSystemDumpOnRunFilename_ = ""; //clear (and set if enabled during configure transition) - activeStateMachineRequireUserLogOnRun_ = false, - activeStateMachineRequireUserLogOnConfigure_ = - false; //clear (and set if enabled during configure transition) - activeStateMachineRunInfoPluginType_ = TableViewColumnInfo:: + activeStateMachineRequireUserLogOnRun_ = false, + activeStateMachineRequireUserLogOnConfigure_ = false; + activeStateMachineRunInfoPluginType_ = TableViewColumnInfo:: DATATYPE_STRING_DEFAULT; //clear (and set if enabled during configure transition) if(currentState != RunControlStateMachine::HALTED_STATE_NAME && @@ -5468,7 +5481,7 @@ try // Claim the next run number from the Run Info plugin (pre-start transition). runNumber = runInfoInterface->claimNextRunNumber( activeStateMachineConfigureConditionID_, - getLastLogEntry(RunControlStateMachine::START_TRANSITION_NAME)); + activeStateMachineRawStartComment_); } // end Run Info Plugin handling @@ -5799,78 +5812,17 @@ void GatewaySupervisor::stateHalted(toolbox::fsm::FiniteStateMachine& /*fsm*/) __SUP_COUTV__( SOAPUtilities::translate(theStateMachine_.getCurrentMessage()).getCommand()); - // if coming from Running or Paused, update Run Info w/HALT + // if coming from Running or Paused (i.e. Abort), record HALT and HALT_COMPLETE if(theStateMachine_.getProvenanceStateName() == RunControlStateMachine::RUNNING_STATE_NAME || theStateMachine_.getProvenanceStateName() == RunControlStateMachine::PAUSED_STATE_NAME) { - try - { - ConfigurationTree configLinkNode = - CorePropertySupervisorBase::theConfigurationManager_ - ->getSupervisorTableNode(supervisorContextUID_, - supervisorApplicationUID_); - if(!configLinkNode.isDisconnected()) - { - ConfigurationTree fsmLinkNode = - configLinkNode.getNode("LinkToStateMachineTable") - .getNode(activeStateMachineName_); - std::string runInfoPluginType = - fsmLinkNode.getNode("RunInfoPluginType").getValue(); - __SUP_COUTV__(runInfoPluginType); - if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT && - runInfoPluginType != - TableViewColumnInfo::DATATYPE_STRING_ALT_DEFAULT && - runInfoPluginType != "No Run Info Plugin") - { - std::unique_ptr runInfoInterface = nullptr; - try - { - runInfoInterface.reset( - makeRunInfo(runInfoPluginType, activeStateMachineName_)); - } - catch(...) - { - } - - if(runInfoInterface == nullptr) - { - __SS__ << "Run Info interface plugin construction failed of type " - << runInfoPluginType << __E__; - __SS_THROW__; - } - - runInfoInterface->updateRunInfo( - activeStateMachineRunConditionID_, - RunInfoVInterface::RunTransitionType::HALT, - getLastLogEntry(RunControlStateMachine::HALT_TRANSITION_NAME)); - } - } - } - catch(const std::runtime_error& e) - { - __SS__ << "RUN INFO HALT TRANSITION UPDATE INTO DATABASE FAILED!!! " - << e.what() << __E__; - __SS_THROW__; - } - catch(...) - { - __SS__ << "RUN INFO HALT TRANSITION UPDATE INTO DATABASE FAILED!!! " << __E__; - try - { - throw; - } //one more try to printout extra info - catch(const std::exception& e) - { - ss << "Exception message: " << e.what(); - } - catch(...) - { - } - __SS_THROW__; - } // End write run info into db - } // end update Run Info handling + writeRunInfoTransition( + RunInfoVInterface::RunTransitionType::HALT, + getLastLogEntry(RunControlStateMachine::HALT_TRANSITION_NAME)); + writeRunInfoTransition(RunInfoVInterface::RunTransitionType::HALT_COMPLETE, ""); + } activeStateMachineWindowName_ = ""; //clear window name to indicate that no window (including Iterator) is in control, which allows GUIs to change cleanup strategy @@ -5889,89 +5841,84 @@ void GatewaySupervisor::stateConfigured(toolbox::fsm::FiniteStateMachine& /*fsm* __COUTV__( SOAPUtilities::translate(theStateMachine_.getCurrentMessage()).getCommand()); - // if coming from Running or Paused, update Run Info w/STOP + // if coming from Running or Paused, record STOP_COMPLETE + // (the STOP record was already written at the start of transitionStopping) if(theStateMachine_.getProvenanceStateName() == RunControlStateMachine::RUNNING_STATE_NAME || theStateMachine_.getProvenanceStateName() == RunControlStateMachine::PAUSED_STATE_NAME) { + writeRunInfoTransition(RunInfoVInterface::RunTransitionType::STOP_COMPLETE, ""); + + // Write consolidated end-of-run summary to ECL if enabled via env var and user didn't opt out + bool doLogConsolidated = true; //default to logging consolidated run summary try { - ConfigurationTree configLinkNode = - CorePropertySupervisorBase::theConfigurationManager_ - ->getSupervisorTableNode(supervisorContextUID_, - supervisorApplicationUID_); - if(!configLinkNode.isDisconnected()) + doLogConsolidated = __ENV__("OTS_LOG_CONSOLIDATED_RUN") == std::string("1"); + } + catch(...) + { /* ignore errors */ + ; + } + if(doLogConsolidated && activeStateMachineWriteToEcl_) + { + try { - __COUTV__(activeStateMachineName_); - ConfigurationTree fsmLinkNode = - configLinkNode.getNode("LinkToStateMachineTable") - .getNode(activeStateMachineName_); - std::string runInfoPluginType = - fsmLinkNode.getNode("RunInfoPluginType").getValue(); - __COUTV__(runInfoPluginType); - if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT && - runInfoPluginType != - TableViewColumnInfo::DATATYPE_STRING_ALT_DEFAULT && - runInfoPluginType != "No Run Info Plugin") + std::stringstream eclSs; + if(!activeStateMachineRawStartComment_.empty()) + eclSs << "Start: " << activeStateMachineRawStartComment_ << "\n"; + if(!activeStateMachineRawStopComment_.empty()) + eclSs << "Stop: " << activeStateMachineRawStopComment_ << "\n"; + + eclSs << "\nRun Number: " << activeStateMachineRunNumber_ << "\n"; + eclSs << "Run Type: " << activeStateMachineName_ << "/" + << activeStateMachineRunAlias_ << "\n"; + + eclSs << "\nStart Time: " + << StringMacros::getTimestampString( + activeStateMachineRunWallClockStartTime_) + << "\n"; + time_t endTime = time(0); + eclSs << "End Time: " << StringMacros::getTimestampString(endTime) + << "\n"; { - std::unique_ptr runInfoInterface = nullptr; - try - { - runInfoInterface.reset( - makeRunInfo(runInfoPluginType, activeStateMachineName_)); - } - catch(...) - { - } + int dur = activeStateMachineRunDuration_ms; + int dur_s = dur / 1000; + dur = dur % 1000; + int dur_m = dur_s / 60; + dur_s = dur_s % 60; + int dur_h = dur_m / 60; + dur_m = dur_m % 60; + eclSs << "Duration: " << std::setw(2) << std::setfill('0') << dur_h + << ":" << std::setw(2) << std::setfill('0') << dur_m << ":" + << std::setw(2) << std::setfill('0') << dur_s << "\n"; + } - if(runInfoInterface == nullptr) + eclSs << "\nConfiguration: " << activeStateMachineConfigurationAlias_ + << " [" << theConfigurationTableGroup_.first << "(" + << theConfigurationTableGroup_.second.str() << ")]\n"; + { + std::lock_guard lock(remoteGatewayAppsMutex_); + for(const auto& remote : remoteGatewayApps_) { - __SS__ << "Run Info interface plugin construction failed of type " - << runInfoPluginType << __E__; - __SS_THROW__; + if(!remote.fsm_included) + continue; + eclSs << " " << remote.appInfo.name << ": " + << remote.selected_config_alias << "\n"; } - - runInfoInterface->updateRunInfo( - activeStateMachineRunConditionID_, - RunInfoVInterface::RunTransitionType::STOP, - getLastLogEntry(RunControlStateMachine::STOP_TRANSITION_NAME)); } - } - else - __COUT__ << "Gateway Supervisor configuration record not found at '" - << ConfigurationManager::XDAQ_CONTEXT_TABLE_NAME << "/" - << supervisorContextUID_ << "/" << supervisorApplicationUID_ - << "' - consider adding one to control configuration dumps " - "and state machine properties." - << __E__; - } - catch(const std::runtime_error& e) - { - __SS__ - << "RUN INFO CONFIGURED STATE INSERT OR UPDATE INTO DATABASE FAILED!!! " - << e.what() << __E__; - __SS_THROW__; - } - catch(...) - { - __SS__ - << "RUN INFO CONFIGURED STATE INSERT OR UPDATE INTO DATABASE FAILED!!! " - << __E__; - try - { - throw; - } //one more try to printout extra info - catch(const std::exception& e) - { - ss << "Exception message: " << e.what(); + + makeSystemLogEntry( + eclSs.str(), + activeStateMachineRunAlias_ + " " + activeStateMachineRunNumber_, + true /* skipFooter */); } catch(...) { + __COUT_WARN__ << "Failed to write end-of-run ECL entry." << __E__; } - __SS_THROW__; - } // End write run info into db - } // end update Run Info handling + } + } } // end stateConfigured() @@ -7638,8 +7585,9 @@ try } // end make logbook entry RunControlStateMachine::theProgressBar_.step(); - activeStateMachineRunStartTime = std::chrono::steady_clock::now(); - activeStateMachineRunDuration_ms = 0; + activeStateMachineRunStartTime = std::chrono::steady_clock::now(); + activeStateMachineRunWallClockStartTime_ = time(0); + activeStateMachineRunDuration_ms = 0; broadcastMessage( theStateMachine_ .getCurrentMessage()); // ---------------------------------- broadcast! @@ -8116,6 +8064,19 @@ try std::chrono::steady_clock::now() - activeStateMachineRunStartTime) .count(); + // Write STOP to DB before the broadcast so the record exists even if the transition fails. + // A STOP_COMPLETE record is written at the end of the transition in stateConfigured(). + try + { + writeRunInfoTransition(RunInfoVInterface::RunTransitionType::STOP, + activeStateMachineRawStopComment_); + } + catch(...) + { + __COUT_WARN__ << "STOP transition DB write failed — will not prevent transition." + << __E__; + } + RunControlStateMachine::theProgressBar_.step(); bool doLog = false; @@ -13409,6 +13370,72 @@ void GatewaySupervisor::setNextRunNumber(unsigned int runNumber, runNumberFile.close(); } // end setNextRunNumber() +//============================================================================== +void GatewaySupervisor::writeRunInfoTransition( + RunInfoVInterface::RunTransitionType transitionType, const std::string& comment) +{ + try + { + ConfigurationTree configLinkNode = + CorePropertySupervisorBase::theConfigurationManager_->getSupervisorTableNode( + supervisorContextUID_, supervisorApplicationUID_); + if(!configLinkNode.isDisconnected()) + { + ConfigurationTree fsmLinkNode = + configLinkNode.getNode("LinkToStateMachineTable") + .getNode(activeStateMachineName_); + std::string runInfoPluginType = + fsmLinkNode.getNode("RunInfoPluginType").getValue(); + if(runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_DEFAULT && + runInfoPluginType != TableViewColumnInfo::DATATYPE_STRING_ALT_DEFAULT && + runInfoPluginType != "No Run Info Plugin") + { + std::unique_ptr runInfoInterface = nullptr; + try + { + runInfoInterface.reset( + makeRunInfo(runInfoPluginType, activeStateMachineName_)); + } + catch(...) + { + } + + if(runInfoInterface == nullptr) + { + __SS__ << "Run Info interface plugin construction failed of type " + << runInfoPluginType << __E__; + __SS_THROW__; + } + + runInfoInterface->updateRunInfo( + activeStateMachineRunConditionID_, transitionType, comment); + } + } + } + catch(const std::runtime_error& e) + { + __SS__ << "RUN INFO TRANSITION UPDATE INTO DATABASE FAILED!!! " << e.what() + << __E__; + __SS_THROW__; + } + catch(...) + { + __SS__ << "RUN INFO TRANSITION UPDATE INTO DATABASE FAILED!!! " << __E__; + try + { + throw; + } + catch(const std::exception& e) + { + ss << "Exception message: " << e.what(); + } + catch(...) + { + } + __SS_THROW__; + } +} // end writeRunInfoTransition() + //============================================================================== /// getLastLogEntry /// diff --git a/otsdaq/GatewaySupervisor/GatewaySupervisor.h b/otsdaq/GatewaySupervisor/GatewaySupervisor.h index ee1bf663..f93f663f 100644 --- a/otsdaq/GatewaySupervisor/GatewaySupervisor.h +++ b/otsdaq/GatewaySupervisor/GatewaySupervisor.h @@ -7,6 +7,7 @@ #include "otsdaq/CoreSupervisors/ConfigurationSupervisorBase.h" #include "otsdaq/CoreSupervisors/CorePropertySupervisorBase.h" #include "otsdaq/FiniteStateMachine/RunControlStateMachine.h" +#include "otsdaq/FiniteStateMachine/RunInfoVInterface.h" #include "otsdaq/GatewaySupervisor/Iterator.h" #include "otsdaq/SOAPUtilities/SOAPMessenger.h" #include "otsdaq/SupervisorInfo/AllSupervisorInfo.h" @@ -146,7 +147,7 @@ class WorkLoopManager; void transitionStartingUp(toolbox::Event::Reference e) override; void enteringError(toolbox::Event::Reference e) override; - void makeSystemLogEntry(const std::string& entryText, const std::string& subjectText = ""); + void makeSystemLogEntry(const std::string& entryText, const std::string& subjectText = "", bool skipFooter = false); static void addSystemMessage(std::string toUserCSV, std::string message); void checkForAsyncError(void); @@ -161,6 +162,7 @@ class WorkLoopManager; void setNextRunNumber (unsigned int runNumber, const std::string& fsmName = ""); std::string getLastLogEntry (const std::string& logType, const std::string& fsmName = ""); void setLastLogEntry (const std::string& logType, const std::string& logEntry, const std::string& fsmName = ""); + void writeRunInfoTransition (RunInfoVInterface::RunTransitionType transitionType, const std::string& comment); static xoap::MessageReference lastTableGroupRequestHandler (const SOAPParameters& parameters); @@ -340,11 +342,14 @@ class WorkLoopManager; std::string activeStateMachineRunInfoPluginType_; /// stateMachineConfigureLogEntry_, stateMachineStartLogEntry_, stateMachineStopLogEntry_; + std::string activeStateMachineRawStartComment_, activeStateMachineRawStopComment_; std::string activeStateMachineRunNumber_, activeStateMachineRunAlias_, activeStateMachineConfigurationAlias_; bool activeStateMachineRollOverLogOnConfigure_, activeStateMachineRollOverLogOnStart_; std::chrono::steady_clock::time_point activeStateMachineRunStartTime; + time_t activeStateMachineRunWallClockStartTime_ = 0; int activeStateMachineRunDuration_ms; ///< For paused runs, don't count time spent in pause state + bool activeStateMachineWriteToEcl_ = true; unsigned int activeStateMachineConfigureConditionID_, activeStateMachineRunConditionID_; std::string activeStateMachineSubsystemCommonList_, activeStateMachineSubsystemCommonOverrideList_; ///