Extend tests of error cases

- test raising SIGINT in every state
 - test going to Error state from every state
 - add new states (bind/connect) to exception tests

fairmq/StateMachine.cxx

@@ -28,6 +28,7 @@
 #include <array>
 #include <unordered_map>
 #include <mutex>
+#include <stdexcept>
 
 using namespace std;
 using namespace boost::msm;
@@ -299,6 +300,10 @@ struct Machine_ : public state_machine_def<Machine_>
                 fWorkDoneCV.notify_one();
             }
         }
+
+        if (fState == State::Error) {
+            throw StateMachine::ErrorStateException("Device transitioned to error state");
+        }
     }
 
     // replaces the default no-transition response.
@@ -463,7 +468,11 @@ void StateMachine::ProcessWork()
     try {
         fsm->CallStateChangeCallbacks(State::Idle);
         fsm->ProcessWork();
+    } catch(ErrorStateException& ese) {
+        LOG(trace) << "ErrorStateException caught in ProcessWork(), rethrowing";
+        throw;
     } catch(...) {
+        LOG(debug) << "Exception caught in ProcessWork(), going to Error state and rethrowing";
         {
             lock_guard<mutex> lock(fsm->fStateMtx);
             fsm->fState = State::Error;
@@ -480,3 +489,4 @@ string StateMachine::GetStateName(const State state) { return stateNames.at(stat
 string StateMachine::GetTransitionName(const Transition transition) { return transitionNames.at(static_cast<int>(transition)); }
 State StateMachine::GetState(const string& state) { return stateNumbers.at(state); }
 Transition StateMachine::GetTransition(const string& transition) { return transitionNumbers.at(transition); }
+

fairmq/StateMachine.h

@@ -94,6 +94,8 @@ class StateMachine
     static State GetState(const std::string& state);
     static Transition GetTransition(const std::string& transition);
 
+    struct ErrorStateException : std::runtime_error { using std::runtime_error::runtime_error; };
+
   private:
     std::shared_ptr<void> fFsm;
 };

fairmq/plugins/Control.cxx

@@ -29,8 +29,7 @@ namespace
         ++gSignalCount;
         gLastSignal = signal;
 
-        if (gSignalCount > 1)
-        {
+        if (gSignalCount > 1) {
             std::abort();
         }
     }
@@ -57,11 +56,18 @@ Control::Control(const string& name, const Plugin::Version version, const string
 {
     SubscribeToDeviceStateChange([&](DeviceState newState) {
         LOG(trace) << "control plugin notified on new state: " << newState;
+
         {
             lock_guard<mutex> lock{fEventsMutex};
             fEvents.push(newState);
         }
         fNewEvent.notify_one();
+
+        if (newState == DeviceState::Error) {
+            fPluginShutdownRequested = true;
+            fDeviceShutdownRequested = true;
+            // throw DeviceErrorState("Controlled device transitioned to error state.");
+        }
     });
 
     try {
@@ -110,6 +116,25 @@ auto Control::RunStartupSequence() -> void
     while (WaitForNextState() != DeviceState::Running) {}
 }
 
+auto Control::WaitForNextState() -> DeviceState
+{
+    unique_lock<mutex> lock{fEventsMutex};
+    while (fEvents.empty()) {
+        fNewEvent.wait_for(lock, chrono::milliseconds(50));
+    }
+
+    auto result = fEvents.front();
+
+    if (result == DeviceState::Error) {
+        ReleaseDeviceControl();
+        throw DeviceErrorState("Controlled device transitioned to error state.");
+    }
+
+    fEvents.pop();
+
+    return result;
+}
+
 auto ControlPluginProgramOptions() -> Plugin::ProgOptions
 {
     namespace po = boost::program_options;
@@ -248,6 +273,7 @@ try {
         }
 
         if (GetCurrentDeviceState() == DeviceState::Error) {
+            ReleaseDeviceControl();
             throw DeviceErrorState("Controlled device transitioned to error state.");
         }
 
@@ -363,64 +389,38 @@ void Control::PrintStateMachine()
     cout << ss.str() << flush;
 }
 
-auto Control::WaitForNextState() -> DeviceState
-{
-    unique_lock<mutex> lock{fEventsMutex};
-    while (fEvents.empty()) {
-        fNewEvent.wait_for(lock, chrono::milliseconds(50));
-    }
-
-    auto result = fEvents.front();
-
-    if (result == DeviceState::Error) {
-        throw DeviceErrorState("Controlled device transitioned to error state.");
-    }
-
-    fEvents.pop();
-
-    return result;
-}
-
 auto Control::StaticMode() -> void
-try
-{
+try {
     RunStartupSequence();
 
     {
         // Wait for next state, which is DeviceState::Ready,
         // or for device shutdown request (Ctrl-C)
         unique_lock<mutex> lock{fEventsMutex};
-        while (fEvents.empty() && !fDeviceShutdownRequested)
-        {
+        while (fEvents.empty() && !fDeviceShutdownRequested) {
             fNewEvent.wait_for(lock, chrono::milliseconds(50));
         }
 
-        if (fEvents.front() == DeviceState::Error)
-        {
+        if (fEvents.front() == DeviceState::Error) {
+            ReleaseDeviceControl();
             throw DeviceErrorState("Controlled device transitioned to error state.");
         }
     }
 
     RunShutdownSequence();
-}
-catch (PluginServices::DeviceControlError& e)
-{
+} catch (PluginServices::DeviceControlError& e) {
     // If we are here, it means another plugin has taken control. That's fine, just print the exception message and do nothing else.
     LOG(debug) << e.what();
-}
-catch (DeviceErrorState&)
-{
+} catch (DeviceErrorState&) {
 }
 
 auto Control::SignalHandler() -> void
 {
-    while (gSignalCount == 0 && !fPluginShutdownRequested)
-    {
+    while (gSignalCount == 0 && !fPluginShutdownRequested) {
         this_thread::sleep_for(chrono::milliseconds(100));
     }
 
-    if (!fPluginShutdownRequested)
-    {
+    if (!fPluginShutdownRequested) {
         LOG(info) << "Received device shutdown request (signal " << gLastSignal << ").";
         LOG(info) << "Waiting for graceful device shutdown. Hit Ctrl-C again to abort immediately.";
 
@@ -431,20 +431,14 @@ auto Control::SignalHandler() -> void
             if (fControllerThread.joinable()) fControllerThread.join();
         }
 
-        if (!fDeviceHasShutdown)
-        {
+        if (!fDeviceHasShutdown) {
             // Take over control and attempt graceful shutdown
             StealDeviceControl();
-            try
-            {
+            try {
                 RunShutdownSequence();
-            }
-            catch (PluginServices::DeviceControlError& e)
-            {
+            } catch (PluginServices::DeviceControlError& e) {
                 LOG(info) << "Graceful device shutdown failed: " << e.what() << " If hanging, hit Ctrl-C again to abort immediately.";
-            }
-            catch (...)
-            {
+            } catch (...) {
                 LOG(info) << "Graceful device shutdown failed. If hanging, hit Ctrl-C again to abort immediately.";
             }
         }