Adding multiple transports support & other fixes:

- Avoid polling when only one input channel is used.
 - Send only handles for shared memory transport.
 - Avoid waiting in the rate logger thread when nothing to log.
 - Hide warnings from generated files
 - Fix #483

fairmq/CMakeLists.txt

@@ -111,6 +111,7 @@ EndIf(NANOMSG_FOUND)
 # manual install (globbing add not recommended)
 Set(FAIRMQHEADERS
   FairMQParts.h
+  FairMQTransports.h
   FairMQConfigPlugin.h
   FairMQControlPlugin.h
   runFairMQDevice.h

fairmq/FairMQChannel.cxx

@@ -13,11 +13,11 @@
  */
 
 #include <set>
+#include <utility> // std::move
 
 #include <boost/algorithm/string.hpp> // join/split
 
 #include "FairMQChannel.h"
-#include "FairMQLogger.h"
 
 using namespace std;
 
@@ -30,18 +30,21 @@ FairMQChannel::FairMQChannel()
     , fType("unspecified")
     , fMethod("unspecified")
     , fAddress("unspecified")
+    , fTransport("default")
     , fSndBufSize(1000)
     , fRcvBufSize(1000)
     , fSndKernelSize(0)
     , fRcvKernelSize(0)
     , fRateLogging(1)
-    , fChannelName("")
+    , fName("")
     , fIsValid(false)
     , fPoller(nullptr)
-    , fCmdSocket(nullptr)
+    , fChannelCmdSocket(nullptr)
+    , fTransportType(FairMQ::Transport::DEFAULT)
     , fTransportFactory(nullptr)
     , fNoBlockFlag(0)
     , fSndMoreFlag(0)
+    , fMultipart(false)
 {
 }
 
@@ -50,18 +53,21 @@ FairMQChannel::FairMQChannel(const string& type, const string& method, const str
     , fType(type)
     , fMethod(method)
     , fAddress(address)
+    , fTransport("default")
     , fSndBufSize(1000)
     , fRcvBufSize(1000)
     , fSndKernelSize(0)
     , fRcvKernelSize(0)
     , fRateLogging(1)
-    , fChannelName("")
+    , fName("")
     , fIsValid(false)
     , fPoller(nullptr)
-    , fCmdSocket(nullptr)
+    , fChannelCmdSocket(nullptr)
+    , fTransportType(FairMQ::Transport::DEFAULT)
     , fTransportFactory(nullptr)
     , fNoBlockFlag(0)
     , fSndMoreFlag(0)
+    , fMultipart(false)
 {
 }
 
@@ -70,18 +76,21 @@ FairMQChannel::FairMQChannel(const FairMQChannel& chan)
     , fType(chan.fType)
     , fMethod(chan.fMethod)
     , fAddress(chan.fAddress)
+    , fTransport(chan.fTransport)
     , fSndBufSize(chan.fSndBufSize)
     , fRcvBufSize(chan.fRcvBufSize)
     , fSndKernelSize(chan.fSndKernelSize)
     , fRcvKernelSize(chan.fRcvKernelSize)
     , fRateLogging(chan.fRateLogging)
-    , fChannelName(chan.fChannelName)
+    , fName(chan.fName)
     , fIsValid(false)
     , fPoller(nullptr)
-    , fCmdSocket(nullptr)
+    , fChannelCmdSocket(nullptr)
+    , fTransportType(FairMQ::Transport::DEFAULT)
     , fTransportFactory(nullptr)
     , fNoBlockFlag(chan.fNoBlockFlag)
     , fSndMoreFlag(chan.fSndMoreFlag)
+    , fMultipart(chan.fMultipart)
 {}
 
 FairMQChannel& FairMQChannel::operator=(const FairMQChannel& chan)
@@ -89,16 +98,18 @@ FairMQChannel& FairMQChannel::operator=(const FairMQChannel& chan)
     fType = chan.fType;
     fMethod = chan.fMethod;
     fAddress = chan.fAddress;
+    fTransport = chan.fTransport;
     fSndBufSize = chan.fSndBufSize;
     fRcvBufSize = chan.fRcvBufSize;
     fSndKernelSize = chan.fSndKernelSize;
     fRcvKernelSize = chan.fRcvKernelSize;
     fRateLogging = chan.fRateLogging;
     fSocket = nullptr;
-    fChannelName = chan.fChannelName;
+    fName = chan.fName;
     fIsValid = false;
     fPoller = nullptr;
-    fCmdSocket = nullptr;
+    fChannelCmdSocket = nullptr;
+    fTransportType = FairMQ::Transport::DEFAULT;
     fTransportFactory = nullptr;
     fNoBlockFlag = chan.fNoBlockFlag;
     fSndMoreFlag = chan.fSndMoreFlag;
@@ -108,13 +119,13 @@ FairMQChannel& FairMQChannel::operator=(const FairMQChannel& chan)
 
 string FairMQChannel::GetChannelName() const
 {
-    return fChannelName;
+    return fName;
 }
 
 string FairMQChannel::GetChannelPrefix() const
 {
-    string prefix = fChannelName;
+    string prefix = fName;
-    return prefix.erase(fChannelName.rfind("["));
+    return prefix.erase(fName.rfind("["));
 }
 
 string FairMQChannel::GetType() const
@@ -159,6 +170,20 @@ string FairMQChannel::GetAddress() const
     }
 }
 
+string FairMQChannel::GetTransport() const
+{
+    try
+    {
+        unique_lock<mutex> lock(fChannelMutex);
+        return fTransport;
+    }
+    catch (exception& e)
+    {
+        LOG(ERROR) << "Exception caught in FairMQChannel::GetTransport: " << e.what();
+        exit(EXIT_FAILURE);
+    }
+}
+
 int FairMQChannel::GetSndBufSize() const
 {
     try
@@ -274,6 +299,21 @@ void FairMQChannel::UpdateAddress(const string& address)
     }
 }
 
+void FairMQChannel::UpdateTransport(const string& transport)
+{
+    try
+    {
+        unique_lock<mutex> lock(fChannelMutex);
+        fIsValid = false;
+        fTransport = transport;
+    }
+    catch (exception& e)
+    {
+        LOG(ERROR) << "Exception caught in FairMQChannel::UpdateTransport: " << e.what();
+        exit(EXIT_FAILURE);
+    }
+}
+
 void FairMQChannel::UpdateSndBufSize(const int sndBufSize)
 {
     try
@@ -370,7 +410,7 @@ bool FairMQChannel::ValidateChannel()
         unique_lock<mutex> lock(fChannelMutex);
 
         stringstream ss;
-        ss << "Validating channel \"" << fChannelName << "\"... ";
+        ss << "Validating channel \"" << fName << "\"... ";
 
         if (fIsValid)
         {
@@ -461,6 +501,17 @@ bool FairMQChannel::ValidateChannel()
             }
         }
 
+        // validate channel transport
+        // const string channelTransportNames[] = { "default", "zeromq", "nanomsg", "shmem" };
+        // const set<string> channelTransports(channelTransportNames, channelTransportNames + sizeof(channelTransportNames) / sizeof(string));
+        if (FairMQ::TransportTypes.find(fTransport) == FairMQ::TransportTypes.end())
+        {
+            ss << "INVALID";
+            LOG(DEBUG) << ss.str();
+            LOG(ERROR) << "Invalid channel transport: \"" << fTransport << "\"";
+            exit(EXIT_FAILURE);
+        }
+
         // validate socket buffer size for sending
         if (fSndBufSize < 0)
         {
@@ -518,19 +569,23 @@ bool FairMQChannel::ValidateChannel()
     }
 }
 
-bool FairMQChannel::InitCommandInterface(shared_ptr<FairMQTransportFactory> factory, int numIoThreads)
+void FairMQChannel::InitTransport(shared_ptr<FairMQTransportFactory> factory)
 {
     fTransportFactory = factory;
+    fTransportType = factory->GetType();
+}
 
-    fCmdSocket = fTransportFactory->CreateSocket("sub", "device-commands", numIoThreads, "internal");
+bool FairMQChannel::InitCommandInterface(int numIoThreads)
-    if (fCmdSocket)
 {
-        fCmdSocket->Connect("inproc://commands");
+    fChannelCmdSocket = fTransportFactory->CreateSocket("sub", "device-commands", numIoThreads, "internal");
+    if (fChannelCmdSocket)
+    {
+        fChannelCmdSocket->Connect("inproc://commands");
 
-        fNoBlockFlag = fCmdSocket->NOBLOCK;
+        fNoBlockFlag = fChannelCmdSocket->NOBLOCK;
-        fSndMoreFlag = fCmdSocket->SNDMORE;
+        fSndMoreFlag = fChannelCmdSocket->SNDMORE;
 
-        fPoller = fTransportFactory->CreatePoller(*fCmdSocket, *fSocket);
+        fPoller = fTransportFactory->CreatePoller(*fChannelCmdSocket, *fSocket);
 
         return true;
     }
@@ -547,7 +602,19 @@ void FairMQChannel::ResetChannel()
     // TODO: implement channel resetting
 }
 
-int FairMQChannel::Send(const unique_ptr<FairMQMessage>& msg, int sndTimeoutInMs) const
+int FairMQChannel::Send(unique_ptr<FairMQMessage>& msg) const
+{
+    CheckCompatibility(msg);
+    return fSocket->Send(msg);
+}
+
+int FairMQChannel::Receive(unique_ptr<FairMQMessage>& msg) const
+{
+    CheckCompatibility(msg);
+    return fSocket->Receive(msg);
+}
+
+int FairMQChannel::Send(unique_ptr<FairMQMessage>& msg, int sndTimeoutInMs) const
 {
     fPoller->Poll(sndTimeoutInMs);
 
@@ -562,13 +629,13 @@ int FairMQChannel::Send(const unique_ptr<FairMQMessage>& msg, int sndTimeoutInMs
 
     if (fPoller->CheckOutput(1))
     {
-        return fSocket->Send(msg.get(), 0);
+        return Send(msg);
     }
 
     return -2;
 }
 
-int FairMQChannel::Receive(const unique_ptr<FairMQMessage>& msg, int rcvTimeoutInMs) const
+int FairMQChannel::Receive(unique_ptr<FairMQMessage>& msg, int rcvTimeoutInMs) const
 {
     fPoller->Poll(rcvTimeoutInMs);
 
@@ -583,13 +650,37 @@ int FairMQChannel::Receive(const unique_ptr<FairMQMessage>& msg, int rcvTimeoutI
 
     if (fPoller->CheckInput(1))
     {
-        return fSocket->Receive(msg.get(), 0);
+        return Receive(msg);
     }
 
     return -2;
 }
 
-int64_t FairMQChannel::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, int sndTimeoutInMs) const
+int FairMQChannel::SendAsync(unique_ptr<FairMQMessage>& msg) const
+{
+    CheckCompatibility(msg);
+    return fSocket->Send(msg, fNoBlockFlag);
+}
+
+int FairMQChannel::ReceiveAsync(unique_ptr<FairMQMessage>& msg) const
+{
+    CheckCompatibility(msg);
+    return fSocket->Receive(msg, fNoBlockFlag);
+}
+
+int64_t FairMQChannel::Send(vector<unique_ptr<FairMQMessage>>& msgVec) const
+{
+    CheckCompatibility(msgVec);
+    return fSocket->Send(msgVec);
+}
+
+int64_t FairMQChannel::Receive(vector<unique_ptr<FairMQMessage>>& msgVec) const
+{
+    CheckCompatibility(msgVec);
+    return fSocket->Receive(msgVec);
+}
+
+int64_t FairMQChannel::Send(vector<unique_ptr<FairMQMessage>>& msgVec, int sndTimeoutInMs) const
 {
     fPoller->Poll(sndTimeoutInMs);
 
@@ -604,7 +695,7 @@ int64_t FairMQChannel::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, int
 
     if (fPoller->CheckOutput(1))
     {
-        return fSocket->Send(msgVec);
+        return Send(msgVec);
     }
 
     return -2;
@@ -625,155 +716,33 @@ int64_t FairMQChannel::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, int rc
 
     if (fPoller->CheckInput(1))
     {
-        return fSocket->Receive(msgVec);
+        return Receive(msgVec);
     }
 
     return -2;
 }
 
-int FairMQChannel::Send(FairMQMessage* msg, const string& flag, int sndTimeoutInMs) const
+int64_t FairMQChannel::SendAsync(vector<unique_ptr<FairMQMessage>>& msgVec) const
 {
-    if (flag == "")
+    CheckCompatibility(msgVec);
-    {
+    return fSocket->Send(msgVec, fNoBlockFlag);
-        fPoller->Poll(sndTimeoutInMs);
-
-        if (fPoller->CheckInput(0))
-        {
-            HandleUnblock();
-            if (fInterrupted)
-            {
-                return -2;
-
-            }
 }
 
-        if (fPoller->CheckOutput(1))
+/// Receives a vector of messages in non-blocking mode.
+///
+/// @param msgVec message vector reference
+/// @return Number of bytes that have been received. If queue is empty, returns -2.
+/// In case of errors, returns -1.
+int64_t FairMQChannel::ReceiveAsync(vector<unique_ptr<FairMQMessage>>& msgVec) const
 {
-            return fSocket->Send(msg, flag);
+    CheckCompatibility(msgVec);
-        }
+    return fSocket->Receive(msgVec, fNoBlockFlag);
-
-        return -2;
-    }
-    else
-    {
-        return fSocket->Send(msg, flag);
-    }
-}
-
-int FairMQChannel::Send(FairMQMessage* msg, const int flags, int sndTimeoutInMs) const
-{
-    if (flags == 0)
-    {
-        fPoller->Poll(sndTimeoutInMs);
-
-        if (fPoller->CheckInput(0))
-        {
-            HandleUnblock();
-            if (fInterrupted)
-            {
-                return -2;
-
-            }
-        }
-
-        if (fPoller->CheckOutput(1))
-        {
-            return fSocket->Send(msg, flags);
-        }
-
-        return -2;
-    }
-    else
-    {
-        return fSocket->Send(msg, flags);
-    }
-}
-
-int FairMQChannel::Receive(FairMQMessage* msg, const string& flag, int rcvTimeoutInMs) const
-{
-    if (flag == "")
-    {
-        fPoller->Poll(rcvTimeoutInMs);
-
-        if (fPoller->CheckInput(0))
-        {
-            HandleUnblock();
-            if (fInterrupted)
-            {
-                return -2;
-
-            }
-        }
-
-        if (fPoller->CheckInput(1))
-        {
-            return fSocket->Receive(msg, flag);
-        }
-
-        return -2;
-    }
-    else
-    {
-        return fSocket->Receive(msg, flag);
-    }
-}
-
-int FairMQChannel::Receive(FairMQMessage* msg, const int flags, int rcvTimeoutInMs) const
-{
-    if (flags == 0)
-    {
-        fPoller->Poll(rcvTimeoutInMs);
-
-        if (fPoller->CheckInput(0))
-        {
-            HandleUnblock();
-            if (fInterrupted)
-            {
-                return -2;
-
-            }
-        }
-
-        if (fPoller->CheckInput(1))
-        {
-            return fSocket->Receive(msg, flags);
-        }
-
-        return -2;
-    }
-    else
-    {
-        return fSocket->Receive(msg, flags);
-    }
-}
-
-bool FairMQChannel::ExpectsAnotherPart() const
-{
-    int64_t more = 0;
-    size_t more_size = sizeof more;
-
-    if (fSocket)
-    {
-        fSocket->GetOption("rcv-more", &more, &more_size);
-        if (more)
-        {
-            return true;
-        }
-        else
-        {
-            return false;
-        }
-    }
-    else
-    {
-        return false;
-    }
 }
 
 inline bool FairMQChannel::HandleUnblock() const
 {
     FairMQMessagePtr cmd(fTransportFactory->CreateMessage());
-    if (fCmdSocket->Receive(cmd.get(), 0) >= 0)
+    if (fChannelCmdSocket->Receive(cmd) >= 0)
     {
         // LOG(DEBUG) << "unblocked";
     }
@@ -788,3 +757,51 @@ void FairMQChannel::Tokenize(vector<string>& output, const string& input, const
 {
     boost::algorithm::split(output, input, boost::algorithm::is_any_of(delimiters));
 }
+
+FairMQTransportFactory* FairMQChannel::Transport()
+{
+    return fTransportFactory.get();
+}
+
+bool FairMQChannel::CheckCompatibility(unique_ptr<FairMQMessage>& msg) const
+{
+    if (fTransportType == msg->GetType())
+    {
+        return true;
+    }
+    else
+    {
+        // LOG(WARN) << "Channel type does not match message type. Copying...";
+        FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage(msg->GetSize()));
+        memcpy(msgCopy->GetData(), msg->GetData(), msg->GetSize());
+        msg = move(msgCopy);
+        return false;
+    }
+}
+
+bool FairMQChannel::CheckCompatibility(vector<unique_ptr<FairMQMessage>>& msgVec) const
+{
+    if (msgVec.size() > 0)
+    {
+        if (fTransportType == msgVec.at(0)->GetType())
+        {
+            return true;
+        }
+        else
+        {
+            // LOG(WARN) << "Channel type does not match message type. Copying...";
+            vector<unique_ptr<FairMQMessage>> tempVec;
+            for (unsigned int i = 0; i < msgVec.size(); ++i)
+            {
+                tempVec.push_back(fTransportFactory->CreateMessage(msgVec[i]->GetSize()));
+                memcpy(tempVec[i]->GetData(), msgVec[i]->GetData(), msgVec[i]->GetSize());
+            }
+            msgVec = move(tempVec);
+            return false;
+        }
+    }
+    else
+    {
+        return true;
+    }
+}

fairmq/FairMQChannel.h

@@ -17,12 +17,15 @@
 
 #include <string>
 #include <memory> // unique_ptr
+#include <vector>
 #include <atomic>
 #include <mutex>
 
 #include "FairMQTransportFactory.h"
 #include "FairMQSocket.h"
 #include "FairMQPoller.h"
+#include "FairMQTransports.h"
+#include "FairMQLogger.h"
 
 class FairMQPoller;
 class FairMQTransportFactory;
@@ -67,9 +70,13 @@ class FairMQChannel
     std::string GetMethod() const;
 
     /// Get socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
-    /// @return Returns socket type (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
+    /// @return Returns socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
     std::string GetAddress() const;
 
+    /// Get channel transport ("default", "zeromq", "nanomsg" or "shmem")
+    /// @return Returns channel transport (e.g. "default", "zeromq", "nanomsg" or "shmem")
+    std::string GetTransport() const;
+
     /// Get socket send buffer size (in number of messages)
     /// @return Returns socket send buffer size (in number of messages)
     int GetSndBufSize() const;
@@ -99,9 +106,13 @@ class FairMQChannel
     void UpdateMethod(const std::string& method);
 
     /// Set socket address
-    /// @param address Socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
+    /// @param Socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
     void UpdateAddress(const std::string& address);
 
+    /// Set channel transport
+    /// @param transport transport string ("default", "zeromq", "nanomsg" or "shmem")
+    void UpdateTransport(const std::string& transport);
+
     /// Set socket send buffer size
     /// @param sndBufSize Socket send buffer size (in number of messages)
     void UpdateSndBufSize(const int sndBufSize);
@@ -135,6 +146,9 @@ class FairMQChannel
 
     std::unique_ptr<FairMQSocket> fSocket;
 
+    int Send(std::unique_ptr<FairMQMessage>& msg) const;
+    int Receive(std::unique_ptr<FairMQMessage>& msg) const;
+
     /// Sends a message to the socket queue.
     /// @details Send method attempts to send a message by
     /// putting it in the output queue. If the queue is full or queueing is not possible
@@ -143,7 +157,16 @@ class FairMQChannel
     /// @param msg Constant reference of unique_ptr to a FairMQMessage
     /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
     /// In case of errors, returns -1.
-    int Send(const std::unique_ptr<FairMQMessage>& msg, int sndTimeoutInMs = -1) const;
+    int Send(std::unique_ptr<FairMQMessage>& msg, int sndTimeoutInMs) const;
+
+    /// Receives a message from the socket queue.
+    /// @details Receive method attempts to receive a message from the input queue.
+    /// If the queue is empty the method blocks.
+    ///
+    /// @param msg Constant reference of unique_ptr to a FairMQMessage
+    /// @return Number of bytes that have been received. -2 If reading from the queue was not possible or timed out.
+    /// In case of errors, returns -1.
+    int Receive(std::unique_ptr<FairMQMessage>& msg, int rcvTimeoutInMs) const;
 
     /// Sends a message in non-blocking mode.
     /// @details SendAsync method attempts to send a message without blocking by
@@ -153,41 +176,31 @@ class FairMQChannel
     /// @return Number of bytes that have been queued. If queueing failed due to
     /// full queue or no connected peers (when binding), returns -2.
     /// In case of errors, returns -1.
-    inline int SendAsync(const std::unique_ptr<FairMQMessage>& msg) const
+    int SendAsync(std::unique_ptr<FairMQMessage>& msg) const;
-    {
-        return fSocket->Send(msg.get(), fNoBlockFlag);
-    }
 
-    /// Queues the current message as a part of a multi-part message
+    /// Receives a message in non-blocking mode.
-    /// @details SendPart method queues the provided message as a part of a multi-part message.
-    /// The actual transfer over the network is initiated once final part has been queued with the Send() or SendAsync() methods.
     ///
     /// @param msg Constant reference of unique_ptr to a FairMQMessage
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible.
+    /// @return Number of bytes that have been received. If queue is empty, returns -2.
     /// In case of errors, returns -1.
-    inline int SendPart(const std::unique_ptr<FairMQMessage>& msg) const
+    int ReceiveAsync(std::unique_ptr<FairMQMessage>& msg) const;
-    {
-        return fSocket->Send(msg.get(), fSndMoreFlag);
-    }
 
-    /// Queues the current message as a part of a multi-part message without blocking
+    int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec) const;
-    /// @details SendPart method queues the provided message as a part of a multi-part message without blocking.
+    int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec) const;
-    /// The actual transfer over the network is initiated once final part has been queued with the Send() or SendAsync() methods.
-    ///
-    /// @param msg Constant reference of unique_ptr to a FairMQMessage
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible.
-    /// In case of errors, returns -1.
-    inline int SendPartAsync(const std::unique_ptr<FairMQMessage>& msg) const
-    {
-        return fSocket->Send(msg.get(), fSndMoreFlag|fNoBlockFlag);
-    }
 
     /// Send a vector of messages
     ///
     /// @param msgVec message vector reference
     /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
     /// In case of errors, returns -1.
-    int64_t Send(const std::vector<std::unique_ptr<FairMQMessage>>& msgVec, int sndTimeoutInMs = -1) const;
+    int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, int sndTimeoutInMs) const;
+
+    /// Receive a vector of messages
+    ///
+    /// @param msgVec message vector reference
+    /// @return Number of bytes that have been received. -2 If reading from the queue was not possible or timed out.
+    /// In case of errors, returns -1.
+    int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, int rcvTimeoutInMs) const;
 
     /// Sends a vector of message in non-blocking mode.
     /// @details SendAsync method attempts to send a vector of messages without blocking by
@@ -196,82 +209,49 @@ class FairMQChannel
     /// @param msgVec message vector reference
     /// @return Number of bytes that have been queued. If queueing failed due to
     /// full queue or no connected peers (when binding), returns -2. In case of errors, returns -1.
-    inline int64_t SendAsync(const std::vector<std::unique_ptr<FairMQMessage>>& msgVec) const
+    int64_t SendAsync(std::vector<std::unique_ptr<FairMQMessage>>& msgVec) const;
-    {
-        return fSocket->Send(msgVec, fNoBlockFlag);
-    }
-
-    /// Receives a message from the socket queue.
-    /// @details Receive method attempts to receive a message from the input queue.
-    /// If the queue is empty the method blocks.
-    ///
-    /// @param msg Constant reference of unique_ptr to a FairMQMessage
-    /// @return Number of bytes that have been received. -2 If reading from the queue was not possible or timed out.
-    /// In case of errors, returns -1.
-    int Receive(const std::unique_ptr<FairMQMessage>& msg, int rcvTimeoutInMs = -1) const;
-
-    /// Receives a message in non-blocking mode.
-    ///
-    /// @param msg Constant reference of unique_ptr to a FairMQMessage
-    /// @return Number of bytes that have been received. If queue is empty, returns -2.
-    /// In case of errors, returns -1.
-    inline int ReceiveAsync(const std::unique_ptr<FairMQMessage>& msg) const
-    {
-        return fSocket->Receive(msg.get(), fNoBlockFlag);
-    }
-
-    /// Receive a vector of messages
-    ///
-    /// @param msgVec message vector reference
-    /// @return Number of bytes that have been received. -2 If reading from the queue was not possible or timed out.
-    /// In case of errors, returns -1.
-    int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, int rcvTimeoutInMs = -1) const;
 
     /// Receives a vector of messages in non-blocking mode.
     ///
     /// @param msgVec message vector reference
     /// @return Number of bytes that have been received. If queue is empty, returns -2.
     /// In case of errors, returns -1.
-    inline int64_t ReceiveAsync(std::vector<std::unique_ptr<FairMQMessage>>& msgVec) const
+    int64_t ReceiveAsync(std::vector<std::unique_ptr<FairMQMessage>>& msgVec) const;
-    {
-        return fSocket->Receive(msgVec, fNoBlockFlag);
-    }
-
-    /// Checks if the socket is expecting to receive another part of a multipart message.
-    /// @return Return true if the socket expects another part of a multipart message and false otherwise.
-    bool ExpectsAnotherPart() const;
-
-    // DEPRECATED socket method wrappers with raw pointers and flag checks
-    int Send(FairMQMessage* msg, const std::string& flag = "", int sndTimeoutInMs = -1) const;
-    int Send(FairMQMessage* msg, const int flags, int sndTimeoutInMs = -1) const;
-    int Receive(FairMQMessage* msg, const std::string& flag = "", int rcvTimeoutInMs = -1) const;
-    int Receive(FairMQMessage* msg, const int flags, int rcvTimeoutInMs = -1) const;
 
     // TODO: this might go to some base utility library
     static void Tokenize(std::vector<std::string>& output, const std::string& input, const std::string delimiters = ",");
 
+    FairMQTransportFactory* Transport();
+
   private:
     std::string fType;
     std::string fMethod;
     std::string fAddress;
+    std::string fTransport;
     int fSndBufSize;
     int fRcvBufSize;
     int fSndKernelSize;
     int fRcvKernelSize;
     int fRateLogging;
 
-    std::string fChannelName;
+    std::string fName;
     std::atomic<bool> fIsValid;
 
-    FairMQPollerPtr fPoller;
-    FairMQSocketPtr fCmdSocket;
 
+    FairMQPollerPtr fPoller;
+    FairMQSocketPtr fChannelCmdSocket;
+
+    FairMQ::Transport fTransportType;
     std::shared_ptr<FairMQTransportFactory> fTransportFactory;
 
     int fNoBlockFlag;
     int fSndMoreFlag;
 
-    bool InitCommandInterface(std::shared_ptr<FairMQTransportFactory> factory, int numIoThreads);
+    bool CheckCompatibility(std::unique_ptr<FairMQMessage>& msg) const;
+    bool CheckCompatibility(std::vector<std::unique_ptr<FairMQMessage>>& msgVec) const;
+
+    void InitTransport(std::shared_ptr<FairMQTransportFactory> factory);
+    bool InitCommandInterface(int numIoThreads);
 
     bool HandleUnblock() const;
 
@@ -282,6 +262,7 @@ class FairMQChannel
     static std::mutex fChannelMutex;
 
     static std::atomic<bool> fInterrupted;
+    bool fMultipart;
 };
 
 #endif /* FAIRMQCHANNEL_H_ */

fairmq/FairMQDevice.cxx

@@ -13,7 +13,6 @@
  */
 
 #include <list>
-#include <algorithm> // std::sort()
 #include <csignal> // catching system signals
 #include <cstdlib>
 #include <stdexcept>
@@ -42,8 +41,8 @@
 
 using namespace std;
 
-// std::function and a wrapper to catch the signals
+// function and a wrapper to catch the signals
-std::function<void(int)> sigHandler;
+function<void(int)> sigHandler;
 static void CallSignalHandler(int signal)
 {
     sigHandler(signal);
@@ -54,13 +53,15 @@ FairMQDevice::FairMQDevice()
     , fConfig(nullptr)
     , fId()
     , fNetworkInterface()
+    , fDefaultTransport()
     , fMaxInitializationAttempts(120)
     , fNumIoThreads(1)
     , fPortRangeMin(22000)
     , fPortRangeMax(32000)
     , fLogIntervalInMs(1000)
-    , fCmdSocket(nullptr)
     , fTransportFactory(nullptr)
+    , fTransports()
+    , fDeviceCmdSockets()
     , fInitialValidationFinished(false)
     , fInitialValidationCondition()
     , fInitialValidationMutex()
@@ -70,6 +71,10 @@ FairMQDevice::FairMQDevice()
     , fDataCallbacks(false)
     , fMsgInputs()
     , fMultipartInputs()
+    , fMultitransportInputs()
+    , fInputChannelKeys()
+    , fMultitransportMutex()
+    , fMultitransportProceed(false)
 {
 }
 
@@ -78,8 +83,8 @@ void FairMQDevice::CatchSignals()
     if (!fCatchingSignals)
     {
         sigHandler = bind1st(mem_fun(&FairMQDevice::SignalHandler), this);
-        std::signal(SIGINT, CallSignalHandler);
+        signal(SIGINT, CallSignalHandler);
-        std::signal(SIGTERM, CallSignalHandler);
+        signal(SIGTERM, CallSignalHandler);
         fCatchingSignals = true;
     }
 }
@@ -109,12 +114,12 @@ void FairMQDevice::SignalHandler(int signal)
     else
     {
         LOG(WARN) << "Repeated termination or bad initialization? Aborting.";
-        std::abort();
+        abort();
         // exit(EXIT_FAILURE);
     }
 }
 
-void FairMQDevice::ConnectChannels(list<FairMQChannel*>& chans)
+void FairMQDevice::AttachChannels(list<FairMQChannel*>& chans)
 {
     auto itr = chans.begin();
 
@@ -124,38 +129,12 @@ void FairMQDevice::ConnectChannels(list<FairMQChannel*>& chans)
         {
             if (AttachChannel(**itr))
             {
-                (*itr)->InitCommandInterface(fTransportFactory, fNumIoThreads);
+                (*itr)->InitCommandInterface(fNumIoThreads);
                 chans.erase(itr++);
             }
             else
             {
-                LOG(ERROR) << "failed to connect channel " << (*itr)->fChannelName;
+                LOG(ERROR) << "failed to attach channel " << (*itr)->fName << " (" << (*itr)->fMethod << ")";
-                ++itr;
-            }
-        }
-        else
-        {
-            ++itr;
-        }
-    }
-}
-
-void FairMQDevice::BindChannels(list<FairMQChannel*>& chans)
-{
-    auto itr = chans.begin();
-
-    while (itr != chans.end())
-    {
-        if ((*itr)->ValidateChannel())
-        {
-            if (AttachChannel(**itr))
-            {
-                (*itr)->InitCommandInterface(fTransportFactory, fNumIoThreads);
-                chans.erase(itr++);
-            }
-            else
-            {
-                LOG(ERROR) << "failed to bind channel " << (*itr)->fChannelName;
                 ++itr;
             }
         }
@@ -174,10 +153,17 @@ void FairMQDevice::InitWrapper()
         exit(EXIT_FAILURE);
     }
 
-    if (!fCmdSocket)
+    if (fDeviceCmdSockets.empty())
     {
-        fCmdSocket = fTransportFactory->CreateSocket("pub", "device-commands", fNumIoThreads, fId);
+        auto p = fDeviceCmdSockets.emplace(fTransportFactory->GetType(), fTransportFactory->CreateSocket("pub", "device-commands", fNumIoThreads, fId));
-        fCmdSocket->Bind("inproc://commands");
+        if (p.second)
+        {
+            p.first->second->Bind("inproc://commands");
+        }
+        else
+        {
+            exit(EXIT_FAILURE);
+        }
 
         FairMQMessagePtr msg(fTransportFactory->CreateMessage());
         msg->SetDeviceId(fId);
@@ -191,13 +177,14 @@ void FairMQDevice::InitWrapper()
     for (auto mi = fChannels.begin(); mi != fChannels.end(); ++mi)
     {
         for (auto vi = (mi->second).begin(); vi != (mi->second).end(); ++vi)
-        {
-            if (vi->fMethod == "bind")
         {
             // set channel name: name + vector index
             stringstream ss;
             ss << mi->first << "[" << vi - (mi->second).begin() << "]";
-                vi->fChannelName = ss.str();
+            vi->fName = ss.str();
+
+            if (vi->fMethod == "bind")
+            {
                 // if binding address is not specified, set it up to try getting it from the configured network interface
                 if (vi->fAddress == "unspecified" || vi->fAddress == "")
                 {
@@ -208,19 +195,11 @@ void FairMQDevice::InitWrapper()
             }
             else if (vi->fMethod == "connect")
             {
-                // set channel name: name + vector index
-                stringstream ss;
-                ss << mi->first << "[" << vi - (mi->second).begin() << "]";
-                vi->fChannelName = ss.str();
                 // fill the uninitialized list
                 uninitializedConnectingChannels.push_back(&(*vi));
             }
             else if (vi->fAddress.find_first_of("@+>") != string::npos)
             {
-                // set channel name: name + vector index
-                stringstream ss;
-                ss << mi->first << "[" << vi - (mi->second).begin() << "]";
-                vi->fChannelName = ss.str();
                 // fill the uninitialized list
                 uninitializedConnectingChannels.push_back(&(*vi));
             }
@@ -234,7 +213,7 @@ void FairMQDevice::InitWrapper()
 
     // Bind channels. Here one run is enough, because bind settings should be available locally
     // If necessary this could be handled in the same way as the connecting channels
-    BindChannels(uninitializedBindingChannels);
+    AttachChannels(uninitializedBindingChannels);
     // notify parent thread about completion of first validation.
     {
         lock_guard<mutex> lock(fInitialValidationMutex);
@@ -246,7 +225,7 @@ void FairMQDevice::InitWrapper()
     int numAttempts = 0;
     while (!uninitializedConnectingChannels.empty())
     {
-        ConnectChannels(uninitializedConnectingChannels);
+        AttachChannels(uninitializedConnectingChannels);
         if (++numAttempts > fMaxInitializationAttempts)
         {
             LOG(ERROR) << "could not connect all channels after " << fMaxInitializationAttempts << " attempts";
@@ -256,7 +235,7 @@ void FairMQDevice::InitWrapper()
 
         if (numAttempts != 0)
         {
-            std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+            this_thread::sleep_for(chrono::milliseconds(1000));
         }
     }
 
@@ -275,44 +254,31 @@ void FairMQDevice::Init()
 {
 }
 
-bool FairMQDevice::BindChannel(FairMQChannel& ch)
-{
-    LOG(DEBUG) << "Initializing channel " << ch.fChannelName << " (" << ch.fType << ")";
-    // initialize the socket
-    ch.fSocket = fTransportFactory->CreateSocket(ch.fType, ch.fChannelName, fNumIoThreads, fId);
-    // set high water marks
-    ch.fSocket->SetOption("snd-hwm", &(ch.fSndBufSize), sizeof(ch.fSndBufSize));
-    ch.fSocket->SetOption("rcv-hwm", &(ch.fRcvBufSize), sizeof(ch.fRcvBufSize));
-
-    LOG(DEBUG) << "Binding channel " << ch.fChannelName << " on " << ch.fAddress;
-
-    return BindEndpoint(*ch.fSocket, ch.fAddress);
-}
-
-bool FairMQDevice::ConnectChannel(FairMQChannel& ch)
-{
-    LOG(DEBUG) << "Initializing channel " << ch.fChannelName << " (" << ch.fType << ")";
-    // initialize the socket
-    ch.fSocket = fTransportFactory->CreateSocket(ch.fType, ch.fChannelName, fNumIoThreads, fId);
-    // set high water marks
-    ch.fSocket->SetOption("snd-hwm", &(ch.fSndBufSize), sizeof(ch.fSndBufSize));
-    ch.fSocket->SetOption("rcv-hwm", &(ch.fRcvBufSize), sizeof(ch.fRcvBufSize));
-    // connect
-    LOG(DEBUG) << "Connecting channel " << ch.fChannelName << " to " << ch.fAddress;
-    ConnectEndpoint(*ch.fSocket, ch.fAddress);
-    return true;
-}
-
 bool FairMQDevice::AttachChannel(FairMQChannel& ch)
 {
-    std::vector<std::string> endpoints;
+    if (!ch.fTransportFactory)
+    {
+        if (ch.fTransport == "default" || ch.fTransport == fDefaultTransport)
+        {
+            LOG(DEBUG) << ch.fName << ": using default transport";
+            ch.InitTransport(fTransportFactory);
+        }
+        else
+        {
+            LOG(DEBUG) << ch.fName << ": channel transport (" << fDefaultTransport << ") overriden to " << ch.fTransport;
+            ch.InitTransport(AddTransport(ch.fTransport));
+        }
+        ch.fTransportType = ch.fTransportFactory->GetType();
+    }
+
+    vector<string> endpoints;
     FairMQChannel::Tokenize(endpoints, ch.fAddress);
     for (auto& endpoint : endpoints)
     {
         //(re-)init socket
         if (!ch.fSocket)
         {
-            ch.fSocket = fTransportFactory->CreateSocket(ch.fType, ch.fChannelName, fNumIoThreads, fId);
+            ch.fSocket = ch.fTransportFactory->CreateSocket(ch.fType, ch.fName, fNumIoThreads, fId);
         }
 
         // set high water marks
@@ -332,7 +298,7 @@ bool FairMQDevice::AttachChannel(FairMQChannel& ch)
         // attach
         bool bind = (ch.fMethod == "bind");
         bool connectionModifier = false;
-        std::string address = endpoint;
+        string address = endpoint;
 
         // check if the default fMethod is overridden by a modifier
         if (endpoint[0] == '+' || endpoint[0] == '>')
@@ -368,7 +334,7 @@ bool FairMQDevice::AttachChannel(FairMQChannel& ch)
         }
         endpoint += address;
 
-        LOG(DEBUG) << "Attached channel " << ch.fChannelName << " to " << endpoint << (bind ? " (bind) " : " (connect) ");
+        LOG(DEBUG) << "Attached channel " << ch.fName << " to " << endpoint << (bind ? " (bind) " : " (connect) ");
 
         // after the book keeping is done, exit in case of errors
         if (!rc)
@@ -383,34 +349,32 @@ bool FairMQDevice::AttachChannel(FairMQChannel& ch)
     return true;
 }
 
-bool FairMQDevice::ConnectEndpoint(FairMQSocket& socket, std::string& endpoint)
+bool FairMQDevice::ConnectEndpoint(FairMQSocket& socket, string& endpoint)
 {
     socket.Connect(endpoint);
 
     return true;
 }
 
-bool FairMQDevice::BindEndpoint(FairMQSocket& socket, std::string& endpoint)
+bool FairMQDevice::BindEndpoint(FairMQSocket& socket, string& endpoint)
 {
     // number of attempts when choosing a random port
     int maxAttempts = 1000;
     int numAttempts = 0;
 
     // initialize random generator
-    std::default_random_engine generator(std::chrono::system_clock::now().time_since_epoch().count());
+    default_random_engine generator(chrono::system_clock::now().time_since_epoch().count());
-    std::uniform_int_distribution<int> randomPort(fPortRangeMin, fPortRangeMax);
+    uniform_int_distribution<int> randomPort(fPortRangeMin, fPortRangeMax);
 
     // try to bind to the saved port. In case of failure, try random one.
     while (!socket.Bind(endpoint))
     {
-        LOG(DEBUG) << "Could not bind to configured (TCP) port, trying random port in range "
+        LOG(DEBUG) << "Could not bind to configured (TCP) port, trying random port in range " << fPortRangeMin << "-" << fPortRangeMax;
-          << fPortRangeMin << "-" << fPortRangeMax;
         ++numAttempts;
 
         if (numAttempts > maxAttempts)
         {
-            LOG(ERROR) << "could not bind to any (TCP) port in the given range after "
+            LOG(ERROR) << "could not bind to any (TCP) port in the given range after " << maxAttempts << " attempts";
-              << maxAttempts << " attempts";
             return false;
         }
 
@@ -453,7 +417,7 @@ void FairMQDevice::SortChannel(const string& name, const bool reindex)
                 // set channel name: name + vector index
                 stringstream ss;
                 ss << name << "[" << vi - fChannels.at(name).begin() << "]";
-                vi->fChannelName = ss.str();
+                vi->fName = ss.str();
             }
         }
     }
@@ -469,7 +433,7 @@ void FairMQDevice::PrintChannel(const string& name)
     {
         for (auto vi = fChannels[name].begin(); vi != fChannels[name].end(); ++vi)
         {
-            LOG(INFO) << vi->fChannelName << ": "
+            LOG(INFO) << vi->fName << ": "
                       << vi->fType << " | "
                       << vi->fMethod << " | "
                       << vi->fAddress << " | "
@@ -488,104 +452,53 @@ void FairMQDevice::OnData(const string& channelName, InputMsgCallback callback)
 {
     fDataCallbacks = true;
     fMsgInputs.insert(make_pair(channelName, callback));
+
+    if (find(fInputChannelKeys.begin(), fInputChannelKeys.end(), channelName) == fInputChannelKeys.end())
+    {
+        fInputChannelKeys.push_back(channelName);
+    }
 }
 
 void FairMQDevice::OnData(const string& channelName, InputMultipartCallback callback)
 {
     fDataCallbacks = true;
     fMultipartInputs.insert(make_pair(channelName, callback));
+
+    if (find(fInputChannelKeys.begin(), fInputChannelKeys.end(), channelName) == fInputChannelKeys.end())
+    {
+        fInputChannelKeys.push_back(channelName);
+    }
 }
 
 void FairMQDevice::RunWrapper()
 {
     LOG(INFO) << "DEVICE: Running...";
 
-    std::thread rateLogger(&FairMQDevice::LogSocketRates, this);
+    // start the rate logger thread
+    thread rateLogger(&FairMQDevice::LogSocketRates, this);
 
+    // notify channels to resume transfers
     FairMQChannel::fInterrupted = false;
-    fCmdSocket->Resume();
+    for (auto& kv : fDeviceCmdSockets)
+    {
+        kv.second->Resume();
+    }
 
     try
     {
         PreRun();
 
+        // process either data callbacks or ConditionalRun/Run
         if (fDataCallbacks)
         {
-            bool exitingRunningCallback = false;
+            // if only one input channel, do lightweight handling without additional polling.
-
+            if (fInputChannelKeys.size() == 1 && fChannels.at(fInputChannelKeys.at(0)).size() == 1)
-            vector<string> inputChannelKeys;
-            for (const auto& i: fMsgInputs)
             {
-                inputChannelKeys.push_back(i.first);
+                HandleSingleChannelInput();
             }
-            for (const auto& i: fMultipartInputs)
+            else // otherwise do full handling with polling
             {
-                inputChannelKeys.push_back(i.first);
+                HandleMultipleChannelInput();
-            }
-
-            FairMQPollerPtr poller(fTransportFactory->CreatePoller(fChannels, inputChannelKeys));
-
-            while (CheckCurrentState(RUNNING) && !exitingRunningCallback)
-            {
-                poller->Poll(200);
-
-                for (const auto& mi : fMsgInputs)
-                {
-                    for (unsigned int i = 0; i < fChannels.at(mi.first).size(); ++i)
-                    {
-                        if (poller->CheckInput(mi.first, i))
-                        {
-                            unique_ptr<FairMQMessage> msg(NewMessage());
-
-                            if (Receive(msg, mi.first, i) >= 0)
-                            {
-                                if (mi.second(msg, i) == false)
-                                {
-                                    exitingRunningCallback = true;
-                                    break;
-                                }
-                            }
-                            else
-                            {
-                                exitingRunningCallback = true;
-                                break;
-                            }
-                        }
-                    }
-                    if (exitingRunningCallback)
-                    {
-                        break;
-                    }
-                }
-
-                for (const auto& mi : fMultipartInputs)
-                {
-                    for (unsigned int i = 0; i < fChannels.at(mi.first).size(); ++i)
-                    {
-                        if (poller->CheckInput(mi.first, i))
-                        {
-                            FairMQParts parts;
-
-                            if (Receive(parts, mi.first, i) >= 0)
-                            {
-                                if (mi.second(parts, i) == false)
-                                {
-                                    exitingRunningCallback = true;
-                                    break;
-                                }
-                            }
-                            else
-                            {
-                                exitingRunningCallback = true;
-                                break;
-                            }
-                        }
-                    }
-                    if (exitingRunningCallback)
-                    {
-                        break;
-                    }
-                }
             }
         }
         else
@@ -601,24 +514,213 @@ void FairMQDevice::RunWrapper()
     }
     catch (const out_of_range& oor)
     {
-        LOG(ERROR) << "Out of Range error: " << oor.what();
+        LOG(ERROR) << "out of range: " << oor.what();
-        LOG(ERROR) << "Incorrect channel name in the Run() or the configuration?";
+        LOG(ERROR) << "incorrect/incomplete channel configuration?";
-        ChangeState(ERROR);
     }
 
+    // if Run() exited and the state is still RUNNING, transition to READY.
     if (CheckCurrentState(RUNNING))
     {
         ChangeState(internal_READY);
     }
 
-    try
-    {
     rateLogger.join();
 }
-    catch (exception& e)
+
+void FairMQDevice::HandleSingleChannelInput()
 {
-        LOG(ERROR) << "Exception cought during Run(): " << e.what();
+    bool proceed = true;
-        exit(EXIT_FAILURE);
+
+    if (fMsgInputs.size() > 0)
+    {
+        while (CheckCurrentState(RUNNING) && proceed)
+        {
+            proceed = HandleMsgInput(fInputChannelKeys.at(0), fMsgInputs.begin()->second, 0);
+        }
+    }
+    else if (fMultipartInputs.size() > 0)
+    {
+        while (CheckCurrentState(RUNNING) && proceed)
+        {
+            proceed = HandleMultipartInput(fInputChannelKeys.at(0), fMultipartInputs.begin()->second, 0);
+        }
+    }
+}
+
+void FairMQDevice::HandleMultipleChannelInput()
+{
+    // check if more than one transport is used
+    fMultitransportInputs.clear();
+    for (const auto& k : fInputChannelKeys)
+    {
+        FairMQ::Transport t = fChannels.at(k).at(0).fTransportType;
+        if (fMultitransportInputs.find(t) == fMultitransportInputs.end())
+        {
+            fMultitransportInputs.insert(pair<FairMQ::Transport, vector<string>>(t, vector<string>()));
+            fMultitransportInputs.at(t).push_back(k);
+        }
+        else
+        {
+            fMultitransportInputs.at(t).push_back(k);
+        }
+    }
+
+    for (const auto& mi : fMsgInputs)
+    {
+        for (unsigned int i = 0; i < fChannels.at(mi.first).size(); ++i)
+        {
+            fChannels.at(mi.first).at(i).fMultipart = false;
+        }
+    }
+
+    for (const auto& mi : fMultipartInputs)
+    {
+        for (unsigned int i = 0; i < fChannels.at(mi.first).size(); ++i)
+        {
+            fChannels.at(mi.first).at(i).fMultipart = true;
+        }
+    }
+
+    // if more than one transport is used, handle poll of each in a separate thread
+    if (fMultitransportInputs.size() > 1)
+    {
+        HandleMultipleTransportInput();
+    }
+    else // otherwise poll directly
+    {
+        bool proceed = true;
+
+        FairMQPollerPtr poller(fChannels.at(fInputChannelKeys.at(0)).at(0).fTransportFactory->CreatePoller(fChannels, fInputChannelKeys));
+
+        while (CheckCurrentState(RUNNING) && proceed)
+        {
+            poller->Poll(200);
+
+            // check which inputs are ready and call their data handlers if they are.
+            for (const auto& ch : fInputChannelKeys)
+            {
+                for (unsigned int i = 0; i < fChannels.at(ch).size(); ++i)
+                {
+                    if (poller->CheckInput(ch, i))
+                    {
+                        if (fChannels.at(ch).at(i).fMultipart)
+                        {
+                            proceed = HandleMultipartInput(ch, fMultipartInputs.at(ch), i);
+                        }
+                        else
+                        {
+                            proceed = HandleMsgInput(ch, fMsgInputs.at(ch), i);
+                        }
+
+                        if (!proceed)
+                        {
+                            break;
+                        }
+                    }
+                }
+                if (!proceed)
+                {
+                    break;
+                }
+            }
+        }
+    }
+}
+
+void FairMQDevice::HandleMultipleTransportInput()
+{
+    vector<thread> threads;
+
+    fMultitransportProceed = true;
+
+    for (const auto& i : fMultitransportInputs)
+    {
+        threads.push_back(thread(&FairMQDevice::PollForTransport, this, fTransports.at(i.first).get(), i.second));
+    }
+
+    for (thread& t : threads)
+    {
+        t.join();
+    }
+}
+
+void FairMQDevice::PollForTransport(const FairMQTransportFactory* factory, const vector<string>& channelKeys)
+{
+    try
+    {
+        FairMQPollerPtr poller(factory->CreatePoller(fChannels, channelKeys));
+
+        while (CheckCurrentState(RUNNING) && fMultitransportProceed)
+        {
+            poller->Poll(500);
+
+            for (const auto& ch : channelKeys)
+            {
+                for (unsigned int i = 0; i < fChannels.at(ch).size(); ++i)
+                {
+                    if (poller->CheckInput(ch, i))
+                    {
+                        lock_guard<mutex> lock(fMultitransportMutex);
+
+                        if (!fMultitransportProceed)
+                        {
+                            break;
+                        }
+
+                        if (fChannels.at(ch).at(i).fMultipart)
+                        {
+                            fMultitransportProceed = HandleMultipartInput(ch, fMultipartInputs.at(ch), i);
+                        }
+                        else
+                        {
+                            fMultitransportProceed = HandleMsgInput(ch, fMsgInputs.at(ch), i);
+                        }
+
+                        if (!fMultitransportProceed)
+                        {
+                            break;
+                        }
+                    }
+                }
+                if (!fMultitransportProceed)
+                {
+                    break;
+                }
+            }
+        }
+    }
+    catch (std::exception& e)
+    {
+        LOG(ERROR) << "FairMQDevice::PollForTransport() failed: " << e.what() << ", going to ERROR state.";
+        ChangeState(ERROR_FOUND);
+    }
+}
+
+bool FairMQDevice::HandleMsgInput(const string& chName, const InputMsgCallback& callback, int i) const
+{
+    unique_ptr<FairMQMessage> input(fChannels.at(chName).at(i).fTransportFactory->CreateMessage());
+
+    if (Receive(input, chName, i) >= 0)
+    {
+        return callback(input, 0);
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool FairMQDevice::HandleMultipartInput(const string& chName, const InputMultipartCallback& callback, int i) const
+{
+    FairMQParts input;
+
+    if (Receive(input, chName, i) >= 0)
+    {
+        return callback(input, 0);
+    }
+    else
+    {
+        return false;
     }
 }
 
@@ -643,7 +745,7 @@ void FairMQDevice::Pause()
 {
     while (CheckCurrentState(PAUSED))
     {
-        std::this_thread::sleep_for(std::chrono::milliseconds(500));
+        this_thread::sleep_for(chrono::milliseconds(500));
         LOG(DEBUG) << "paused...";
     }
     LOG(DEBUG) << "Unpausing";
@@ -759,39 +861,97 @@ int FairMQDevice::GetProperty(const int key, const int default_ /*= 0*/)
     }
 }
 
+// DEPRECATED, use the string version
 void FairMQDevice::SetTransport(FairMQTransportFactory* factory)
+{
+    if (fTransports.empty())
     {
         fTransportFactory = shared_ptr<FairMQTransportFactory>(factory);
+        pair<FairMQ::Transport, shared_ptr<FairMQTransportFactory>> t(fTransportFactory->GetType(), fTransportFactory);
+        fTransports.insert(t);
+    }
+    else
+    {
+        LOG(ERROR) << "Transports container is not empty when setting transport. Setting twice?";
+        ChangeState(ERROR_FOUND);
+    }
 }
 
-void FairMQDevice::SetTransport(const string& transport)
+shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(const string& transport)
 {
+    unordered_map<FairMQ::Transport, shared_ptr<FairMQTransportFactory>>::const_iterator i = fTransports.find(FairMQ::TransportTypes.at(transport));
+
+    if (i == fTransports.end())
+    {
+        shared_ptr<FairMQTransportFactory> tr;
+
         if (transport == "zeromq")
         {
-        fTransportFactory = make_shared<FairMQTransportFactoryZMQ>();
+            tr = make_shared<FairMQTransportFactoryZMQ>();
         }
         else if (transport == "shmem")
         {
-        fTransportFactory = make_shared<FairMQTransportFactorySHM>();
+            tr = make_shared<FairMQTransportFactorySHM>();
         }
 #ifdef NANOMSG_FOUND
         else if (transport == "nanomsg")
         {
-        fTransportFactory = make_shared<FairMQTransportFactoryNN>();
+            tr = make_shared<FairMQTransportFactoryNN>();
         }
 #endif
         else
         {
-        LOG(ERROR) << "Unavailable transport implementation requested: "
+            LOG(ERROR) << "Unavailable transport requested: "
                        << "\"" << transport << "\""
                        << ". Available are: "
                        << "\"zeromq\""
+                       << "\"shmem\""
 #ifdef NANOMSG_FOUND
                        << ", \"nanomsg\""
 #endif
                        << ". Exiting.";
             exit(EXIT_FAILURE);
         }
+
+        LOG(DEBUG) << "Adding '" << transport << "' transport to the device.";
+
+        pair<FairMQ::Transport, shared_ptr<FairMQTransportFactory>> trPair(FairMQ::TransportTypes.at(transport), tr);
+        fTransports.insert(trPair);
+
+        auto p = fDeviceCmdSockets.emplace(tr->GetType(), tr->CreateSocket("pub", "device-commands", fNumIoThreads, fId));
+        if (p.second)
+        {
+            p.first->second->Bind("inproc://commands");
+        }
+        else
+        {
+            exit(EXIT_FAILURE);
+        }
+
+        FairMQMessagePtr msg(tr->CreateMessage());
+        msg->SetDeviceId(fId);
+
+        return move(tr);
+    }
+    else
+    {
+        LOG(DEBUG) << "Reusing existing '" << transport << "' transport.";
+        return i->second;
+    }
+}
+
+void FairMQDevice::SetTransport(const string& transport)
+{
+    if (fTransports.empty())
+    {
+        LOG(DEBUG) << "Requesting '" << transport << "' as default transport for the device";
+        fTransportFactory = AddTransport(transport);
+    }
+    else
+    {
+        LOG(ERROR) << "Transports container is not empty when setting transport. Setting default twice?";
+        ChangeState(ERROR_FOUND);
+    }
 }
 
 void FairMQDevice::SetConfig(FairMQProgOptions& config)
@@ -799,7 +959,8 @@ void FairMQDevice::SetConfig(FairMQProgOptions& config)
     LOG(DEBUG) << "PID: " << getpid();
     fConfig = &config;
     fChannels = config.GetFairMQMap();
-    SetTransport(config.GetValue<string>("transport"));
+    fDefaultTransport = config.GetValue<string>("transport");
+    SetTransport(fDefaultTransport);
     fId = config.GetValue<string>("id");
     fNetworkInterface = config.GetValue<string>("network-interface");
     fNumIoThreads = config.GetValue<int>("io-threads");
@@ -812,7 +973,6 @@ void FairMQDevice::LogSocketRates()
 
     timestamp_t msSinceLastLog;
 
-    int numFilteredSockets = 0;
     vector<FairMQSocket*> filteredSockets;
     vector<string> filteredChannelNames;
     vector<int> logIntervals;
@@ -832,11 +992,14 @@ void FairMQDevice::LogSocketRates()
                 stringstream ss;
                 ss << mi.first << "[" << vi - (mi.second).begin() << "]";
                 filteredChannelNames.push_back(ss.str());
-                ++numFilteredSockets;
             }
         }
     }
 
+    unsigned int numFilteredSockets = filteredSockets.size();
+
+    if (numFilteredSockets > 0)
+    {
         vector<unsigned long> bytesIn(numFilteredSockets);
         vector<unsigned long> msgIn(numFilteredSockets);
         vector<unsigned long> bytesOut(numFilteredSockets);
@@ -881,7 +1044,7 @@ void FairMQDevice::LogSocketRates()
                     intervalCounters.at(i) = 0;
 
                     bytesInNew.at(i) = vi->GetBytesRx();
-                mbPerSecIn.at(i) = (static_cast<double>(bytesInNew.at(i) - bytesIn.at(i)) / (1024. * 1024.)) / static_cast<double>(msSinceLastLog) * 1000.;
+                    mbPerSecIn.at(i) = (static_cast<double>(bytesInNew.at(i) - bytesIn.at(i)) / (1000. * 1000.)) / static_cast<double>(msSinceLastLog) * 1000.;
                     bytesIn.at(i) = bytesInNew.at(i);
 
                     msgInNew.at(i) = vi->GetMessagesRx();
@@ -889,7 +1052,7 @@ void FairMQDevice::LogSocketRates()
                     msgIn.at(i) = msgInNew.at(i);
 
                     bytesOutNew.at(i) = vi->GetBytesTx();
-                mbPerSecOut.at(i) = (static_cast<double>(bytesOutNew.at(i) - bytesOut.at(i)) / (1024. * 1024.)) / static_cast<double>(msSinceLastLog) * 1000.;
+                    mbPerSecOut.at(i) = (static_cast<double>(bytesOutNew.at(i) - bytesOut.at(i)) / (1000. * 1000.)) / static_cast<double>(msSinceLastLog) * 1000.;
                     bytesOut.at(i) = bytesOutNew.at(i);
 
                     msgOutNew.at(i) = vi->GetMessagesTx();
@@ -905,10 +1068,11 @@ void FairMQDevice::LogSocketRates()
             }
 
             t0 = t1;
-        std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+            this_thread::sleep_for(chrono::milliseconds(1000));
+        }
     }
 
-    LOG(DEBUG) << "FairMQDevice::LogSocketRates() stopping";
+    // LOG(DEBUG) << "FairMQDevice::LogSocketRates() stopping";
 }
 
 void FairMQDevice::InteractiveStateLoop()
@@ -1011,9 +1175,12 @@ void FairMQDevice::InteractiveStateLoop()
 void FairMQDevice::Unblock()
 {
     FairMQChannel::fInterrupted = true;
-    fCmdSocket->Interrupt();
+    for (auto& kv : fDeviceCmdSockets)
-    FairMQMessagePtr cmd(fTransportFactory->CreateMessage());
+    {
-    fCmdSocket->Send(cmd.get(), 0);
+        kv.second->Interrupt();
+        FairMQMessagePtr cmd(fTransports.at(kv.first)->CreateMessage());
+        kv.second->Send(cmd);
+    }
 }
 
 void FairMQDevice::ResetTaskWrapper()
@@ -1047,8 +1214,8 @@ void FairMQDevice::Reset()
 
             vi.fPoller = nullptr;
 
-            vi.fCmdSocket->Close();
+            vi.fChannelCmdSocket->Close();
-            vi.fCmdSocket = nullptr;
+            vi.fChannelCmdSocket = nullptr;
         }
     }
 }
@@ -1061,10 +1228,14 @@ bool FairMQDevice::Terminated()
 void FairMQDevice::Terminate()
 {
     // Termination signal has to be sent only once to any socket.
-    if (fCmdSocket)
+    for (auto& kv : fDeviceCmdSockets)
     {
-        fCmdSocket->Terminate();
+        kv.second->Terminate();
     }
+    // if (!fDeviceCmdSockets.empty())
+    // {
+    //     fDeviceCmdSockets[0]->Terminate();
+    // }
 }
 
 void FairMQDevice::Shutdown()
@@ -1081,18 +1252,23 @@ void FairMQDevice::Shutdown()
             {
                 vi.fSocket->Close();
             }
-            if (vi.fCmdSocket)
+            if (vi.fChannelCmdSocket)
             {
-                vi.fCmdSocket->Close();
+                vi.fChannelCmdSocket->Close();
             }
         }
     }
 
-    if (fCmdSocket)
+    for (auto& s : fDeviceCmdSockets)
     {
-        fCmdSocket->Close();
+        s.second->Close();
     }
 
+    // if (!fDeviceCmdSockets.empty())
+    // {
+    //     fDeviceCmdSockets[0]->Close();
+    // }
+
     LOG(DEBUG) << "Closed all sockets!";
 }
 

fairmq/FairMQDevice.h

@@ -17,6 +17,7 @@
 
 #include <vector>
 #include <memory> // unique_ptr
+#include <algorithm> // std::sort()
 #include <string>
 #include <iostream>
 #include <unordered_map>
@@ -30,6 +31,7 @@
 #include "FairMQConfigurable.h"
 #include "FairMQStateMachine.h"
 #include "FairMQTransportFactory.h"
+#include "FairMQTransports.h"
 
 #include "FairMQSocket.h"
 #include "FairMQChannel.h"
@@ -96,83 +98,103 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
         Deserializer().Deserialize(msg, std::forward<DataType>(data), std::forward<Args>(args)...);
     }
 
+    inline int Send(FairMQMessagePtr& msg, const std::string& chan, const int i = 0) const
+    {
+        return fChannels.at(chan).at(i).Send(msg);
+    }
+
+    inline int Receive(FairMQMessagePtr& msg, const std::string& chan, const int i = 0) const
+    {
+        return fChannels.at(chan).at(i).Receive(msg);
+    }
+
     /// Shorthand method to send `msg` on `chan` at index `i`
     /// @param msg message reference
     /// @param chan channel name
     /// @param i channel index
     /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
     /// In case of errors, returns -1.
-    inline int Send(const FairMQMessagePtr& msg, const std::string& chan, const int i = 0, int sndTimeoutInMs = -1) const
+    inline int Send(FairMQMessagePtr& msg, const std::string& chan, const int i, int sndTimeoutInMs) const
     {
         return fChannels.at(chan).at(i).Send(msg, sndTimeoutInMs);
     }
 
-    /// Shorthand method to send `msg` on `chan` at index `i` without blocking
-    /// @param msg message reference
-    /// @param chan channel name
-    /// @param i channel index
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
-    /// In case of errors, returns -1.
-    inline int SendAsync(const FairMQMessagePtr& msg, const std::string& chan, const int i = 0) const
-    {
-        return fChannels.at(chan).at(i).SendAsync(msg);
-    }
-
-    /// Shorthand method to send FairMQParts on `chan` at index `i`
-    /// @param parts parts reference
-    /// @param chan channel name
-    /// @param i channel index
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
-    /// In case of errors, returns -1.
-    inline int64_t Send(const FairMQParts& parts, const std::string& chan, const int i = 0, int sndTimeoutInMs = -1) const
-    {
-        return fChannels.at(chan).at(i).Send(parts.fParts, sndTimeoutInMs);
-    }
-
-    /// Shorthand method to send FairMQParts on `chan` at index `i` without blocking
-    /// @param parts parts reference
-    /// @param chan channel name
-    /// @param i channel index
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
-    /// In case of errors, returns -1.
-    inline int64_t SendAsync(const FairMQParts& parts, const std::string& chan, const int i = 0) const
-    {
-        return fChannels.at(chan).at(i).SendAsync(parts.fParts);
-    }
-
     /// Shorthand method to receive `msg` on `chan` at index `i`
     /// @param msg message reference
     /// @param chan channel name
     /// @param i channel index
     /// @return Number of bytes that have been received. -2 If reading from the queue was not possible or timed out.
     /// In case of errors, returns -1.
-    inline int Receive(const FairMQMessagePtr& msg, const std::string& chan, const int i = 0, int rcvTimeoutInMs = -1) const
+    inline int Receive(FairMQMessagePtr& msg, const std::string& chan, const int i, int rcvTimeoutInMs) const
     {
         return fChannels.at(chan).at(i).Receive(msg, rcvTimeoutInMs);
     }
 
+    /// Shorthand method to send `msg` on `chan` at index `i` without blocking
+    /// @param msg message reference
+    /// @param chan channel name
+    /// @param i channel index
+    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
+    /// In case of errors, returns -1.
+    inline int SendAsync(FairMQMessagePtr& msg, const std::string& chan, const int i = 0) const
+    {
+        return fChannels.at(chan).at(i).SendAsync(msg);
+    }
+
     /// Shorthand method to receive `msg` on `chan` at index `i` without blocking
     /// @param msg message reference
     /// @param chan channel name
     /// @param i channel index
     /// @return Number of bytes that have been received. -2 If reading from the queue was not possible or timed out.
     /// In case of errors, returns -1.
-    inline int ReceiveAsync(const FairMQMessagePtr& msg, const std::string& chan, const int i = 0) const
+    inline int ReceiveAsync(FairMQMessagePtr& msg, const std::string& chan, const int i = 0) const
     {
         return fChannels.at(chan).at(i).ReceiveAsync(msg);
     }
 
+    inline int64_t Send(FairMQParts& parts, const std::string& chan, const int i = 0) const
+    {
+        return fChannels.at(chan).at(i).Send(parts.fParts);
+    }
+
+    inline int64_t Receive(FairMQParts& parts, const std::string& chan, const int i = 0) const
+    {
+        return fChannels.at(chan).at(i).Receive(parts.fParts);
+    }
+
+    /// Shorthand method to send FairMQParts on `chan` at index `i`
+    /// @param parts parts reference
+    /// @param chan channel name
+    /// @param i channel index
+    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
+    /// In case of errors, returns -1.
+    inline int64_t Send(FairMQParts& parts, const std::string& chan, const int i, int sndTimeoutInMs) const
+    {
+        return fChannels.at(chan).at(i).Send(parts.fParts, sndTimeoutInMs);
+    }
+
     /// Shorthand method to receive FairMQParts on `chan` at index `i`
     /// @param parts parts reference
     /// @param chan channel name
     /// @param i channel index
     /// @return Number of bytes that have been received. -2 If reading from the queue was not possible or timed out.
     /// In case of errors, returns -1.
-    inline int64_t Receive(FairMQParts& parts, const std::string& chan, const int i = 0, int rcvTimeoutInMs = -1) const
+    inline int64_t Receive(FairMQParts& parts, const std::string& chan, const int i, int rcvTimeoutInMs) const
     {
         return fChannels.at(chan).at(i).Receive(parts.fParts, rcvTimeoutInMs);
     }
 
+    /// Shorthand method to send FairMQParts on `chan` at index `i` without blocking
+    /// @param parts parts reference
+    /// @param chan channel name
+    /// @param i channel index
+    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out.
+    /// In case of errors, returns -1.
+    inline int64_t SendAsync(FairMQParts& parts, const std::string& chan, const int i = 0) const
+    {
+        return fChannels.at(chan).at(i).SendAsync(parts.fParts);
+    }
+
     /// Shorthand method to receive FairMQParts on `chan` at index `i` without blocking
     /// @param parts parts reference
     /// @param chan channel name
@@ -199,25 +221,25 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
         return fTransportFactory->CreateMessage(size);
     }
 
-    template<typename T>
-    static void FairMQSimpleMsgCleanup(void* /*data*/, void* hint)
-    {
-        delete static_cast<T*>(hint);
-    }
-
-    static void FairMQNoCleanup(void* /*data*/, void* /*hint*/)
-    {
-    }
-
     /// @brief Create new FairMQMessage with user provided buffer and size
     /// @param data pointer to user provided buffer
     /// @param size size of the user provided buffer
-    /// @param ffn optional callback, called when the message is transfered (and can be deleted)
+    /// @param ffn callback, called when the message is transfered (and can be deleted)
-    /// @param hint optional helper pointer that can be used in the callback
+    /// @param obj optional helper pointer that can be used in the callback
     /// @return pointer to FairMQMessage
-    inline FairMQMessagePtr NewMessage(void* data, int size, fairmq_free_fn* ffn, void* hint = nullptr) const
+    inline FairMQMessagePtr NewMessage(void* data, int size, fairmq_free_fn* ffn, void* obj = nullptr) const
+    {
+        return fTransportFactory->CreateMessage(data, size, ffn, obj);
+    }
+
+    template<typename... Args>
+    inline FairMQMessagePtr NewMessageFor(const std::string& channel, int index, Args&&... args) const
+    {
+        return fChannels.at(channel).at(index).fTransportFactory->CreateMessage(std::forward<Args>(args)...);
+    }
+
+    static void FairMQNoCleanup(void* /*data*/, void* /*obj*/)
     {
-        return fTransportFactory->CreateMessage(data, size, ffn, hint);
     }
 
     template<typename T>
@@ -231,6 +253,23 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
         return fTransportFactory->CreateMessage(const_cast<char*>(str.c_str()), str.length(), FairMQNoCleanup, nullptr);
     }
 
+    template<typename T>
+    inline FairMQMessagePtr NewStaticMessageFor(const std::string& channel, int index, const T& data) const
+    {
+        return fChannels.at(channel).at(index).fTransportFactory->CreateMessage(data, sizeof(T), FairMQNoCleanup, nullptr);
+    }
+
+    inline FairMQMessagePtr NewStaticMessageFor(const std::string& channel, int index, const std::string& str) const
+    {
+        return fChannels.at(channel).at(index).fTransportFactory->CreateMessage(const_cast<char*>(str.c_str()), str.length(), FairMQNoCleanup, nullptr);
+    }
+
+    template<typename T>
+    static void FairMQSimpleMsgCleanup(void* /*data*/, void* obj)
+    {
+        delete static_cast<T*>(obj);
+    }
+
     template<typename T>
     inline FairMQMessagePtr NewSimpleMessage(const T& data) const
     {
@@ -295,8 +334,11 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
     /// Configures the device with a transport factory (DEPRECATED)
     /// @param factory  Pointer to the transport factory object
     void SetTransport(FairMQTransportFactory* factory);
-    /// Configures the device with a transport factory
+    /// Adds a transport to the device if it doesn't exist
-    /// @param transport  Transport string ("zeromq"/"nanomsg")
+    /// @param transport  Transport string ("zeromq"/"nanomsg"/"shmem")
+    std::shared_ptr<FairMQTransportFactory> AddTransport(const std::string& transport);
+    /// Sets the default transport for the device
+    /// @param transport  Transport string ("zeromq"/"nanomsg"/"shmem")
     void SetTransport(const std::string& transport = "zeromq");
 
     void SetConfig(FairMQProgOptions& config);
@@ -317,6 +359,11 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
         {
             return (static_cast<T*>(this)->*memberFunction)(msg, index);
         }));
+
+        if (find(fInputChannelKeys.begin(), fInputChannelKeys.end(), channelName) == fInputChannelKeys.end())
+        {
+            fInputChannelKeys.push_back(channelName);
+        }
     }
 
     void OnData(const std::string& channelName, InputMsgCallback);
@@ -329,6 +376,11 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
         {
             return (static_cast<T*>(this)->*memberFunction)(parts, index);
         }));
+
+        if (find(fInputChannelKeys.begin(), fInputChannelKeys.end(), channelName) == fInputChannelKeys.end())
+        {
+            fInputChannelKeys.push_back(channelName);
+        }
     }
 
     void OnData(const std::string& channelName, InputMultipartCallback);
@@ -338,6 +390,7 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
   protected:
     std::string fId; ///< Device ID
     std::string fNetworkInterface; ///< Network interface to use for dynamic binding
+    std::string fDefaultTransport; ///< Default transport for the device
 
     int fMaxInitializationAttempts; ///< Timeout for the initialization
 
@@ -348,9 +401,9 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
 
     int fLogIntervalInMs; ///< Interval for logging the socket transfer rates
 
-    FairMQSocketPtr fCmdSocket; ///< Socket used for the internal unblocking mechanism
-
     std::shared_ptr<FairMQTransportFactory> fTransportFactory; ///< Transport factory
+    std::unordered_map<FairMQ::Transport, std::shared_ptr<FairMQTransportFactory>> fTransports; ///< Container for transports
+    std::unordered_map<FairMQ::Transport, FairMQSocketPtr> fDeviceCmdSockets; ///< Sockets used for the internal unblocking mechanism
 
     /// Additional user initialization (can be overloaded in child classes). Prefer to use InitTask().
     virtual void Init();
@@ -403,14 +456,8 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
     /// Unblocks blocking channel send/receive calls
     void Unblock();
 
-    /// Binds channel in the list
+    /// Attach (bind/connect) channels in the list
-    void BindChannels(std::list<FairMQChannel*>& chans);
+    void AttachChannels(std::list<FairMQChannel*>& chans);
-    /// Connects channel in the list
-    void ConnectChannels(std::list<FairMQChannel*>& chans);
-    /// Binds a single channel (used in InitWrapper)
-    bool BindChannel(FairMQChannel& ch);
-    /// Connects a single channel (used in InitWrapper)
-    bool ConnectChannel(FairMQChannel& ch);
 
     /// Sets up and connects/binds a socket to an endpoint
     /// return a string with the actual endpoint if it happens
@@ -422,6 +469,14 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
     /// to override default: prepend "@" to bind, "+" or ">" to connect endpoint.
     bool AttachChannel(FairMQChannel& ch);
 
+    void HandleSingleChannelInput();
+    void HandleMultipleChannelInput();
+    void HandleMultipleTransportInput();
+    void PollForTransport(const FairMQTransportFactory* factory, const std::vector<std::string>& channelKeys);
+
+    bool HandleMsgInput(const std::string& chName, const InputMsgCallback& callback, int i) const;
+    bool HandleMultipartInput(const std::string& chName, const InputMultipartCallback& callback, int i) const;
+
     /// Signal handler
     void SignalHandler(int signal);
     bool fCatchingSignals;
@@ -432,6 +487,10 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
     bool fDataCallbacks;
     std::unordered_map<std::string, InputMsgCallback> fMsgInputs;
     std::unordered_map<std::string, InputMultipartCallback> fMultipartInputs;
+    std::unordered_map<FairMQ::Transport, std::vector<std::string>> fMultitransportInputs;
+    std::vector<std::string> fInputChannelKeys;
+    std::mutex fMultitransportMutex;
+    std::atomic<bool> fMultitransportProceed;
 };
 
 #endif /* FAIRMQDEVICE_H_ */

fairmq/FairMQMessage.h

@@ -18,6 +18,8 @@
 #include <cstddef> // for size_t
 #include <memory> // unique_ptr
 
+#include "FairMQTransports.h"
+
 using fairmq_free_fn = void(void* data, void* hint);
 
 class FairMQMessage
@@ -34,6 +36,8 @@ class FairMQMessage
 
     virtual void SetDeviceId(const std::string& deviceId) = 0;
 
+    virtual FairMQ::Transport GetType() const = 0;
+
     virtual void Copy(const std::unique_ptr<FairMQMessage>& msg) = 0;
 
     virtual ~FairMQMessage() {};

fairmq/FairMQSocket.h

@@ -40,12 +40,10 @@ class FairMQSocket
     virtual void Connect(const std::string& address) = 0;
     virtual bool Attach(const std::string& address, bool serverish = false);
 
-    virtual int Send(FairMQMessage* msg, const std::string& flag = "") = 0;
+    virtual int Send(FairMQMessagePtr& msg, const int flags = 0) = 0;
-    virtual int Send(FairMQMessage* msg, const int flags = 0) = 0;
+    virtual int Receive(FairMQMessagePtr& msg, const int flags = 0) = 0;
-    virtual int64_t Send(const std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0) = 0;
 
-    virtual int Receive(FairMQMessage* msg, const std::string& flag = "") = 0;
+    virtual int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0) = 0;
-    virtual int Receive(FairMQMessage* msg, const int flags = 0) = 0;
     virtual int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0) = 0;
 
     virtual void* GetSocket() const = 0;

fairmq/FairMQStateMachine.h

@@ -99,7 +99,7 @@ struct FairMQFSM_ : public msmf::state_machine_def<FairMQFSM_>
     }
 
     template <class Event, class FSM>
-    void on_exit(Event const&, FSM& fsm)
+    void on_exit(Event const&, FSM& /*fsm*/)
     {
         LOG(STATE) << "Exiting FairMQ state machine";
     }

fairmq/FairMQTransportFactory.h

@@ -21,10 +21,10 @@
 #include <unordered_map>
 
 #include "FairMQMessage.h"
-#include "FairMQChannel.h"
 #include "FairMQSocket.h"
 #include "FairMQPoller.h"
 #include "FairMQLogger.h"
+#include "FairMQTransports.h"
 
 class FairMQChannel;
 
@@ -41,6 +41,8 @@ class FairMQTransportFactory
     virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const = 0;
     virtual FairMQPollerPtr CreatePoller(const FairMQSocket& cmdSocket, const FairMQSocket& dataSocket) const = 0;
 
+    virtual FairMQ::Transport GetType() const = 0;
+
     virtual ~FairMQTransportFactory() {};
 };
 

fairmq/FairMQTransports.h

@@ -0,0 +1,47 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *         GNU Lesser General Public Licence version 3 (LGPL) version 3,        *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+#ifndef FAIRMQTRANSPORTS_H_
+#define FAIRMQTRANSPORTS_H_
+
+#include <string>
+#include <unordered_map>
+
+namespace FairMQ
+{
+
+enum class Transport
+{
+    DEFAULT,
+    ZMQ,
+    NN,
+    SHM
+};
+
+static std::unordered_map<std::string, Transport> TransportTypes {
+    { "default", Transport::DEFAULT },
+    { "zeromq", Transport::ZMQ },
+    { "nanomsg", Transport::NN },
+    { "shmem", Transport::SHM }
+};
+
+}
+
+namespace std
+{
+
+template <>
+struct hash<FairMQ::Transport>
+{
+    size_t operator()(const FairMQ::Transport& v) const
+    {
+        return hash<int>()(static_cast<int>(v));
+    }
+};
+
+}
+#endif /* FAIRMQTRANSPORTS_H_ */

fairmq/devices/FairMQBenchmarkSampler.cxx

@@ -51,10 +51,10 @@ void FairMQBenchmarkSampler::Run()
 
     uint64_t numSentMsgs = 0;
 
-    FairMQMessagePtr baseMsg(fTransportFactory->CreateMessage(fMsgSize));
-
     // store the channel reference to avoid traversing the map on every loop iteration
-    const FairMQChannel& dataOutChannel = fChannels.at(fOutChannelName).at(0);
+    FairMQChannel& dataOutChannel = fChannels.at(fOutChannelName).at(0);
+
+    FairMQMessagePtr baseMsg(dataOutChannel.Transport()->CreateMessage(fMsgSize));
 
     LOG(INFO) << "Starting the benchmark with message size of " << fMsgSize << " and number of messages " << fNumMsgs << ".";
     auto tStart = chrono::high_resolution_clock::now();
@@ -63,7 +63,7 @@ void FairMQBenchmarkSampler::Run()
     {
         if (fSameMessage)
         {
-            FairMQMessagePtr msg(fTransportFactory->CreateMessage());
+            FairMQMessagePtr msg(dataOutChannel.Transport()->CreateMessage());
             msg->Copy(baseMsg);
 
             if (dataOutChannel.Send(msg) >= 0)
@@ -80,7 +80,7 @@ void FairMQBenchmarkSampler::Run()
         }
         else
         {
-            FairMQMessagePtr msg(fTransportFactory->CreateMessage(fMsgSize));
+            FairMQMessagePtr msg(dataOutChannel.Transport()->CreateMessage(fMsgSize));
 
             if (dataOutChannel.Send(msg) >= 0)
             {

fairmq/devices/FairMQSink.cxx

@@ -36,14 +36,14 @@ void FairMQSink::Run()
 {
     uint64_t numReceivedMsgs = 0;
     // store the channel reference to avoid traversing the map on every loop iteration
-    const FairMQChannel& dataInChannel = fChannels.at(fInChannelName).at(0);
+    FairMQChannel& dataInChannel = fChannels.at(fInChannelName).at(0);
 
     LOG(INFO) << "Starting the benchmark and expecting to receive " << fNumMsgs << " messages.";
     auto tStart = chrono::high_resolution_clock::now();
 
     while (CheckCurrentState(RUNNING))
     {
-        FairMQMessagePtr msg(fTransportFactory->CreateMessage());
+        FairMQMessagePtr msg(dataInChannel.Transport()->CreateMessage());
 
         if (dataInChannel.Receive(msg) >= 0)
         {

fairmq/nanomsg/FairMQMessageNN.cxx

@@ -22,6 +22,8 @@
 
 using namespace std;
 
+static FairMQ::Transport gTransportType = FairMQ::Transport::NN;
+
 string FairMQMessageNN::fDeviceID = string();
 
 FairMQMessageNN::FairMQMessageNN()
@@ -145,6 +147,11 @@ void FairMQMessageNN::SetDeviceId(const string& deviceId)
     fDeviceID = deviceId;
 }
 
+FairMQ::Transport FairMQMessageNN::GetType() const
+{
+    return gTransportType;
+}
+
 void FairMQMessageNN::Copy(const unique_ptr<FairMQMessage>& msg)
 {
     if (fMessage)

fairmq/nanomsg/FairMQMessageNN.h

@@ -41,6 +41,8 @@ class FairMQMessageNN : public FairMQMessage
 
     virtual void SetDeviceId(const std::string& deviceId);
 
+    virtual FairMQ::Transport GetType() const;
+
     virtual void Copy(const std::unique_ptr<FairMQMessage>& msg);
 
     virtual ~FairMQMessageNN();

fairmq/nanomsg/FairMQPollerNN.cxx

@@ -53,7 +53,7 @@ FairMQPollerNN::FairMQPollerNN(const vector<FairMQChannel>& channels)
         }
         else
         {
-            LOG(ERROR) << "invalid poller configuration, exiting.";
+            LOG(ERROR) << "nanomsg: invalid poller configuration, exiting.";
             exit(EXIT_FAILURE);
         }
     }
@@ -104,7 +104,7 @@ FairMQPollerNN::FairMQPollerNN(const unordered_map<string, vector<FairMQChannel>
                 }
                 else
                 {
-                    LOG(ERROR) << "invalid poller configuration, exiting.";
+                    LOG(ERROR) << "nanomsg: invalid poller configuration, exiting.";
                     exit(EXIT_FAILURE);
                 }
             }
@@ -112,8 +112,8 @@ FairMQPollerNN::FairMQPollerNN(const unordered_map<string, vector<FairMQChannel>
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "At least one of the provided channel keys for poller initialization is invalid";
+        LOG(ERROR) << "nanomsg: at least one of the provided channel keys for poller initialization is invalid";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "nanomsg: out of range error: " << oor.what() << '\n';
         exit(EXIT_FAILURE);
     }
 }
@@ -150,7 +150,7 @@ FairMQPollerNN::FairMQPollerNN(const FairMQSocket& cmdSocket, const FairMQSocket
     }
     else
     {
-        LOG(ERROR) << "invalid poller configuration, exiting.";
+        LOG(ERROR) << "nanomsg: invalid poller configuration, exiting.";
         exit(EXIT_FAILURE);
     }
 }
@@ -161,11 +161,12 @@ void FairMQPollerNN::Poll(const int timeout)
     {
         if (errno == ETERM)
         {
-            LOG(DEBUG) << "polling exited, reason: " << nn_strerror(errno);
+            LOG(DEBUG) << "nanomsg: polling exited, reason: " << nn_strerror(errno);
         }
         else
         {
-            LOG(ERROR) << "polling failed, reason: " << nn_strerror(errno);
+            LOG(ERROR) << "nanomsg: polling failed, reason: " << nn_strerror(errno);
+            throw std::runtime_error("nanomsg: polling failed");
         }
     }
 }
@@ -203,8 +204,8 @@ bool FairMQPollerNN::CheckInput(const string channelKey, const int index)
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
+        LOG(ERROR) << "nanomsg: invalid channel key: \"" << channelKey << "\"";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "nanomsg: out of range error: " << oor.what() << '\n';
         exit(EXIT_FAILURE);
     }
 }
@@ -222,8 +223,8 @@ bool FairMQPollerNN::CheckOutput(const string channelKey, const int index)
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
+        LOG(ERROR) << "nanomsg: invalid channel key: \"" << channelKey << "\"";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "nanomsg: out of range error: " << oor.what() << '\n';
         exit(EXIT_FAILURE);
     }
 }

fairmq/nanomsg/FairMQSocketNN.cxx

@@ -29,6 +29,8 @@
 
 using namespace std;
 
+atomic<bool> FairMQSocketNN::fInterrupted(false);
+
 FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
     : FairMQSocket(0, 0, NN_DONTWAIT)
     , fSocket(-1)
@@ -71,6 +73,18 @@ FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const int
         }
     }
 
+    int sndTimeout = 700;
+    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_SNDTIMEO, &sndTimeout, sizeof(sndTimeout)) != 0)
+    {
+        LOG(ERROR) << "Failed setting NN_SNDTIMEO socket option, reason: " << nn_strerror(errno);
+    }
+
+    int rcvTimeout = 700;
+    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVTIMEO, &rcvTimeout, sizeof(rcvTimeout)) != 0)
+    {
+        LOG(ERROR) << "Failed setting NN_RCVTIMEO socket option, reason: " << nn_strerror(errno);
+    }
+
 #ifdef NN_RCVMAXSIZE
     int rcvSize = -1;
     nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVMAXSIZE, &rcvSize, sizeof(rcvSize));
@@ -108,105 +122,155 @@ void FairMQSocketNN::Connect(const string& address)
     }
 }
 
-int FairMQSocketNN::Send(FairMQMessage* msg, const string& flag)
+int FairMQSocketNN::Send(FairMQMessagePtr& msg, const int flags)
 {
-    return Send(msg, GetConstant(flag));
+    int nbytes = -1;
-}
 
-int FairMQSocketNN::Send(FairMQMessage* msg, const int flags)
+    while (true)
     {
         void* ptr = msg->GetMessage();
-    int nbytes = nn_send(fSocket, &ptr, NN_MSG, flags);
+        nbytes = nn_send(fSocket, &ptr, NN_MSG, flags);
         if (nbytes >= 0)
         {
             fBytesTx += nbytes;
             ++fMessagesTx;
-        static_cast<FairMQMessageNN*>(msg)->fReceiving = false;
+            static_cast<FairMQMessageNN*>(msg.get())->fReceiving = false;
+
             return nbytes;
         }
-    if (nn_errno() == EAGAIN)
+        else if (nn_errno() == ETIMEDOUT)
+        {
+            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
+            {
+                continue;
+            }
+            else
             {
                 return -2;
             }
-    if (nn_errno() == ETERM)
+        }
+        else if (nn_errno() == EAGAIN)
+        {
+            return -2;
+        }
+        else if (nn_errno() == ETERM)
         {
             LOG(INFO) << "terminating socket " << fId;
             return -1;
         }
+        else
+        {
             LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
             return nbytes;
         }
+    }
+}
 
-int64_t FairMQSocketNN::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
+int FairMQSocketNN::Receive(FairMQMessagePtr& msg, const int flags)
+{
+    int nbytes = -1;
+
+    while (true)
+    {
+        void* ptr = NULL;
+        nbytes = nn_recv(fSocket, &ptr, NN_MSG, flags);
+        if (nbytes >= 0)
+        {
+            fBytesRx += nbytes;
+            ++fMessagesRx;
+            msg->SetMessage(ptr, nbytes);
+            static_cast<FairMQMessageNN*>(msg.get())->fReceiving = true;
+            return nbytes;
+        }
+        else if (nn_errno() == ETIMEDOUT)
+        {
+            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
+            {
+                continue;
+            }
+            else
+            {
+                return -2;
+            }
+        }
+        else if (nn_errno() == EAGAIN)
+        {
+            return -2;
+        }
+        else if (nn_errno() == ETERM)
+        {
+            LOG(INFO) << "terminating socket " << fId;
+            return -1;
+        }
+        else
+        {
+            LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
+            return nbytes;
+        }
+    }
+}
+
+int64_t FairMQSocketNN::Send(vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
 {
 #ifdef MSGPACK_FOUND
+    const unsigned int vecSize = msgVec.size();
+
     // create msgpack simple buffer
     msgpack::sbuffer sbuf;
     // create msgpack packer
     msgpack::packer<msgpack::sbuffer> packer(&sbuf);
 
     // pack all parts into a single msgpack simple buffer
-    for (unsigned int i = 0; i < msgVec.size(); ++i)
+    for (unsigned int i = 0; i < vecSize; ++i)
     {
         static_cast<FairMQMessageNN*>(msgVec[i].get())->fReceiving = false;
         packer.pack_bin(msgVec[i]->GetSize());
         packer.pack_bin_body(static_cast<char*>(msgVec[i]->GetData()), msgVec[i]->GetSize());
     }
 
-    int64_t nbytes = nn_send(fSocket, sbuf.data(), sbuf.size(), flags);
+    int64_t nbytes = -1;
+
+    while (true)
+    {
+        nbytes = nn_send(fSocket, sbuf.data(), sbuf.size(), flags);
         if (nbytes >= 0)
         {
             fBytesTx += nbytes;
             ++fMessagesTx;
             return nbytes;
         }
-    if (nn_errno() == EAGAIN)
+        else if (nn_errno() == ETIMEDOUT)
+        {
+            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
+            {
+                continue;
+            }
+            else
             {
                 return -2;
             }
-    if (nn_errno() == ETERM)
+        }
+        else if (nn_errno() == EAGAIN)
+        {
+            return -2;
+        }
+        else if (nn_errno() == ETERM)
         {
             LOG(INFO) << "terminating socket " << fId;
             return -1;
         }
+        else
+        {
             LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
             return nbytes;
+        }
+    }
 #else /*MSGPACK_FOUND*/
-    LOG(ERROR) << "Cannot send message from vector of size " << msgVec.size() << " and flags " << flags << " with nanomsg multipart because MessagePack is not available.";
+    LOG(ERROR) << "Cannot send message from vector of size " << vecSize << " and flags " << flags << " with nanomsg multipart because MessagePack is not available.";
     exit(EXIT_FAILURE);
 #endif /*MSGPACK_FOUND*/
 }
 
-int FairMQSocketNN::Receive(FairMQMessage* msg, const string& flag)
-{
-    return Receive(msg, GetConstant(flag));
-}
-
-int FairMQSocketNN::Receive(FairMQMessage* msg, const int flags)
-{
-    void* ptr = NULL;
-    int nbytes = nn_recv(fSocket, &ptr, NN_MSG, flags);
-    if (nbytes >= 0)
-    {
-        fBytesRx += nbytes;
-        ++fMessagesRx;
-        msg->SetMessage(ptr, nbytes);
-        static_cast<FairMQMessageNN*>(msg)->fReceiving = true;
-        return nbytes;
-    }
-    if (nn_errno() == EAGAIN)
-    {
-        return -2;
-    }
-    if (nn_errno() == ETERM)
-    {
-        LOG(INFO) << "terminating socket " << fId;
-        return -1;
-    }
-    LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
-    return nbytes;
-}
-
 int64_t FairMQSocketNN::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
 {
 #ifdef MSGPACK_FOUND
@@ -217,6 +281,8 @@ int64_t FairMQSocketNN::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, const
         msgVec.clear();
     }
 
+    while (true)
+    {
         // pointer to point to received message buffer
         char* ptr = NULL;
         // receive the message into a buffer allocated by nanomsg and let ptr point to it
@@ -251,17 +317,32 @@ int64_t FairMQSocketNN::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, const
             nn_freemsg(ptr);
             return nbytes;
         }
-    if (nn_errno() == EAGAIN)
+        else if (nn_errno() == ETIMEDOUT)
+        {
+            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
+            {
+                continue;
+            }
+            else
             {
                 return -2;
             }
-    if (nn_errno() == ETERM)
+        }
+        else if (nn_errno() == EAGAIN)
+        {
+            return -2;
+        }
+        else if (nn_errno() == ETERM)
         {
             LOG(INFO) << "terminating socket " << fId;
             return -1;
         }
+        else
+        {
             LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
             return nbytes;
+        }
+    }
 #else /*MSGPACK_FOUND*/
     LOG(ERROR) << "Cannot receive message into vector of size " << msgVec.size() << " and flags " << flags << " with nanomsg multipart because MessagePack is not available.";
     exit(EXIT_FAILURE);
@@ -280,10 +361,12 @@ void FairMQSocketNN::Terminate()
 
 void FairMQSocketNN::Interrupt()
 {
+    fInterrupted = true;
 }
 
 void FairMQSocketNN::Resume()
 {
+    fInterrupted = false;
 }
 
 void* FairMQSocketNN::GetSocket() const

fairmq/nanomsg/FairMQSocketNN.h

@@ -19,6 +19,7 @@
 #include <atomic>
 
 #include "FairMQSocket.h"
+#include "FairMQMessage.h"
 
 class FairMQSocketNN : public FairMQSocket
 {
@@ -32,12 +33,10 @@ class FairMQSocketNN : public FairMQSocket
     virtual bool Bind(const std::string& address);
     virtual void Connect(const std::string& address);
 
-    virtual int Send(FairMQMessage* msg, const std::string& flag = "");
+    virtual int Send(FairMQMessagePtr& msg, const int flags = 0);
-    virtual int Send(FairMQMessage* msg, const int flags = 0);
+    virtual int Receive(FairMQMessagePtr& msg, const int flags = 0);
-    virtual int64_t Send(const std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
 
-    virtual int Receive(FairMQMessage* msg, const std::string& flag = "");
+    virtual int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
-    virtual int Receive(FairMQMessage* msg, const int flags = 0);
     virtual int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
 
     virtual void* GetSocket() const;
@@ -72,6 +71,7 @@ class FairMQSocketNN : public FairMQSocket
     std::atomic<unsigned long> fBytesRx;
     std::atomic<unsigned long> fMessagesTx;
     std::atomic<unsigned long> fMessagesRx;
+    static std::atomic<bool> fInterrupted;
 };
 
 #endif /* FAIRMQSOCKETNN_H_ */

fairmq/nanomsg/FairMQTransportFactoryNN.cxx

@@ -16,9 +16,11 @@
 
 using namespace std;
 
+static FairMQ::Transport gTransportType = FairMQ::Transport::NN;
+
 FairMQTransportFactoryNN::FairMQTransportFactoryNN()
 {
-    LOG(INFO) << "Using nanomsg library";
+    LOG(DEBUG) << "Transport: Using nanomsg library";
 }
 
 FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage() const
@@ -55,3 +57,8 @@ FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const FairMQSocket& cmdSo
 {
     return unique_ptr<FairMQPoller>(new FairMQPollerNN(cmdSocket, dataSocket));
 }
+
+FairMQ::Transport FairMQTransportFactoryNN::GetType() const
+{
+    return gTransportType;
+}

fairmq/nanomsg/FairMQTransportFactoryNN.h

@@ -16,6 +16,7 @@
 #define FAIRMQTRANSPORTFACTORYNN_H_
 
 #include <vector>
+#include <string>
 
 #include "FairMQTransportFactory.h"
 #include "FairMQMessageNN.h"
@@ -37,6 +38,8 @@ class FairMQTransportFactoryNN : public FairMQTransportFactory
     virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const;
     virtual FairMQPollerPtr CreatePoller(const FairMQSocket& cmdSocket, const FairMQSocket& dataSocket) const;
 
+    virtual FairMQ::Transport GetType() const;
+
     virtual ~FairMQTransportFactoryNN() {};
 };
 

fairmq/options/FairMQParser.cxx

@@ -228,6 +228,7 @@ void ChannelParser(const boost::property_tree::ptree& tree, FairMQMap& channelMa
                 commonChannel.UpdateType(q.second.get<string>("type", commonChannel.GetType()));
                 commonChannel.UpdateMethod(q.second.get<string>("method", commonChannel.GetMethod()));
                 commonChannel.UpdateAddress(q.second.get<string>("address", commonChannel.GetAddress()));
+                commonChannel.UpdateTransport(q.second.get<string>("transport", commonChannel.GetTransport()));
                 commonChannel.UpdateSndBufSize(q.second.get<int>("sndBufSize", commonChannel.GetSndBufSize()));
                 commonChannel.UpdateRcvBufSize(q.second.get<int>("rcvBufSize", commonChannel.GetRcvBufSize()));
                 commonChannel.UpdateSndKernelSize(q.second.get<int>("sndKernelSize", commonChannel.GetSndKernelSize()));
@@ -246,6 +247,7 @@ void ChannelParser(const boost::property_tree::ptree& tree, FairMQMap& channelMa
                     LOG(DEBUG) << "\ttype          = " << commonChannel.GetType();
                     LOG(DEBUG) << "\tmethod        = " << commonChannel.GetMethod();
                     LOG(DEBUG) << "\taddress       = " << commonChannel.GetAddress();
+                    LOG(DEBUG) << "\ttransport     = " << commonChannel.GetTransport();
                     LOG(DEBUG) << "\tsndBufSize    = " << commonChannel.GetSndBufSize();
                     LOG(DEBUG) << "\trcvBufSize    = " << commonChannel.GetRcvBufSize();
                     LOG(DEBUG) << "\tsndKernelSize = " << commonChannel.GetSndKernelSize();
@@ -289,6 +291,7 @@ void ChannelParser(const boost::property_tree::ptree& tree, FairMQMap& channelMa
                 commonChannel.UpdateType(p.second.get<string>("type", commonChannel.GetType()));
                 commonChannel.UpdateMethod(p.second.get<string>("method", commonChannel.GetMethod()));
                 commonChannel.UpdateAddress(p.second.get<string>("address", commonChannel.GetAddress()));
+                commonChannel.UpdateTransport(p.second.get<string>("transport", commonChannel.GetTransport()));
                 commonChannel.UpdateSndBufSize(p.second.get<int>("sndBufSize", commonChannel.GetSndBufSize()));
                 commonChannel.UpdateRcvBufSize(p.second.get<int>("rcvBufSize", commonChannel.GetRcvBufSize()));
                 commonChannel.UpdateSndKernelSize(p.second.get<int>("sndKernelSize", commonChannel.GetSndKernelSize()));
@@ -308,6 +311,7 @@ void ChannelParser(const boost::property_tree::ptree& tree, FairMQMap& channelMa
                 LOG(DEBUG) << "\ttype          = " << commonChannel.GetType();
                 LOG(DEBUG) << "\tmethod        = " << commonChannel.GetMethod();
                 LOG(DEBUG) << "\taddress       = " << commonChannel.GetAddress();
+                LOG(DEBUG) << "\ttransport     = " << commonChannel.GetTransport();
                 LOG(DEBUG) << "\tsndBufSize    = " << commonChannel.GetSndBufSize();
                 LOG(DEBUG) << "\trcvBufSize    = " << commonChannel.GetRcvBufSize();
                 LOG(DEBUG) << "\tsndKernelSize = " << commonChannel.GetSndKernelSize();
@@ -348,6 +352,7 @@ void SocketParser(const boost::property_tree::ptree& tree, vector<FairMQChannel>
                 channel.UpdateType(q.second.get<string>("type", channel.GetType()));
                 channel.UpdateMethod(q.second.get<string>("method", channel.GetMethod()));
                 channel.UpdateAddress(q.second.get<string>("address", channel.GetAddress()));
+                channel.UpdateTransport(q.second.get<string>("transport", channel.GetTransport()));
                 channel.UpdateSndBufSize(q.second.get<int>("sndBufSize", channel.GetSndBufSize()));
                 channel.UpdateRcvBufSize(q.second.get<int>("rcvBufSize", channel.GetRcvBufSize()));
                 channel.UpdateSndKernelSize(q.second.get<int>("sndKernelSize", channel.GetSndKernelSize()));
@@ -358,6 +363,7 @@ void SocketParser(const boost::property_tree::ptree& tree, vector<FairMQChannel>
                 LOG(DEBUG) << "\ttype          = " << channel.GetType();
                 LOG(DEBUG) << "\tmethod        = " << channel.GetMethod();
                 LOG(DEBUG) << "\taddress       = " << channel.GetAddress();
+                LOG(DEBUG) << "\ttransport     = " << channel.GetTransport();
                 LOG(DEBUG) << "\tsndBufSize    = " << channel.GetSndBufSize();
                 LOG(DEBUG) << "\trcvBufSize    = " << channel.GetRcvBufSize();
                 LOG(DEBUG) << "\tsndKernelSize = " << channel.GetSndKernelSize();
@@ -378,6 +384,7 @@ void SocketParser(const boost::property_tree::ptree& tree, vector<FairMQChannel>
             channel.UpdateType(p.second.get<string>("type", channel.GetType()));
             channel.UpdateMethod(p.second.get<string>("method", channel.GetMethod()));
             channel.UpdateAddress(p.second.get<string>("address", channel.GetAddress()));
+            channel.UpdateTransport(p.second.get<string>("transport", channel.GetTransport()));
             channel.UpdateSndBufSize(p.second.get<int>("sndBufSize", channel.GetSndBufSize()));
             channel.UpdateRcvBufSize(p.second.get<int>("rcvBufSize", channel.GetRcvBufSize()));
             channel.UpdateSndKernelSize(p.second.get<int>("sndKernelSize", channel.GetSndKernelSize()));
@@ -388,6 +395,7 @@ void SocketParser(const boost::property_tree::ptree& tree, vector<FairMQChannel>
             LOG(DEBUG) << "\ttype          = " << channel.GetType();
             LOG(DEBUG) << "\tmethod        = " << channel.GetMethod();
             LOG(DEBUG) << "\taddress       = " << channel.GetAddress();
+            LOG(DEBUG) << "\ttransport     = " << channel.GetTransport();
             LOG(DEBUG) << "\tsndBufSize    = " << channel.GetSndBufSize();
             LOG(DEBUG) << "\trcvBufSize    = " << channel.GetRcvBufSize();
             LOG(DEBUG) << "\tsndKernelSize = " << channel.GetSndKernelSize();
@@ -414,6 +422,7 @@ void SocketParser(const boost::property_tree::ptree& tree, vector<FairMQChannel>
         LOG(DEBUG) << "\ttype          = " << channel.GetType();
         LOG(DEBUG) << "\tmethod        = " << channel.GetMethod();
         LOG(DEBUG) << "\taddress       = " << channel.GetAddress();
+        LOG(DEBUG) << "\ttransport     = " << channel.GetTransport();
         LOG(DEBUG) << "\tsndBufSize    = " << channel.GetSndBufSize();
         LOG(DEBUG) << "\trcvBufSize    = " << channel.GetRcvBufSize();
         LOG(DEBUG) << "\tsndKernelSize = " << channel.GetSndKernelSize();

fairmq/options/FairMQProgOptions.cxx

@@ -233,6 +233,7 @@ void FairMQProgOptions::UpdateMQValues()
             string typeKey = p.first + "." + to_string(index) + ".type";
             string methodKey = p.first + "." + to_string(index) + ".method";
             string addressKey = p.first + "." + to_string(index) + ".address";
+            string transportKey = p.first + "." + to_string(index) + ".transport";
             string sndBufSizeKey = p.first + "." + to_string(index) + ".sndBufSize";
             string rcvBufSizeKey = p.first + "." + to_string(index) + ".rcvBufSize";
             string sndKernelSizeKey = p.first + "." + to_string(index) + ".sndKernelSize";
@@ -242,6 +243,7 @@ void FairMQProgOptions::UpdateMQValues()
             fMQKeyMap[typeKey] = make_tuple(p.first, index, "type");
             fMQKeyMap[methodKey] = make_tuple(p.first, index, "method");
             fMQKeyMap[addressKey] = make_tuple(p.first, index, "address");
+            fMQKeyMap[transportKey] = make_tuple(p.first, index, "transport");
             fMQKeyMap[sndBufSizeKey] = make_tuple(p.first, index, "sndBufSize");
             fMQKeyMap[rcvBufSizeKey] = make_tuple(p.first, index, "rcvBufSize");
             fMQKeyMap[sndKernelSizeKey] = make_tuple(p.first, index, "sndKernelSize");
@@ -251,6 +253,7 @@ void FairMQProgOptions::UpdateMQValues()
             UpdateVarMap<string>(typeKey, channel.GetType());
             UpdateVarMap<string>(methodKey, channel.GetMethod());
             UpdateVarMap<string>(addressKey, channel.GetAddress());
+            UpdateVarMap<string>(transportKey, channel.GetTransport());
 
 
             //UpdateVarMap<string>(sndBufSizeKey, to_string(channel.GetSndBufSize()));// string API
@@ -384,6 +387,12 @@ int FairMQProgOptions::UpdateChannelMap(const string& channelName, int index, co
         fFairMQMap.at(channelName).at(index).UpdateAddress(val);
         return 0;
     }
+
+    if (member == "transport")
+    {
+        fFairMQMap.at(channelName).at(index).UpdateTransport(val);
+        return 0;
+    }
     else
     {
         //if we get there it means something is wrong

fairmq/run/startMQBenchmark.sh.in

@@ -4,6 +4,10 @@ numMsgs="0"
 msgSize="1000000"
 transport="zeromq"
 sameMsg="true"
+affinity="false"
+affinitySamp=""
+affinitySink=""
+
 
 if [[ $1 =~ ^[0-9]+$ ]]; then
     msgSize=$1
@@ -21,17 +25,40 @@ if [[ $4 =~ ^[a-z]+$ ]]; then
     sameMsg=$4
 fi
 
-echo "Starting benchmark with message size of $msgSize bytes ($numMsgs messages) and $transport transport."
+if [[ $5 =~ ^[a-z]+$ ]]; then
-echo "Using $transport transport."
+    affinity=$5
+fi
+
+
+echo "Starting benchmark with following settings:"
+
+echo ""
+echo "message size: $msgSize bytes"
 
 if [ $numMsgs = 0 ]; then
-    echo "Unlimited number of messages."
+    echo "number of messages: unlimited"
 else
-    echo "Number of messages: $numMsgs."
+    echo "number of messages: $numMsgs"
+fi
+
+echo "transport: $transport"
+
+if [ $sameMsg = "true" ]; then
+    echo "resend same message: yes, using Copy() method to resend the same message"
+else
+    echo "resend same message: no, allocating each message separately"
+fi
+
+if [ $affinity = "true" ]; then
+    affinitySamp="taskset -c 0"
+    affinitySink="taskset -c 1"
+    echo "affinity: assigning sampler to core 0, sink to core 1"
+else
+    echo ""
 fi
 
 echo ""
-echo "Usage: startBenchmark [message size=1000000] [number of messages=0] [transport=zeromq/nanomsg/shmem] [resend same message=true]"
+echo "Usage: startBenchmark [message size=1000000] [number of messages=0] [transport=zeromq/nanomsg/shmem] [resend same message=true] [affinity=false]"
 
 SAMPLER="bsampler"
 SAMPLER+=" --id bsampler1"
@@ -43,7 +70,7 @@ SAMPLER+=" --same-msg $sameMsg"
 # SAMPLER+=" --msg-rate 1000"
 SAMPLER+=" --num-msgs $numMsgs"
 SAMPLER+=" --mq-config @CMAKE_BINARY_DIR@/bin/config/benchmark.json"
-xterm -geometry 80x23+0+0 -hold -e @CMAKE_BINARY_DIR@/bin/$SAMPLER &
+xterm -geometry 90x23+0+0 -hold -e $affinitySamp @CMAKE_BINARY_DIR@/bin/$SAMPLER &
 
 SINK="sink"
 SINK+=" --id sink1"
@@ -52,4 +79,4 @@ SINK+=" --id sink1"
 SINK+=" --transport $transport"
 SINK+=" --num-msgs $numMsgs"
 SINK+=" --mq-config @CMAKE_BINARY_DIR@/bin/config/benchmark.json"
-xterm -geometry 80x23+500+0 -hold -e @CMAKE_BINARY_DIR@/bin/$SINK &
+xterm -geometry 90x23+550+0 -hold -e $affinitySink @CMAKE_BINARY_DIR@/bin/$SINK &

fairmq/shmem/FairMQContextSHM.cxx

@@ -7,6 +7,8 @@
  ********************************************************************************/
 #include <zmq.h>
 
+#include <cstdio>
+
 #include <boost/interprocess/managed_shared_memory.hpp>
 
 #include "FairMQLogger.h"
@@ -46,11 +48,11 @@ FairMQContextSHM::~FairMQContextSHM()
 
     if (boost::interprocess::shared_memory_object::remove("FairMQSharedMemory"))
     {
-        LOG(INFO) << "Successfully removed shared memory after the device has stopped.";
+        printf("Successfully removed shared memory after the device has stopped.\n");
     }
     else
     {
-        LOG(INFO) << "Did not remove shared memory after the device stopped. Still in use?";
+        printf("Did not remove shared memory after the device stopped. Already removed?\n");
     }
 }
 

fairmq/shmem/FairMQMessageSHM.cxx

@@ -14,45 +14,60 @@
 using namespace std;
 using namespace FairMQ::shmem;
 
-uint64_t FairMQMessageSHM::fMessageID = 0;
+static FairMQ::Transport gTransportType = FairMQ::Transport::SHM;
-string FairMQMessageSHM::fDeviceID = string();
+
+// uint64_t FairMQMessageSHM::fMessageID = 0;
+// string FairMQMessageSHM::fDeviceID = string();
 atomic<bool> FairMQMessageSHM::fInterrupted(false);
 
 FairMQMessageSHM::FairMQMessageSHM()
     : fMessage()
-    , fOwner(nullptr)
+    // , fOwner(nullptr)
-    , fReceiving(false)
+    // , fReceiving(false)
     , fQueued(false)
+    , fMetaCreated(false)
+    , fHandle()
+    , fChunkSize(0)
+    , fLocalPtr(nullptr)
 {
     if (zmq_msg_init(&fMessage) != 0)
     {
         LOG(ERROR) << "failed initializing message, reason: " << zmq_strerror(errno);
     }
+    fMetaCreated = true;
 }
 
-void FairMQMessageSHM::StringDeleter(void* /*data*/, void* str)
+// void FairMQMessageSHM::StringDeleter(void* /*data*/, void* str)
-{
+// {
-    delete static_cast<string*>(str);
+//     delete static_cast<string*>(str);
-}
+// }
 
 FairMQMessageSHM::FairMQMessageSHM(const size_t size)
     : fMessage()
-    , fOwner(nullptr)
+    // , fOwner(nullptr)
-    , fReceiving(false)
+    // , fReceiving(false)
     , fQueued(false)
+    , fMetaCreated(false)
+    , fHandle()
+    , fChunkSize(0)
+    , fLocalPtr(nullptr)
 {
     InitializeChunk(size);
 }
 
 FairMQMessageSHM::FairMQMessageSHM(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
     : fMessage()
-    , fOwner(nullptr)
+    // , fOwner(nullptr)
-    , fReceiving(false)
+    // , fReceiving(false)
     , fQueued(false)
+    , fMetaCreated(false)
+    , fHandle()
+    , fChunkSize(0)
+    , fLocalPtr(nullptr)
 {
     if (InitializeChunk(size))
     {
-        memcpy(fOwner->fPtr->GetData(), data, size);
+        memcpy(fLocalPtr, data, size);
         if (ffn)
         {
             ffn(data, hint);
@@ -66,66 +81,76 @@ FairMQMessageSHM::FairMQMessageSHM(void* data, const size_t size, fairmq_free_fn
 
 bool FairMQMessageSHM::InitializeChunk(const size_t size)
 {
-    string chunkID = fDeviceID + "c" + to_string(fMessageID);
+    // string chunkID = fDeviceID + "c" + to_string(fMessageID);
-    string* ownerID = new string(fDeviceID + "o" + to_string(fMessageID));
+    // string* ownerID = new string(fDeviceID + "o" + to_string(fMessageID));
 
-    bool success = false;
+    while (!fHandle)
-
-    while (!success)
     {
         try
         {
-            fOwner = Manager::Instance().Segment()->construct<ShPtrOwner>(ownerID->c_str())(
+            fLocalPtr = Manager::Instance().Segment()->allocate(size);
-                make_managed_shared_ptr(Manager::Instance().Segment()->construct<Chunk>(chunkID.c_str())(size),
+
-                                        *(Manager::Instance().Segment())));
+            // fOwner = Manager::Instance().Segment()->construct<ShPtrOwner>(ownerID->c_str())(
-            success = true;
+            //     make_managed_shared_ptr(Manager::Instance().Segment()->construct<Chunk>(chunkID.c_str())(size),
+            //                             *(Manager::Instance().Segment())));
         }
         catch (bipc::bad_alloc& ba)
         {
-            LOG(WARN) << "Shared memory full...";
+            // LOG(WARN) << "Shared memory full...";
             this_thread::sleep_for(chrono::milliseconds(50));
             if (fInterrupted)
             {
-                break;
+                return false;
             }
             else
             {
                 continue;
             }
         }
+        fHandle = Manager::Instance().Segment()->get_handle_from_address(fLocalPtr);
     }
 
-    if (success)
+    fChunkSize = size;
-    {
+
-        if (zmq_msg_init_data(&fMessage, const_cast<char*>(ownerID->c_str()), ownerID->length(), StringDeleter, ownerID) != 0)
+    if (zmq_msg_init_size(&fMessage, sizeof(MetaHeader)) != 0)
     {
         LOG(ERROR) << "failed initializing meta message, reason: " << zmq_strerror(errno);
+        return false;
     }
+    MetaHeader* metaPtr = new(zmq_msg_data(&fMessage)) MetaHeader();
+    metaPtr->fSize = size;
+    metaPtr->fHandle = fHandle;
 
-        ++fMessageID;
+    // if (zmq_msg_init_data(&fMessage, const_cast<char*>(ownerID->c_str()), ownerID->length(), StringDeleter, ownerID) != 0)
-    }
+    // {
+    //     LOG(ERROR) << "failed initializing meta message, reason: " << zmq_strerror(errno);
+    // }
+    fMetaCreated = true;
 
-    return success;
+    // ++fMessageID;
+
+    return true;
 }
 
 void FairMQMessageSHM::Rebuild()
 {
     CloseMessage();
 
-    fReceiving = false;
+    // fReceiving = false;
     fQueued = false;
 
     if (zmq_msg_init(&fMessage) != 0)
     {
         LOG(ERROR) << "failed initializing message, reason: " << zmq_strerror(errno);
     }
+    fMetaCreated = true;
 }
 
 void FairMQMessageSHM::Rebuild(const size_t size)
 {
     CloseMessage();
 
-    fReceiving = false;
+    // fReceiving = false;
     fQueued = false;
 
     InitializeChunk(size);
@@ -135,12 +160,12 @@ void FairMQMessageSHM::Rebuild(void* data, const size_t size, fairmq_free_fn* ff
 {
     CloseMessage();
 
-    fReceiving = false;
+    // fReceiving = false;
     fQueued = false;
 
     if (InitializeChunk(size))
     {
-        memcpy(fOwner->fPtr->GetData(), data, size);
+        memcpy(fLocalPtr, data, size);
         if (ffn)
         {
             ffn(data, hint);
@@ -159,27 +184,42 @@ void* FairMQMessageSHM::GetMessage()
 
 void* FairMQMessageSHM::GetData()
 {
-    if (fOwner)
+    if (fLocalPtr)
     {
-        return fOwner->fPtr->GetData();
+        return fLocalPtr;
+    }
+    else if (fHandle)
+    {
+        return Manager::Instance().Segment()->get_address_from_handle(fHandle);
     }
     else
     {
-        LOG(ERROR) << "Trying to get data of an empty shared memory message";
+        // LOG(ERROR) << "Trying to get data of an empty shared memory message";
-        exit(EXIT_FAILURE);
+        return nullptr;
     }
+
+    // if (fOwner)
+    // {
+    //     return fOwner->fPtr->GetData();
+    // }
+    // else
+    // {
+    //     LOG(ERROR) << "Trying to get data of an empty shared memory message";
+    //     exit(EXIT_FAILURE);
+    // }
 }
 
 size_t FairMQMessageSHM::GetSize()
 {
-    if (fOwner)
+    return fChunkSize;
-    {
+    // if (fOwner)
-        return fOwner->fPtr->GetSize();
+    // {
-    }
+    //     return fOwner->fPtr->GetSize();
-    else
+    // }
-    {
+    // else
-        return 0;
+    // {
-    }
+    //     return 0;
+    // }
 }
 
 void FairMQMessageSHM::SetMessage(void*, const size_t)
@@ -187,21 +227,26 @@ void FairMQMessageSHM::SetMessage(void*, const size_t)
     // dummy method to comply with the interface. functionality not allowed in zeromq.
 }
 
-void FairMQMessageSHM::SetDeviceId(const string& deviceId)
+void FairMQMessageSHM::SetDeviceId(const string& /*deviceId*/)
 {
-    fDeviceID = deviceId;
+    // fDeviceID = deviceId;
+}
+
+FairMQ::Transport FairMQMessageSHM::GetType() const
+{
+    return gTransportType;
 }
 
 void FairMQMessageSHM::Copy(const unique_ptr<FairMQMessage>& msg)
 {
-    if (!fOwner)
+    if (!fHandle)
     {
-        FairMQ::shmem::ShPtrOwner* otherOwner = static_cast<FairMQMessageSHM*>(msg.get())->fOwner;
+        bipc::managed_shared_memory::handle_t otherHandle = static_cast<FairMQMessageSHM*>(msg.get())->fHandle;
-        if (otherOwner)
+        if (otherHandle)
         {
-            if (InitializeChunk(otherOwner->fPtr->GetSize()))
+            if (InitializeChunk(msg->GetSize()))
             {
-                memcpy(fOwner->fPtr->GetData(), otherOwner->fPtr->GetData(), otherOwner->fPtr->GetSize());
+                memcpy(GetData(), msg->GetData(), msg->GetSize());
             }
         }
         else
@@ -266,33 +311,37 @@ void FairMQMessageSHM::Copy(const unique_ptr<FairMQMessage>& msg)
 
 void FairMQMessageSHM::CloseMessage()
 {
-    if (fReceiving)
+    // if (fReceiving)
+    // {
+    //     if (fOwner)
+    //     {
+    //         Manager::Instance().Segment()->destroy_ptr(fOwner);
+    //         fOwner = nullptr;
+    //     }
+    //     else
+    //     {
+    //         LOG(ERROR) << "No shared pointer owner when closing a received message";
+    //     }
+    // }
+    // else
+    // {
+        if (fHandle && !fQueued)
         {
-        if (fOwner)
+            // LOG(WARN) << "Destroying unsent message";
-        {
+            // Manager::Instance().Segment()->destroy_ptr(fHandle);
-            Manager::Instance().Segment()->destroy_ptr(fOwner);
+            Manager::Instance().Segment()->deallocate(Manager::Instance().Segment()->get_address_from_handle(fHandle));
-            fOwner = nullptr;
+            fHandle = 0;
-        }
-        else
-        {
-            LOG(ERROR) << "No shared pointer owner when closing a received message";
-        }
-    }
-    else
-    {
-        if (fOwner && !fQueued)
-        {
-            LOG(WARN) << "Destroying unsent message";
-            Manager::Instance().Segment()->destroy_ptr(fOwner);
-            fOwner = nullptr;
-        }
         }
+    // }
 
+    if (fMetaCreated)
+    {
         if (zmq_msg_close(&fMessage) != 0)
         {
             LOG(ERROR) << "failed closing message, reason: " << zmq_strerror(errno);
         }
     }
+}
 
 FairMQMessageSHM::~FairMQMessageSHM()
 {

fairmq/shmem/FairMQMessageSHM.h

@@ -42,22 +42,28 @@ class FairMQMessageSHM : public FairMQMessage
 
     virtual void SetDeviceId(const std::string& deviceId);
 
+    virtual FairMQ::Transport GetType() const;
+
     virtual void Copy(const std::unique_ptr<FairMQMessage>& msg);
 
     void CloseMessage();
 
     virtual ~FairMQMessageSHM();
 
-    static void StringDeleter(void* data, void* str);
+    // static void StringDeleter(void* data, void* str);
 
   private:
     zmq_msg_t fMessage;
-    FairMQ::shmem::ShPtrOwner* fOwner;
+    // FairMQ::shmem::ShPtrOwner* fOwner;
-    static uint64_t fMessageID;
+    // static uint64_t fMessageID;
-    static std::string fDeviceID;
+    // static std::string fDeviceID;
-    bool fReceiving;
+    // bool fReceiving;
     bool fQueued;
+    bool fMetaCreated;
     static std::atomic<bool> fInterrupted;
+    bipc::managed_shared_memory::handle_t fHandle;
+    size_t fChunkSize;
+    void* fLocalPtr;
 };
 
 #endif /* FAIRMQMESSAGESHM_H_ */

fairmq/shmem/FairMQPollerSHM.cxx

@@ -51,7 +51,7 @@ FairMQPollerSHM::FairMQPollerSHM(const vector<FairMQChannel>& channels)
         }
         else
         {
-            LOG(ERROR) << "invalid poller configuration, exiting.";
+            LOG(ERROR) << "shmem: invalid poller configuration, exiting.";
             exit(EXIT_FAILURE);
         }
     }
@@ -105,7 +105,7 @@ FairMQPollerSHM::FairMQPollerSHM(const unordered_map<string, vector<FairMQChanne
                 }
                 else
                 {
-                    LOG(ERROR) << "invalid poller configuration, exiting.";
+                    LOG(ERROR) << "shmem: invalid poller configuration, exiting.";
                     exit(EXIT_FAILURE);
                 }
             }
@@ -113,8 +113,8 @@ FairMQPollerSHM::FairMQPollerSHM(const unordered_map<string, vector<FairMQChanne
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "At least one of the provided channel keys for poller initialization is invalid";
+        LOG(ERROR) << "shmem: at least one of the provided channel keys for poller initialization is invalid";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "shmem: out of range error: " << oor.what() << '\n';
         exit(EXIT_FAILURE);
     }
 }
@@ -153,7 +153,7 @@ FairMQPollerSHM::FairMQPollerSHM(const FairMQSocket& cmdSocket, const FairMQSock
     }
     else
     {
-        LOG(ERROR) << "invalid poller configuration, exiting.";
+        LOG(ERROR) << "shmem: invalid poller configuration, exiting.";
         exit(EXIT_FAILURE);
     }
 }
@@ -164,11 +164,12 @@ void FairMQPollerSHM::Poll(const int timeout)
     {
         if (errno == ETERM)
         {
-            LOG(DEBUG) << "polling exited, reason: " << zmq_strerror(errno);
+            LOG(DEBUG) << "shmem: polling exited, reason: " << zmq_strerror(errno);
         }
         else
         {
-            LOG(ERROR) << "polling failed, reason: " << zmq_strerror(errno);
+            LOG(ERROR) << "shmem: polling failed, reason: " << zmq_strerror(errno);
+            throw std::runtime_error("shmem: polling failed");
         }
     }
 }
@@ -206,8 +207,8 @@ bool FairMQPollerSHM::CheckInput(const string channelKey, const int index)
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
+        LOG(ERROR) << "shmem: invalid channel key: \"" << channelKey << "\"";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "shmem: out of range error: " << oor.what() << '\n';
         exit(EXIT_FAILURE);
     }
 }
@@ -225,8 +226,8 @@ bool FairMQPollerSHM::CheckOutput(const string channelKey, const int index)
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
+        LOG(ERROR) << "shmem: Invalid channel key: \"" << channelKey << "\"";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "shmem: out of range error: " << oor.what() << '\n';
         exit(EXIT_FAILURE);
     }
 }

fairmq/shmem/FairMQShmManager.h

@@ -121,62 +121,68 @@ class Manager
     bipc::managed_shared_memory* fSegment;
 };
 
-class Chunk
+struct alignas(16) MetaHeader
 {
-  public:
+    uint64_t fSize;
-    Chunk()
+    bipc::managed_shared_memory::handle_t fHandle;
-        : fHandle()
+};
-        , fSize(0)
-    {
-    }
 
-    Chunk(const size_t size)
+// class Chunk
-        : fHandle()
+// {
-        , fSize(size)
+//   public:
-    {
+//     Chunk()
-        void* ptr = Manager::Instance().Segment()->allocate(size);
+//         : fHandle()
-        fHandle = Manager::Instance().Segment()->get_handle_from_address(ptr);
+//         , fSize(0)
-    }
+//     {
+//     }
 
-    ~Chunk()
+//     Chunk(const size_t size)
-    {
+//         : fHandle()
-        Manager::Instance().Segment()->deallocate(Manager::Instance().Segment()->get_address_from_handle(fHandle));
+//         , fSize(size)
-    }
+//     {
+//         void* ptr = Manager::Instance().Segment()->allocate(size);
+//         fHandle = Manager::Instance().Segment()->get_handle_from_address(ptr);
+//     }
+
+//     ~Chunk()
+//     {
+//         Manager::Instance().Segment()->deallocate(Manager::Instance().Segment()->get_address_from_handle(fHandle));
+//     }
 
 //     bipc::managed_shared_memory::handle_t GetHandle() const
 //     {
 //         return fHandle;
 //     }
 
-    void* GetData() const
+//     void* GetData() const
-    {
+//     {
-        return Manager::Instance().Segment()->get_address_from_handle(fHandle);
+//         return Manager::Instance().Segment()->get_address_from_handle(fHandle);
-    }
+//     }
 
-    size_t GetSize() const
+//     size_t GetSize() const
-    {
+//     {
-        return fSize;
+//         return fSize;
-    }
+//     }
 
-  private:
+//   private:
-    bipc::managed_shared_memory::handle_t fHandle;
+//     bipc::managed_shared_memory::handle_t fHandle;
-    size_t fSize;
+//     size_t fSize;
-};
+// };
 
-typedef bipc::managed_shared_ptr<Chunk, bipc::managed_shared_memory>::type ShPtrType;
+// typedef bipc::managed_shared_ptr<Chunk, bipc::managed_shared_memory>::type ShPtrType;
 
-struct ShPtrOwner
+// struct ShPtrOwner
-{
+// {
-    ShPtrOwner(const ShPtrType& other)
+//     ShPtrOwner(const ShPtrType& other)
-        : fPtr(other)
+//         : fPtr(other)
-        {}
+//         {}
 
-    ShPtrOwner(const ShPtrOwner& other)
+//     ShPtrOwner(const ShPtrOwner& other)
-        : fPtr(other.fPtr)
+//         : fPtr(other.fPtr)
-        {}
+//         {}
 
-    ShPtrType fPtr;
+//     ShPtrType fPtr;
-};
+// };
 
 } // namespace shmem
 

fairmq/shmem/FairMQSocketSHM.cxx

@@ -19,6 +19,7 @@ using namespace FairMQ::shmem;
 // Context to hold the ZeroMQ sockets
 unique_ptr<FairMQContextSHM> FairMQSocketSHM::fContext; // = unique_ptr<FairMQContextSHM>(new FairMQContextSHM(1));
 bool FairMQSocketSHM::fContextInitialized = false;
+atomic<bool> FairMQSocketSHM::fInterrupted(false);
 
 FairMQSocketSHM::FairMQSocketSHM(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
     : FairMQSocket(ZMQ_SNDMORE, ZMQ_RCVMORE, ZMQ_DONTWAIT)
@@ -57,22 +58,22 @@ FairMQSocketSHM::FairMQSocketSHM(const string& type, const string& name, const i
 
     // Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
     // Default value for ZeroMQ is -1, which is to wait forever.
-    int linger = 500;
+    int linger = 1000;
     if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0)
     {
         LOG(ERROR) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
     }
 
-    int kernelSndSize = 10000;
+    int sndTimeout = 700;
-    if (zmq_setsockopt(fSocket, ZMQ_SNDBUF, &kernelSndSize, sizeof(kernelSndSize)) != 0)
+    if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &sndTimeout, sizeof(sndTimeout)) != 0)
     {
-        LOG(ERROR) << "Failed setting ZMQ_SNDBUF socket option, reason: " << zmq_strerror(errno);
+        LOG(ERROR) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
     }
 
-    int kernelRcvSize = 10000;
+    int rcvTimeout = 700;
-    if (zmq_setsockopt(fSocket, ZMQ_RCVBUF, &kernelRcvSize, sizeof(kernelRcvSize)) != 0)
+    if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &rcvTimeout, sizeof(rcvTimeout)) != 0)
     {
-        LOG(ERROR) << "Failed setting ZMQ_RCVBUF socket option, reason: " << zmq_strerror(errno);
+        LOG(ERROR) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
     }
 
     if (type == "sub")
@@ -119,136 +120,78 @@ void FairMQSocketSHM::Connect(const string& address)
     }
 }
 
-int FairMQSocketSHM::Send(FairMQMessage* msg, const string& flag)
+int FairMQSocketSHM::Send(FairMQMessagePtr& msg, const int flags)
 {
-    return Send(msg, GetConstant(flag));
+    int nbytes = -1;
+    while (true && !fInterrupted)
+    {
+        nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
+        if (nbytes == 0)
+        {
+            return nbytes;
         }
+        else if (nbytes > 0)
+        {
+            // static_cast<FairMQMessageSHM*>(msg.get())->fReceiving = false;
+            static_cast<FairMQMessageSHM*>(msg.get())->fQueued = true;
 
-int FairMQSocketSHM::Send(FairMQMessage* msg, const int flags)
-{
-    int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
-    if (nbytes >= 0)
-    {
-        static_cast<FairMQMessageSHM*>(msg)->fReceiving = false;
-        static_cast<FairMQMessageSHM*>(msg)->fQueued = true;
             size_t size = msg->GetSize();
-
             fBytesTx += size;
             ++fMessagesTx;
 
             return size;
         }
-    if (zmq_errno() == EAGAIN)
+        else if (zmq_errno() == EAGAIN)
         {
-        return -2;
+            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
-    }
-    if (zmq_errno() == ETERM)
             {
-        LOG(INFO) << "terminating socket " << fId;
+                continue;
-        return -1;
-    }
-    LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-    return nbytes;
-}
-
-int64_t FairMQSocketSHM::Send(const vector<FairMQMessagePtr>& msgVec, const int flags)
-{
-    // Sending vector typicaly handles more then one part
-    if (msgVec.size() > 1)
-    {
-        int64_t totalSize = 0;
-
-        for (unsigned int i = 0; i < msgVec.size() - 1; ++i)
-        {
-            int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()), fSocket, ZMQ_SNDMORE|flags);
-            if (nbytes >= 0)
-            {
-                static_cast<FairMQMessageSHM*>(msgVec[i].get())->fReceiving = false;
-                static_cast<FairMQMessageSHM*>(msgVec[i].get())->fQueued = true;
-                size_t size = msgVec[i]->GetSize();
-
-                totalSize += size;
-                fBytesTx += size;
             }
             else
-            {
-                if (zmq_errno() == EAGAIN)
             {
                 return -2;
             }
-                if (zmq_errno() == ETERM)
+        }
+        else if (zmq_errno() == ETERM)
         {
             LOG(INFO) << "terminating socket " << fId;
             return -1;
         }
-                LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-                return nbytes;
-            }
-        }
-
-        int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec.back()->GetMessage()), fSocket, flags);
-        if (nbytes >= 0)
-        {
-            static_cast<FairMQMessageSHM*>(msgVec.back().get())->fReceiving = false;
-            static_cast<FairMQMessageSHM*>(msgVec.back().get())->fQueued = true;
-            size_t size = msgVec.back()->GetSize();
-
-            totalSize += size;
-            fBytesTx += size;
-        }
         else
         {
-            if (zmq_errno() == EAGAIN)
-            {
-                return -2;
-            }
-            if (zmq_errno() == ETERM)
-            {
-                LOG(INFO) << "terminating socket " << fId;
-                return -1;
-            }
             LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
             return nbytes;
         }
-
-        // store statistics on how many messages have been sent (handle all parts as a single message)
-        ++fMessagesTx;
-        return totalSize;
-    } // If there's only one part, send it as a regular message
-    else if (msgVec.size() == 1)
-    {
-        return Send(msgVec.back().get(), flags);
     }
-    else // if the vector is empty, something might be wrong
+
-    {
-        LOG(WARN) << "Will not send empty vector";
     return -1;
 }
-}
 
-int FairMQSocketSHM::Receive(FairMQMessage* msg, const string& flag)
+int FairMQSocketSHM::Receive(FairMQMessagePtr& msg, const int flags)
-{
-    return Receive(msg, GetConstant(flag));
-}
-
-int FairMQSocketSHM::Receive(FairMQMessage* msg, const int flags)
 {
+    int nbytes = -1;
     zmq_msg_t* msgPtr = static_cast<zmq_msg_t*>(msg->GetMessage());
-    int nbytes = zmq_msg_recv(msgPtr, fSocket, flags);
+    while (true)
+    {
+        nbytes = zmq_msg_recv(msgPtr, fSocket, flags);
         if (nbytes == 0)
         {
             ++fMessagesRx;
+
             return nbytes;
         }
         else if (nbytes > 0)
         {
-        string ownerID(static_cast<char*>(zmq_msg_data(msgPtr)), zmq_msg_size(msgPtr));
+            // string ownerID(static_cast<char*>(zmq_msg_data(msgPtr)), zmq_msg_size(msgPtr));
-        ShPtrOwner* owner = Manager::Instance().Segment()->find<ShPtrOwner>(ownerID.c_str()).first;
+            // ShPtrOwner* owner = Manager::Instance().Segment()->find<ShPtrOwner>(ownerID.c_str()).first;
+            MetaHeader* hdr = static_cast<MetaHeader*>(zmq_msg_data(msgPtr));
             size_t size = 0;
-        if (owner)
+            if (hdr->fHandle)
             {
-            static_cast<FairMQMessageSHM*>(msg)->fOwner = owner;
+                static_cast<FairMQMessageSHM*>(msg.get())->fHandle = hdr->fHandle;
-            static_cast<FairMQMessageSHM*>(msg)->fReceiving = true;
+                static_cast<FairMQMessageSHM*>(msg.get())->fChunkSize = hdr->fSize;
+                // static_cast<FairMQMessageSHM*>(msg.get())->fOwner = owner;
+                // static_cast<FairMQMessageSHM*>(msg.get())->fReceiving = true;
                 size = msg->GetSize();
 
                 fBytesRx += size;
@@ -262,20 +205,113 @@ int FairMQSocketSHM::Receive(FairMQMessage* msg, const int flags)
                 return -1;
             }
         }
-    if (zmq_errno() == EAGAIN)
+        else if (zmq_errno() == EAGAIN)
+        {
+            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
+            {
+                continue;
+            }
+            else
             {
                 return -2;
             }
+        }
+        else if (zmq_errno() == ETERM)
+        {
+            LOG(INFO) << "terminating socket " << fId;
+            return -1;
+        }
+        else
+        {
+            LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
+            return nbytes;
+        }
+    }
+}
+
+int64_t FairMQSocketSHM::Send(vector<FairMQMessagePtr>& msgVec, const int flags)
+{
+    const unsigned int vecSize = msgVec.size();
+
+    // Sending vector typicaly handles more then one part
+    if (vecSize > 1)
+    {
+        int64_t totalSize = 0;
+        int nbytes = -1;
+        bool repeat = false;
+
+        while (true && !fInterrupted)
+        {
+            for (unsigned int i = 0; i < vecSize; ++i)
+            {
+                nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()),
+                                      fSocket,
+                                      (i < vecSize - 1) ? ZMQ_SNDMORE|flags : flags);
+                if (nbytes >= 0)
+                {
+                    static_cast<FairMQMessageSHM*>(msgVec[i].get())->fQueued = true;
+                    // static_cast<FairMQMessageSHM*>(msgVec[i].get())->fReceiving = false;
+                    // static_cast<FairMQMessageSHM*>(msgVec[i].get())->fQueued = true;
+                    size_t size = msgVec[i]->GetSize();
+
+                    totalSize += size;
+                }
+                else
+                {
+                    // according to ZMQ docs, this can only occur for the first part
+                    if (zmq_errno() == EAGAIN)
+                    {
+                        if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
+                        {
+                            repeat = true;
+                            break;
+                        }
+                        else
+                        {
+                            return -2;
+                        }
+                    }
                     if (zmq_errno() == ETERM)
                     {
                         LOG(INFO) << "terminating socket " << fId;
                         return -1;
                     }
-    LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
+                    LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
                     return nbytes;
                 }
+            }
+
+            if (repeat)
+            {
+                continue;
+            }
+
+            // store statistics on how many messages have been sent (handle all parts as a single message)
+            ++fMessagesTx;
+            fBytesTx += totalSize;
+            return totalSize;
+        }
+
+        return -1;
+    } // If there's only one part, send it as a regular message
+    else if (vecSize == 1)
+    {
+        return Send(msgVec.back(), flags);
+    }
+    else // if the vector is empty, something might be wrong
+    {
+        LOG(WARN) << "Will not send empty vector";
+        return -1;
+    }
+}
 
 int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int flags)
+{
+    int64_t totalSize = 0;
+    int64_t more = 0;
+    bool repeat = false;
+
+    while (true)
     {
         // Warn if the vector is filled before Receive() and empty it.
         if (msgVec.size() > 0)
@@ -284,8 +320,9 @@ int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int fla
             msgVec.clear();
         }
 
-    int64_t totalSize = 0;
+        totalSize = 0;
-    int64_t more = 0;
+        more = 0;
+        repeat = false;
 
         do
         {
@@ -299,18 +336,20 @@ int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int fla
             }
             else if (nbytes > 0)
             {
-            string ownerID(static_cast<char*>(zmq_msg_data(msgPtr)), zmq_msg_size(msgPtr));
+                // string ownerID(static_cast<char*>(zmq_msg_data(msgPtr)), zmq_msg_size(msgPtr));
-            ShPtrOwner* owner = Manager::Instance().Segment()->find<ShPtrOwner>(ownerID.c_str()).first;
+                // ShPtrOwner* owner = Manager::Instance().Segment()->find<ShPtrOwner>(ownerID.c_str()).first;
+                MetaHeader* hdr = static_cast<MetaHeader*>(zmq_msg_data(msgPtr));
                 size_t size = 0;
-            if (owner)
+                if (hdr->fHandle)
                 {
-                static_cast<FairMQMessageSHM*>(part.get())->fOwner = owner;
+                    static_cast<FairMQMessageSHM*>(part.get())->fHandle = hdr->fHandle;
-                static_cast<FairMQMessageSHM*>(part.get())->fReceiving = true;
+                    static_cast<FairMQMessageSHM*>(part.get())->fChunkSize = hdr->fSize;
+                    // static_cast<FairMQMessageSHM*>(msg.get())->fOwner = owner;
+                    // static_cast<FairMQMessageSHM*>(msg.get())->fReceiving = true;
                     size = part->GetSize();
 
                     msgVec.push_back(move(part));
 
-                fBytesRx += size;
                     totalSize += size;
                 }
                 else
@@ -319,6 +358,18 @@ int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int fla
                     return -1;
                 }
             }
+            else if (zmq_errno() == EAGAIN)
+            {
+                if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
+                {
+                    repeat = true;
+                    break;
+                }
+                else
+                {
+                    return -2;
+                }
+            }
             else
             {
                 return nbytes;
@@ -329,10 +380,17 @@ int64_t FairMQSocketSHM::Receive(vector<FairMQMessagePtr>& msgVec, const int fla
         }
         while (more);
 
+        if (repeat)
+        {
+            continue;
+        }
+
         // store statistics on how many messages have been received (handle all parts as a single message)
         ++fMessagesRx;
+        fBytesRx += totalSize;
         return totalSize;
     }
+}
 
 void FairMQSocketSHM::Close()
 {
@@ -362,11 +420,13 @@ void FairMQSocketSHM::Terminate()
 void FairMQSocketSHM::Interrupt()
 {
     FairMQMessageSHM::fInterrupted = true;
+    fInterrupted = true;
 }
 
 void FairMQSocketSHM::Resume()
 {
     FairMQMessageSHM::fInterrupted = false;
+    fInterrupted = false;
 }
 
 void* FairMQSocketSHM::GetSocket() const

fairmq/shmem/FairMQSocketSHM.h

@@ -13,6 +13,7 @@
 #include <memory> // unique_ptr
 
 #include "FairMQSocket.h"
+#include "FairMQMessage.h"
 #include "FairMQContextSHM.h"
 #include "FairMQShmManager.h"
 
@@ -28,12 +29,10 @@ class FairMQSocketSHM : public FairMQSocket
     virtual bool Bind(const std::string& address);
     virtual void Connect(const std::string& address);
 
-    virtual int Send(FairMQMessage* msg, const std::string& flag = "");
+    virtual int Send(FairMQMessagePtr& msg, const int flags = 0);
-    virtual int Send(FairMQMessage* msg, const int flags = 0);
+    virtual int Receive(FairMQMessagePtr& msg, const int flags = 0);
-    virtual int64_t Send(const std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
 
-    virtual int Receive(FairMQMessage* msg, const std::string& flag = "");
+    virtual int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
-    virtual int Receive(FairMQMessage* msg, const int flags = 0);
     virtual int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
 
     virtual void* GetSocket() const;
@@ -71,6 +70,7 @@ class FairMQSocketSHM : public FairMQSocket
 
     static std::unique_ptr<FairMQContextSHM> fContext;
     static bool fContextInitialized;
+    static std::atomic<bool> fInterrupted;
 };
 
 #endif /* FAIRMQSOCKETSHM_H_ */

fairmq/shmem/FairMQTransportFactorySHM.cxx

@@ -12,11 +12,13 @@
 
 using namespace std;
 
+static FairMQ::Transport gTransportType = FairMQ::Transport::SHM;
+
 FairMQTransportFactorySHM::FairMQTransportFactorySHM()
 {
     int major, minor, patch;
     zmq_version(&major, &minor, &patch);
-    LOG(DEBUG) << "Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
+    LOG(DEBUG) << "Transport: Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
                << "boost::interprocess (" << (BOOST_VERSION / 100000) << "." << (BOOST_VERSION / 100 % 1000) << "." << (BOOST_VERSION % 100) << ")";
 }
 
@@ -54,3 +56,9 @@ FairMQPollerPtr FairMQTransportFactorySHM::CreatePoller(const FairMQSocket& cmdS
 {
     return unique_ptr<FairMQPoller>(new FairMQPollerSHM(cmdSocket, dataSocket));
 }
+
+FairMQ::Transport FairMQTransportFactorySHM::GetType() const
+{
+    return gTransportType;
+}
+

fairmq/shmem/FairMQTransportFactorySHM.h

@@ -9,6 +9,7 @@
 #define FAIRMQTRANSPORTFACTORYSHM_H_
 
 #include <vector>
+#include <string>
 
 #include "FairMQTransportFactory.h"
 #include "FairMQContextSHM.h"
@@ -31,6 +32,8 @@ class FairMQTransportFactorySHM : public FairMQTransportFactory
     virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const;
     virtual FairMQPollerPtr CreatePoller(const FairMQSocket& cmdSocket, const FairMQSocket& dataSocket) const;
 
+    virtual FairMQ::Transport GetType() const;
+
     virtual ~FairMQTransportFactorySHM() {};
 };
 

fairmq/zeromq/FairMQMessageZMQ.cxx

@@ -20,6 +20,8 @@
 
 using namespace std;
 
+static FairMQ::Transport gTransportType = FairMQ::Transport::ZMQ;
+
 string FairMQMessageZMQ::fDeviceID = string();
 
 FairMQMessageZMQ::FairMQMessageZMQ()
@@ -101,6 +103,11 @@ void FairMQMessageZMQ::SetDeviceId(const string& deviceId)
     fDeviceID = deviceId;
 }
 
+FairMQ::Transport FairMQMessageZMQ::GetType() const
+{
+    return gTransportType;
+}
+
 void FairMQMessageZMQ::Copy(const unique_ptr<FairMQMessage>& msg)
 {
     // Shares the message buffer between msg and this fMessage.

fairmq/zeromq/FairMQMessageZMQ.h

@@ -41,6 +41,8 @@ class FairMQMessageZMQ : public FairMQMessage
 
     virtual void SetDeviceId(const std::string& deviceId);
 
+    virtual FairMQ::Transport GetType() const;
+
     virtual void Copy(const std::unique_ptr<FairMQMessage>& msg);
 
     void CloseMessage();

fairmq/zeromq/FairMQPollerZMQ.cxx

@@ -51,7 +51,7 @@ FairMQPollerZMQ::FairMQPollerZMQ(const vector<FairMQChannel>& channels)
         }
         else
         {
-            LOG(ERROR) << "invalid poller configuration, exiting.";
+            LOG(ERROR) << "zeromq: invalid poller configuration, exiting.";
             exit(EXIT_FAILURE);
         }
     }
@@ -105,7 +105,7 @@ FairMQPollerZMQ::FairMQPollerZMQ(const unordered_map<string, vector<FairMQChanne
                 }
                 else
                 {
-                    LOG(ERROR) << "invalid poller configuration, exiting.";
+                    LOG(ERROR) << "zeromq: invalid poller configuration, exiting.";
                     exit(EXIT_FAILURE);
                 }
             }
@@ -113,9 +113,9 @@ FairMQPollerZMQ::FairMQPollerZMQ(const unordered_map<string, vector<FairMQChanne
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "At least one of the provided channel keys for poller initialization is invalid";
+        LOG(ERROR) << "zeromq: at least one of the provided channel keys for poller initialization is invalid";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "zeromq: out of range error: " << oor.what() << '\n';
-        throw std::out_of_range("Invalid channel during poller initialization");
+        throw std::out_of_range("invalid channel during poller initialization");
     }
 }
 
@@ -153,7 +153,7 @@ FairMQPollerZMQ::FairMQPollerZMQ(const FairMQSocket& cmdSocket, const FairMQSock
     }
     else
     {
-        LOG(ERROR) << "invalid poller configuration, exiting.";
+        LOG(ERROR) << "zeromq: invalid poller configuration, exiting.";
         exit(EXIT_FAILURE);
     }
 }
@@ -164,11 +164,12 @@ void FairMQPollerZMQ::Poll(const int timeout)
     {
         if (errno == ETERM)
         {
-            LOG(DEBUG) << "polling exited, reason: " << zmq_strerror(errno);
+            LOG(DEBUG) << "zeromq: polling exited, reason: " << zmq_strerror(errno);
         }
         else
         {
-            LOG(ERROR) << "polling failed, reason: " << zmq_strerror(errno);
+            LOG(ERROR) << "zeromq: polling failed, reason: " << zmq_strerror(errno);
+            throw std::runtime_error("zeromq: polling failed");
         }
     }
 }
@@ -206,8 +207,8 @@ bool FairMQPollerZMQ::CheckInput(const string channelKey, const int index)
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
+        LOG(ERROR) << "zeromq: invalid channel key: \"" << channelKey << "\"";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "zeromq: out of range error: " << oor.what() << '\n';
         exit(EXIT_FAILURE);
     }
 }
@@ -225,8 +226,8 @@ bool FairMQPollerZMQ::CheckOutput(const string channelKey, const int index)
     }
     catch (const std::out_of_range& oor)
     {
-        LOG(ERROR) << "Invalid channel key: \"" << channelKey << "\"";
+        LOG(ERROR) << "zeromq: invalid channel key: \"" << channelKey << "\"";
-        LOG(ERROR) << "Out of Range error: " << oor.what() << '\n';
+        LOG(ERROR) << "zeromq: out of range error: " << oor.what() << '\n';
         exit(EXIT_FAILURE);
     }
 }

fairmq/zeromq/FairMQSocketZMQ.cxx

@@ -23,7 +23,8 @@
 using namespace std;
 
 // Context to hold the ZeroMQ sockets
-std::unique_ptr<FairMQContextZMQ> FairMQSocketZMQ::fContext = std::unique_ptr<FairMQContextZMQ>(new FairMQContextZMQ(1));
+unique_ptr<FairMQContextZMQ> FairMQSocketZMQ::fContext = unique_ptr<FairMQContextZMQ>(new FairMQContextZMQ(1));
+atomic<bool> FairMQSocketZMQ::fInterrupted(false);
 
 FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
     : FairMQSocket(ZMQ_SNDMORE, ZMQ_RCVMORE, ZMQ_DONTWAIT)
@@ -56,12 +57,24 @@ FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const i
 
     // Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
     // Default value for ZeroMQ is -1, which is to wait forever.
-    int linger = 500;
+    int linger = 1000;
     if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0)
     {
         LOG(ERROR) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
     }
 
+    int sndTimeout = 700;
+    if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &sndTimeout, sizeof(sndTimeout)) != 0)
+    {
+        LOG(ERROR) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
+    }
+
+    int rcvTimeout = 700;
+    if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &rcvTimeout, sizeof(rcvTimeout)) != 0)
+    {
+        LOG(ERROR) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
+    }
+
     if (type == "sub")
     {
         if (zmq_setsockopt(fSocket, ZMQ_SUBSCRIBE, NULL, 0) != 0)
@@ -106,53 +119,121 @@ void FairMQSocketZMQ::Connect(const string& address)
     }
 }
 
-int FairMQSocketZMQ::Send(FairMQMessage* msg, const string& flag)
+int FairMQSocketZMQ::Send(FairMQMessagePtr& msg, const int flags)
 {
-    return Send(msg, GetConstant(flag));
+    int nbytes = -1;
-}
 
-int FairMQSocketZMQ::Send(FairMQMessage* msg, const int flags)
+    while (true)
     {
-    int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
+        nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
         if (nbytes >= 0)
         {
             fBytesTx += nbytes;
             ++fMessagesTx;
+
             return nbytes;
         }
-    if (zmq_errno() == EAGAIN)
+        else if (zmq_errno() == EAGAIN)
+        {
+            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
+            {
+                continue;
+            }
+            else
             {
                 return -2;
             }
-    if (zmq_errno() == ETERM)
+        }
+        else if (zmq_errno() == ETERM)
         {
             LOG(INFO) << "terminating socket " << fId;
             return -1;
         }
+        else
+        {
             LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
             return nbytes;
         }
+    }
+}
 
-int64_t FairMQSocketZMQ::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
+int FairMQSocketZMQ::Receive(FairMQMessagePtr& msg, const int flags)
 {
+    int nbytes = -1;
+    while (true)
+    {
+        nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
+        if (nbytes >= 0)
+        {
+            fBytesRx += nbytes;
+            ++fMessagesRx;
+            return nbytes;
+        }
+        else if (zmq_errno() == EAGAIN)
+        {
+            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
+            {
+                continue;
+            }
+            else
+            {
+                return -2;
+            }
+        }
+        else if (zmq_errno() == ETERM)
+        {
+            LOG(INFO) << "terminating socket " << fId;
+            return -1;
+        }
+        else
+        {
+            LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
+            return nbytes;
+        }
+    }
+}
+
+int64_t FairMQSocketZMQ::Send(vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
+{
+    const unsigned int vecSize = msgVec.size();
+
     // Sending vector typicaly handles more then one part
-    if (msgVec.size() > 1)
+    if (vecSize > 1)
     {
         int64_t totalSize = 0;
+        int nbytes = -1;
+        bool repeat = false;
 
-        for (unsigned int i = 0; i < msgVec.size() - 1; ++i)
+        while (true)
         {
-            int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()), fSocket, ZMQ_SNDMORE|flags);
+            totalSize = 0;
+            nbytes = -1;
+            repeat = false;
+
+            for (unsigned int i = 0; i < vecSize; ++i)
+            {
+                nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()),
+                                      fSocket,
+                                      (i < vecSize - 1) ? ZMQ_SNDMORE|flags : flags);
                 if (nbytes >= 0)
                 {
                     totalSize += nbytes;
                 }
                 else
                 {
+                    // according to ZMQ docs, this can only occur for the first part
                     if (zmq_errno() == EAGAIN)
+                    {
+                        if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
+                        {
+                            repeat = true;
+                            break;
+                        }
+                        else
                         {
                             return -2;
                         }
+                    }
                     if (zmq_errno() == ETERM)
                     {
                         LOG(INFO) << "terminating socket " << fId;
@@ -163,34 +244,20 @@ int64_t FairMQSocketZMQ::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, c
                 }
             }
 
-        int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec.back()->GetMessage()), fSocket, flags);
+            if (repeat)
-        if (nbytes >= 0)
             {
-            totalSize += nbytes;
+                continue;
-        }
-        else
-        {
-            if (zmq_errno() == EAGAIN)
-            {
-                return -2;
-            }
-            if (zmq_errno() == ETERM)
-            {
-                LOG(INFO) << "terminating socket " << fId;
-                return -1;
-            }
-            LOG(ERROR) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-            return nbytes;
             }
 
             // store statistics on how many messages have been sent (handle all parts as a single message)
             ++fMessagesTx;
             fBytesTx += totalSize;
             return totalSize;
+        }
     } // If there's only one part, send it as a regular message
-    else if (msgVec.size() == 1)
+    else if (vecSize == 1)
     {
-        return Send(msgVec.back().get(), flags);
+        return Send(msgVec.back(), flags);
     }
     else // if the vector is empty, something might be wrong
     {
@@ -199,34 +266,13 @@ int64_t FairMQSocketZMQ::Send(const vector<unique_ptr<FairMQMessage>>& msgVec, c
     }
 }
 
-int FairMQSocketZMQ::Receive(FairMQMessage* msg, const string& flag)
-{
-    return Receive(msg, GetConstant(flag));
-}
-
-int FairMQSocketZMQ::Receive(FairMQMessage* msg, const int flags)
-{
-    int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, flags);
-    if (nbytes >= 0)
-    {
-        fBytesRx += nbytes;
-        ++fMessagesRx;
-        return nbytes;
-    }
-    if (zmq_errno() == EAGAIN)
-    {
-        return -2;
-    }
-    if (zmq_errno() == ETERM)
-    {
-        LOG(INFO) << "terminating socket " << fId;
-        return -1;
-    }
-    LOG(ERROR) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
-    return nbytes;
-}
-
 int64_t FairMQSocketZMQ::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, const int flags)
+{
+    int64_t totalSize = 0;
+    int64_t more = 0;
+    bool repeat = false;
+
+    while (true)
     {
         // Warn if the vector is filled before Receive() and empty it.
         if (msgVec.size() > 0)
@@ -235,8 +281,9 @@ int64_t FairMQSocketZMQ::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, cons
             msgVec.clear();
         }
 
-    int64_t totalSize = 0;
+        totalSize = 0;
-    int64_t more = 0;
+        more = 0;
+        repeat = false;
 
         do
         {
@@ -248,6 +295,18 @@ int64_t FairMQSocketZMQ::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, cons
                 msgVec.push_back(move(part));
                 totalSize += nbytes;
             }
+            else if (zmq_errno() == EAGAIN)
+            {
+                if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
+                {
+                    repeat = true;
+                    break;
+                }
+                else
+                {
+                    return -2;
+                }
+            }
             else
             {
                 return nbytes;
@@ -258,11 +317,17 @@ int64_t FairMQSocketZMQ::Receive(vector<unique_ptr<FairMQMessage>>& msgVec, cons
         }
         while (more);
 
+        if (repeat)
+        {
+            continue;
+        }
+
         // store statistics on how many messages have been received (handle all parts as a single message)
         ++fMessagesRx;
         fBytesRx += totalSize;
         return totalSize;
     }
+}
 
 void FairMQSocketZMQ::Close()
 {
@@ -291,10 +356,12 @@ void FairMQSocketZMQ::Terminate()
 
 void FairMQSocketZMQ::Interrupt()
 {
+    fInterrupted = true;
 }
 
 void FairMQSocketZMQ::Resume()
 {
+    fInterrupted = false;
 }
 
 void* FairMQSocketZMQ::GetSocket() const
@@ -506,10 +573,6 @@ int FairMQSocketZMQ::GetConstant(const string& constant)
 
     if (constant == "linger")
         return ZMQ_LINGER;
-    if (constant == "no-block")
-        return ZMQ_DONTWAIT;
-    if (constant == "snd-more no-block")
-        return ZMQ_DONTWAIT|ZMQ_SNDMORE;
 
     return -1;
 }

fairmq/zeromq/FairMQSocketZMQ.h

@@ -20,6 +20,7 @@
 #include <memory> // unique_ptr
 
 #include "FairMQSocket.h"
+#include "FairMQMessage.h"
 #include "FairMQContextZMQ.h"
 
 class FairMQSocketZMQ : public FairMQSocket
@@ -34,12 +35,10 @@ class FairMQSocketZMQ : public FairMQSocket
     virtual bool Bind(const std::string& address);
     virtual void Connect(const std::string& address);
 
-    virtual int Send(FairMQMessage* msg, const std::string& flag = "");
+    virtual int Send(FairMQMessagePtr& msg, const int flags = 0);
-    virtual int Send(FairMQMessage* msg, const int flags = 0);
+    virtual int Receive(FairMQMessagePtr& msg, const int flags = 0);
-    virtual int64_t Send(const std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
 
-    virtual int Receive(FairMQMessage* msg, const std::string& flag = "");
+    virtual int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
-    virtual int Receive(FairMQMessage* msg, const int flags = 0);
     virtual int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int flags = 0);
 
     virtual void* GetSocket() const;
@@ -76,6 +75,7 @@ class FairMQSocketZMQ : public FairMQSocket
     std::atomic<unsigned long> fMessagesRx;
 
     static std::unique_ptr<FairMQContextZMQ> fContext;
+    static std::atomic<bool> fInterrupted;
 };
 
 #endif /* FAIRMQSOCKETZMQ_H_ */

fairmq/zeromq/FairMQTransportFactoryZMQ.cxx

@@ -18,11 +18,13 @@
 
 using namespace std;
 
+static FairMQ::Transport gTransportType = FairMQ::Transport::ZMQ;
+
 FairMQTransportFactoryZMQ::FairMQTransportFactoryZMQ()
 {
     int major, minor, patch;
     zmq_version(&major, &minor, &patch);
-    LOG(DEBUG) << "Using ZeroMQ library, version: " << major << "." << minor << "." << patch;
+    LOG(DEBUG) << "Transport: Using ZeroMQ library, version: " << major << "." << minor << "." << patch;
 }
 
 FairMQMessagePtr FairMQTransportFactoryZMQ::CreateMessage() const
@@ -59,3 +61,8 @@ FairMQPollerPtr FairMQTransportFactoryZMQ::CreatePoller(const FairMQSocket& cmdS
 {
     return unique_ptr<FairMQPoller>(new FairMQPollerZMQ(cmdSocket, dataSocket));
 }
+
+FairMQ::Transport FairMQTransportFactoryZMQ::GetType() const
+{
+    return gTransportType;
+}

fairmq/zeromq/FairMQTransportFactoryZMQ.h

@@ -16,6 +16,7 @@
 #define FAIRMQTRANSPORTFACTORYZMQ_H_
 
 #include <vector>
+#include <string>
 
 #include "FairMQTransportFactory.h"
 #include "FairMQContextZMQ.h"
@@ -38,6 +39,8 @@ class FairMQTransportFactoryZMQ : public FairMQTransportFactory
     virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const;
     virtual FairMQPollerPtr CreatePoller(const FairMQSocket& cmdSocket, const FairMQSocket& dataSocket) const;
 
+    virtual FairMQ::Transport GetType() const;
+
     virtual ~FairMQTransportFactoryZMQ() {};
 };