Update return types in ofi::Socket

A transer is attempted even if the transport has been interrupted
(with a timeout). When the timeout is reached, transfer methods will
return TransferResult::interrupted (-3).

CMakeLists.txt

@@ -31,28 +31,30 @@ include(CTest)
 
 
 # Build options ################################################################
-fairmq_build_option(BUILD_FAIRMQ            "Build FairMQ library and devices."
-                                             DEFAULT ON)
-fairmq_build_option(BUILD_TESTING           "Build tests."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_OFI_TRANSPORT     "Build experimental OFI transport."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_SDK_COMMANDS      "Build the FairMQ SDK commands."
-                                             DEFAULT OFF)
-fairmq_build_option(BUILD_DDS_PLUGIN        "Build DDS plugin."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
-fairmq_build_option(BUILD_PMIX_PLUGIN       "Build PMIx plugin."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
-fairmq_build_option(BUILD_EXAMPLES          "Build FairMQ examples."
-                                             DEFAULT ON  REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_SDK               "Build the FairMQ controller SDK."
-                                             DEFAULT OFF REQUIRES "BUILD_DDS_PLUGIN;BUILD_SDK_COMMANDS")
-fairmq_build_option(BUILD_DOCS              "Build FairMQ documentation."
-                                             DEFAULT OFF)
-fairmq_build_option(FAST_BUILD              "Fast production build. Not recommended for development."
-                                             DEFAULT OFF)
-fairmq_build_option(USE_EXTERNAL_GTEST      "Do not use bundled GTest. Not recommended."
-                                             DEFAULT OFF)
+fairmq_build_option(BUILD_FAIRMQ        "Build FairMQ library and devices."
+                                         DEFAULT ON)
+fairmq_build_option(BUILD_TESTING       "Build tests."
+                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_OFI_TRANSPORT "Build experimental OFI transport."
+                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_SDK_COMMANDS  "Build the FairMQ SDK commands."
+                                         DEFAULT OFF)
+fairmq_build_option(BUILD_DDS_PLUGIN    "Build DDS plugin."
+                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
+fairmq_build_option(BUILD_PMIX_PLUGIN   "Build PMIx plugin."
+                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ;BUILD_SDK_COMMANDS")
+fairmq_build_option(BUILD_EXAMPLES      "Build FairMQ examples."
+                                         DEFAULT ON  REQUIRES "BUILD_FAIRMQ")
+fairmq_build_option(BUILD_SDK           "Build the FairMQ controller SDK."
+                                         DEFAULT OFF REQUIRES "BUILD_DDS_PLUGIN;BUILD_SDK_COMMANDS")
+fairmq_build_option(BUILD_DOCS          "Build FairMQ documentation."
+                                         DEFAULT OFF)
+fairmq_build_option(FAST_BUILD          "Fast production build. Not recommended for development."
+                                         DEFAULT OFF)
+fairmq_build_option(USE_EXTERNAL_GTEST  "Do not use bundled GTest. Not recommended."
+                                         DEFAULT OFF)
+fairmq_build_option(FAIRMQ_DEBUG_MODE   "Compile in debug mode (may decrease performance)."
+                                         DEFAULT OFF)
 ################################################################################
 
 
@@ -94,7 +96,7 @@ endif()
 
 if(BUILD_FAIRMQ OR BUILD_SDK)
   find_package2(PUBLIC FairLogger REQUIRED
-    VERSION 1.2.0
+    VERSION 1.6.0
   )
 
   foreach(dep IN LISTS FairLogger_PACKAGE_DEPENDENCIES)
@@ -429,4 +431,10 @@ message(STATUS "  ${Cyan}INSTALL PREFIX${CR}     ${BGreen}${CMAKE_INSTALL_PREFIX
 message(STATUS "  ")
 message(STATUS "  ${Cyan}RUN STATIC ANALYSIS  ${static_ana_summary}")
 message(STATUS "  ")
+if(FAIRMQ_DEBUG_MODE)
+  message(STATUS "  ${Cyan}DEBUG_MODE${CR}         ${BGreen}${FAIRMQ_DEBUG_MODE}${CR} (disable with ${BMagenta}-DFAIRMQ_DEBUG_MODE=OFF${CR})")
+else()
+  message(STATUS "  ${Cyan}DEBUG_MODE${CR}         ${BRed}${FAIRMQ_DEBUG_MODE}${CR} (enable with ${BMagenta}-DFAIRMQ_DEBUG_MODE=ON${CR})")
+endif()
+message(STATUS "  ")
 ################################################################################

FairMQTest.cmake

@@ -32,6 +32,7 @@ Set(configure_options "${configure_options};-DBUILD_SDK=ON")
 Set(configure_options "${configure_options};-DBUILD_SDK_COMMANDS=ON")
 Set(configure_options "${configure_options};-DFAST_BUILD=ON")
 Set(configure_options "${configure_options};-DCOTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES=-j$ENV{number_of_processors}")
+Set(configure_options "${configure_options};-DBoost_NO_BOOST_CMAKE=ON")
 
 Set(EXTRA_FLAGS $ENV{EXTRA_FLAGS})
 If(EXTRA_FLAGS)

Jenkinsfile

@@ -8,11 +8,15 @@ def jobMatrix(String prefix, List specs, Closure callback) {
   def nodes = [:]
   for (spec in specs) {
     def label = specToLabel(spec)
+    def node_tag = label
+    if (spec.os =~ /macOS/) {
+      node_tag = spec.os
+    }
     def fairsoft = spec.fairsoft
     def os = spec.os
     def compiler = spec.compiler
     nodes["${prefix}/${label}"] = {
-      node(label) {
+      node(node_tag) {
         githubNotify(context: "${prefix}/${label}", description: 'Building ...', status: 'PENDING')
         try {
           deleteDir()
@@ -29,7 +33,7 @@ def jobMatrix(String prefix, List specs, Closure callback) {
               echo "module load compiler/gcc/9.1.0" >> Dart.cfg
             '''
           }
-          if (os =~ /MacOS/) {
+          if (os =~ /[Mm]acOS/) {
             sh "echo \"export EXTRA_FLAGS='-DCMAKE_CXX_COMPILER=clang++'\" >> Dart.cfg"
           } else {
             sh "echo \"export EXTRA_FLAGS='-DCMAKE_CXX_COMPILER=g++'\" >> Dart.cfg"
@@ -71,8 +75,7 @@ pipeline{
         script {
           def build_jobs = jobMatrix('build', [
             [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc9.1.0',        fairsoft: 'fairmq_dev'],
-            [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
-            [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
+            [os: 'macOS10.15', arch: 'x86_64', compiler: 'AppleLLVM11.0.3', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh alfa_ci Dart.cfg'
           }

cmake/FairMQLib.cmake

@@ -145,7 +145,7 @@ macro(set_fairmq_defaults)
   # Configure build types
   set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo" "Nightly" "Profile" "Experimental" "AddressSan" "ThreadSan")
   set(_warnings "-Wshadow -Wall -Wextra -Wpedantic")
-  set(CMAKE_CXX_FLAGS_DEBUG          "-g ${_warnings}")
+  set(CMAKE_CXX_FLAGS_DEBUG          "-Og -g ${_warnings}")
   set(CMAKE_CXX_FLAGS_RELEASE        "-O2 -DNDEBUG")
   set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g ${_warnings} -DNDEBUG")
   set(CMAKE_CXX_FLAGS_NIGHTLY        "-O2 -g ${_warnings}")

examples/1-1/Sampler.cxx

@@ -17,8 +17,7 @@ Sampler::Sampler()
     : fText()
     , fMaxIterations(0)
     , fNumIterations(0)
-{
-}
+{}
 
 void Sampler::InitTask()
 {
@@ -32,25 +31,22 @@ bool Sampler::ConditionalRun()
     // create a copy of the data with new(), that will be deleted after the transfer is complete
     string* text = new string(fText);
 
-    // create message object with a pointer to the data buffer,
-    // its size,
+    // create message object with a pointer to the data buffer, its size,
     // custom deletion function (called when transfer is done),
     // and pointer to the object managing the data buffer
-    FairMQMessagePtr msg(NewMessage(const_cast<char*>(text->c_str()),
-                                    text->length(),
-                                    [](void* /*data*/, void* object) { delete static_cast<string*>(object); },
-                                    text));
+    FairMQMessagePtr msg(NewMessage(
+        const_cast<char*>(text->c_str()),
+        text->length(),
+        [](void* /*data*/, void* object) { delete static_cast<string*>(object); },
+        text));
 
     LOG(info) << "Sending \"" << fText << "\"";
 
-    // in case of error or transfer interruption, return false to go to IDLE state
+    // in case of error or transfer interruption, return false to go to the Ready state
     // successfull transfer will return number of bytes transfered (can be 0 if sending an empty message).
-    if (Send(msg, "data") < 0)
-    {
+    if (Send(msg, "data") < 0) {
         return false;
-    }
-    else if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
-    {
+    } else if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
         LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
         return false;
     }
@@ -58,8 +54,6 @@ bool Sampler::ConditionalRun()
     return true;
 }
 
-Sampler::~Sampler()
-{
-}
+Sampler::~Sampler() {}
 
 } // namespace example_1_1

examples/1-1/Sink.cxx

@@ -33,23 +33,22 @@ void Sink::InitTask()
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
 }
 
-// handler is called whenever a message arrives on "data", with a reference to the message and a sub-channel index (here 0)
+// handler is called whenever a message arrives on "data", with a reference to the message and a
+// sub-channel index (here 0)
 bool Sink::HandleData(FairMQMessagePtr& msg, int /*index*/)
 {
     LOG(info) << "Received: \"" << string(static_cast<char*>(msg->GetData()), msg->GetSize()) << "\"";
 
-    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
-    {
+    if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
         LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
         return false;
     }
 
-    // return true if want to be called again (otherwise return false go to IDLE state)
+    // return true if you want the handler to be called again (otherwise return false go to the
+    // Ready state)
     return true;
 }
 
-Sink::~Sink()
-{
-}
+Sink::~Sink() {}
 
 } // namespace example_1_1

examples/dds/CMakeLists.txt

@@ -29,7 +29,7 @@ target_link_libraries(fairmq-ex-dds-sink PRIVATE ExampleDDSLib)
 add_custom_target(ExampleDDS DEPENDS fairmq-ex-dds-sampler fairmq-ex-dds-processor fairmq-ex-dds-sink)
 
 list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
-set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/sdk)
 set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-dds-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology.xml @ONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-dds-topology-infinite.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-dds-topology-infinite.xml @ONLY)

examples/n-m/CMakeLists.txt

@@ -18,7 +18,7 @@ target_link_libraries(fairmq-ex-n-m-receiver PRIVATE FairMQ)
 add_custom_target(ExampleNM DEPENDS fairmq-ex-n-m-synchronizer fairmq-ex-n-m-sender fairmq-ex-n-m-receiver)
 
 list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
-set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/sdk)
 set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-topology.xml                ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-topology.xml @ONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-n-m-pair-topology.xml           ${CMAKE_CURRENT_BINARY_DIR}/ex-n-m-pair-topology.xml @ONLY)

examples/n-m/ex-n-m-pair-topology.xml

@@ -10,7 +10,7 @@
         <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_sync</id>
+            <name access="write">fmqchan_sync</name>
         </properties>
     </decltask>
 
@@ -18,8 +18,8 @@
         <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=pair,method=connect,numSockets=${numReceivers} --dds-i data:%taskIndex%</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="read">fmqchan_sync</id>
-            <name access="read">fmqchan_data</id>
+            <name access="read">fmqchan_sync</name>
+            <name access="read">fmqchan_data</name>
         </properties>
     </decltask>
 
@@ -27,7 +27,7 @@
         <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pair,method=bind,numSockets=${numSenders}</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_data</id>
+            <name access="write">fmqchan_data</name>
         </properties>
     </decltask>
 

examples/n-m/ex-n-m-topology.xml

@@ -10,7 +10,7 @@
         <exe reachable="true">fairmq-ex-n-m-synchronizer --id sync --rate 100 --color false -P dds --channel-config name=sync,type=pub,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_sync</id>
+            <name access="write">fmqchan_sync</name>
         </properties>
     </decltask>
 
@@ -18,8 +18,8 @@
         <exe reachable="true">fairmq-ex-n-m-sender --id sender%taskIndex% --timeframe-size 100000 --num-receivers ${numReceivers} --color false -P dds --channel-config name=sync,type=sub,method=connect name=data,type=push,method=connect,numSockets=${numReceivers}</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="read">fmqchan_sync</id>
-            <name access="read">fmqchan_data</id>
+            <name access="read">fmqchan_sync</name>
+            <name access="read">fmqchan_data</name>
         </properties>
     </decltask>
 
@@ -27,7 +27,7 @@
         <exe reachable="true">fairmq-ex-n-m-receiver --id receiver%taskIndex% --num-senders ${numSenders} --color false -P dds --max-timeframes 10 --channel-config name=data,type=pull,method=bind</exe>
         <env reachable="false">fairmq-ex-n-m-env.sh</env>
         <properties>
-            <name access="write">fmqchan_data</id>
+            <name access="write">fmqchan_data</name>
         </properties>
     </decltask>
 

examples/qc/CMakeLists.txt

@@ -21,7 +21,7 @@ target_link_libraries(fairmq-ex-qc-sink PRIVATE FairMQ)
 add_custom_target(ExampleQC DEPENDS fairmq-ex-qc-sampler fairmq-ex-qc-dispatcher fairmq-ex-qc-task fairmq-ex-qc-sink)
 
 list(JOIN Boost_LIBRARY_DIRS ":" LIB_DIR)
-set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/plugins/DDS)
+set(BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/fairmq/sdk)
 set(DATA_DIR ${CMAKE_CURRENT_BINARY_DIR})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-qc-topology.xml ${CMAKE_CURRENT_BINARY_DIR}/ex-qc-topology.xml @ONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-ex-qc-env.sh ${CMAKE_CURRENT_BINARY_DIR}/fairmq-ex-qc-env.sh @ONLY)

examples/qc/fairmq-start-ex-qc.sh.in

@@ -51,7 +51,7 @@ fairmq-dds-command-ui -c k
 fairmq-dds-command-ui -c b
 fairmq-dds-command-ui -c x
 fairmq-dds-command-ui -c j
-allexceptqctasks="main/(Sampler|QCDispatcher|Sink)"
+allexceptqctasks="main/(Sampler|QCDispatcher|Sink).*"
 fairmq-dds-command-ui -c r -p $allexceptqctasks
 qctask="main/QCTask.*"
 qcdispatcher="main/QCDispatcher.*"

examples/region/Sampler.cxx

@@ -23,36 +23,41 @@ namespace example_region
 
 Sampler::Sampler()
     : fMsgSize(10000)
+    , fLinger(100)
     , fMaxIterations(0)
     , fNumIterations(0)
     , fRegion(nullptr)
     , fNumUnackedMsgs(0)
-{
-}
+{}
 
 void Sampler::InitTask()
 {
     fMsgSize = fConfig->GetProperty<int>("msg-size");
+    fLinger = fConfig->GetProperty<uint32_t>("region-linger");
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
 
     fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
-        LOG(warn) << ">>>" << info.event;
-        LOG(warn) << "id: " << info.id;
-        LOG(warn) << "ptr: " << info.ptr;
-        LOG(warn) << "size: " << info.size;
-        LOG(warn) << "flags: " << info.flags;
+        LOG(info) << "Region event: " << info.event
+                  << ", managed: " << info.managed
+                  << ", id: " << info.id
+                  << ", ptr: " << info.ptr
+                  << ", size: " << info.size
+                  << ", flags: " << info.flags;
     });
 
     fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("data",
                                                              0,
                                                              10000000,
                                                              [this](const std::vector<fair::mq::RegionBlock>& blocks) { // callback to be called when message buffers no longer needed by transport
+                                                                 lock_guard<mutex> lock(fMtx);
                                                                  fNumUnackedMsgs -= blocks.size();
+
                                                                  if (fMaxIterations > 0) {
-                                                                    LOG(debug) << "Received " << blocks.size() << " acks";
+                                                                    LOG(info) << "Received " << blocks.size() << " acks";
                                                                  }
                                                              }
                                                              ));
+    fRegion->SetLinger(fLinger);
 }
 
 bool Sampler::ConditionalRun()
@@ -69,27 +74,30 @@ bool Sampler::ConditionalRun()
     // LOG(info) << "check: " << static_cast<char*>(fRegion->GetData())[3];
     // std::this_thread::sleep_for(std::chrono::seconds(1));
 
+    lock_guard<mutex> lock(fMtx);
     if (Send(msg, "data", 0) > 0) {
-        ++fNumUnackedMsgs;
-
         if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
             LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
             return false;
         }
     }
+    ++fNumUnackedMsgs;
 
     return true;
 }
 
 void Sampler::ResetTask()
 {
-    // if not all messages acknowledged, wait for a bit. But only once, since receiver could be already dead.
-    if (fNumUnackedMsgs != 0) {
-        LOG(debug) << "waiting for all acknowledgements... (" << fNumUnackedMsgs << ")";
-        this_thread::sleep_for(chrono::milliseconds(500));
-        LOG(debug) << "done, still unacked: " << fNumUnackedMsgs;
-    }
+    // On destruction UnmanagedRegion will try to TODO
     fRegion.reset();
+    {
+        lock_guard<mutex> lock(fMtx);
+        if (fNumUnackedMsgs != 0) {
+            LOG(info) << "Done, still not acknowledged: " << fNumUnackedMsgs;
+        } else {
+            LOG(info) << "All acknowledgements received.";
+        }
+    }
     fChannels.at("data").at(0).Transport()->UnsubscribeFromRegionEvents();
 }
 

examples/region/Sampler.h

@@ -15,7 +15,8 @@
 #ifndef FAIRMQEXAMPLEREGIONSAMPLER_H
 #define FAIRMQEXAMPLEREGIONSAMPLER_H
 
-#include <atomic>
+#include <mutex>
+#include <cstdint>
 
 #include "FairMQDevice.h"
 
@@ -35,10 +36,12 @@ class Sampler : public FairMQDevice
 
   private:
     int fMsgSize;
+    uint32_t fLinger;
     uint64_t fMaxIterations;
     uint64_t fNumIterations;
     FairMQUnmanagedRegionPtr fRegion;
-    std::atomic<uint64_t> fNumUnackedMsgs;
+    std::mutex fMtx;
+    uint64_t fNumUnackedMsgs;
 };
 
 } // namespace example_region

examples/region/Sink.cxx

@@ -30,11 +30,12 @@ void Sink::InitTask()
     // Get the fMaxIterations value from the command line options (via fConfig)
     fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
     fChannels.at("data").at(0).Transport()->SubscribeToRegionEvents([](FairMQRegionInfo info) {
-        LOG(warn) << ">>>" << info.event;
-        LOG(warn) << "id: " << info.id;
-        LOG(warn) << "ptr: " << info.ptr;
-        LOG(warn) << "size: " << info.size;
-        LOG(warn) << "flags: " << info.flags;
+        LOG(info) << "Region event: " << info.event
+                  << ", managed: " << info.managed
+                  << ", id: " << info.id
+                  << ", ptr: " << info.ptr
+                  << ", size: " << info.size
+                  << ", flags: " << info.flags;
     });
 }
 

examples/region/runSampler.cxx

@@ -15,6 +15,7 @@ void addCustomOptions(bpo::options_description& options)
 {
     options.add_options()
         ("msg-size", bpo::value<int>()->default_value(1000), "Message size in bytes")
+        ("region-linger", bpo::value<uint32_t>()->default_value(100), "Linger period for regions")
         ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
 }
 

examples/region/test-ex-region.sh.in

@@ -23,6 +23,7 @@ SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --msg-size $msgSize"
+SAMPLER+=" --region-linger 500"
 SAMPLER+=" --channel-config name=data,type=push,method=bind,address=tcp://127.0.0.1:7777"
 @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER &
 SAMPLER_PID=$!

fairmq/CMakeLists.txt

@@ -170,15 +170,10 @@ if(BUILD_FAIRMQ)
     PluginManager.h
     PluginServices.h
     runFairMQDevice.h
+    shmem/Monitor.h
   )
 
   set(FAIRMQ_PRIVATE_HEADER_FILES
-    devices/FairMQBenchmarkSampler.h
-    devices/FairMQMerger.h
-    devices/FairMQMultiplier.h
-    devices/FairMQProxy.h
-    devices/FairMQSink.h
-    devices/FairMQSplitter.h
     plugins/Builtin.h
     plugins/config/Config.h
     plugins/Control.h
@@ -220,10 +215,6 @@ if(BUILD_FAIRMQ)
     FairMQPoller.cxx
     FairMQSocket.cxx
     FairMQTransportFactory.cxx
-    devices/FairMQMerger.cxx
-    devices/FairMQMultiplier.cxx
-    devices/FairMQProxy.cxx
-    devices/FairMQSplitter.cxx
     Plugin.cxx
     PluginManager.cxx
     PluginServices.cxx
@@ -234,6 +225,7 @@ if(BUILD_FAIRMQ)
     plugins/config/Config.cxx
     plugins/Control.cxx
     MemoryResources.cxx
+    shmem/Monitor.cxx
   )
 
   if(BUILD_OFI_TRANSPORT)
@@ -276,6 +268,9 @@ if(BUILD_FAIRMQ)
   # preprocessor definitions #
   ############################
   target_compile_definitions(${_target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+  if(FAIRMQ_DEBUG_MODE)
+    target_compile_definitions(${_target} PUBLIC FAIRMQ_DEBUG_MODE)
+  endif()
   if(BUILD_OFI_TRANSPORT)
     target_compile_definitions(${_target} PRIVATE BUILD_OFI_TRANSPORT)
   endif()
@@ -372,12 +367,16 @@ if(BUILD_FAIRMQ)
 
   add_executable(fairmq-shmmonitor shmem/Monitor.cxx shmem/Monitor.h shmem/runMonitor.cxx)
   target_compile_definitions(fairmq-shmmonitor PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+  if(FAIRMQ_DEBUG_MODE)
+    target_compile_definitions(fairmq-shmmonitor PUBLIC FAIRMQ_DEBUG_MODE)
+  endif()
   target_link_libraries(fairmq-shmmonitor PUBLIC
     Threads::Threads
     $<$<PLATFORM_ID:Linux>:rt>
     Boost::boost
     Boost::date_time
     Boost::program_options
+    FairLogger::FairLogger
     PicoSHA2
   )
   target_include_directories(fairmq-shmmonitor PUBLIC

fairmq/DeviceRunner.cxx

@@ -8,10 +8,14 @@
 
 #include "DeviceRunner.h"
 
-#include <exception>
-#include <fairmq/Tools.h>
+#include <fairmq/tools/Strings.h>
+#include <fairmq/tools/Version.h>
 #include <fairmq/Version.h>
 
+#include <fairlogger/Logger.h>
+
+#include <exception>
+
 using namespace std;
 using namespace fair::mq;
 

fairmq/DeviceRunner.h

@@ -13,7 +13,6 @@
 #include <fairmq/PluginManager.h>
 #include <fairmq/ProgOptions.h>
 #include <FairMQDevice.h>
-#include <FairMQLogger.h>
 
 #include <functional>
 #include <memory>

fairmq/FairMQChannel.cxx

@@ -7,11 +7,15 @@
  ********************************************************************************/
 
 #include "FairMQChannel.h"
+
 #include <fairmq/tools/Strings.h>
 #include <fairmq/Properties.h>
 
+#include <fairlogger/Logger.h>
+
 #include <boost/algorithm/string.hpp> // join/split
 
+#include <cstddef> // size_t
 #include <regex>
 #include <set>
 #include <random>
@@ -671,7 +675,6 @@ void FairMQChannel::Init()
 bool FairMQChannel::ConnectEndpoint(const string& endpoint)
 {
     lock_guard<mutex> lock(fMtx);
-
     return fSocket->Connect(endpoint);
 }
 
@@ -683,6 +686,13 @@ bool FairMQChannel::BindEndpoint(string& endpoint)
     if (fSocket->Bind(endpoint)) {
         return true;
     } else {
+        // auto-bind only implemented for TCP
+        size_t protocolPos = endpoint.find(':');
+        string protocol = endpoint.substr(0, protocolPos);
+        if (protocol != "tcp") {
+            return false;
+        }
+
         if (fAutoBind) {
             // number of attempts when choosing a random port
             int numAttempts = 0;

fairmq/FairMQChannel.h

@@ -13,7 +13,6 @@
 #include <FairMQUnmanagedRegion.h>
 #include <FairMQSocket.h>
 #include <fairmq/Transports.h>
-#include <FairMQLogger.h>
 #include <FairMQParts.h>
 #include <fairmq/Properties.h>
 #include <FairMQMessage.h>
@@ -24,7 +23,6 @@
 #include <mutex>
 #include <stdexcept>
 #include <utility> // std::move
-#include <cstddef> // size_t
 #include <cstdint> // int64_t
 
 class FairMQChannel
@@ -68,13 +66,13 @@ class FairMQChannel
     FairMQChannel(const FairMQChannel&, const std::string& name);
 
     /// Move constructor
-    FairMQChannel(FairMQChannel&&) = delete;
+    // FairMQChannel(FairMQChannel&&) = delete;
 
     /// Assignment operator
     FairMQChannel& operator=(const FairMQChannel&);
 
     /// Move assignment operator
-    FairMQChannel& operator=(FairMQChannel&&) = delete;
+    // FairMQChannel& operator=(FairMQChannel&&) = delete;
 
     /// Destructor
     virtual ~FairMQChannel()
@@ -252,7 +250,7 @@ class FairMQChannel
     /// Sends a message to the socket queue.
     /// @param msg Constant reference of unique_ptr to a FairMQMessage
     /// @param sndTimeoutInMs send timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int Send(FairMQMessagePtr& msg, int sndTimeoutInMs = -1)
     {
         CheckSendCompatibility(msg);
@@ -262,7 +260,7 @@ class FairMQChannel
     /// Receives a message from the socket queue.
     /// @param msg Constant reference of unique_ptr to a FairMQMessage
     /// @param rcvTimeoutInMs receive timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int Receive(FairMQMessagePtr& msg, int rcvTimeoutInMs = -1)
     {
         CheckReceiveCompatibility(msg);
@@ -272,7 +270,7 @@ class FairMQChannel
     /// Send a vector of messages
     /// @param msgVec message vector reference
     /// @param sndTimeoutInMs send timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Send(std::vector<FairMQMessagePtr>& msgVec, int sndTimeoutInMs = -1)
     {
         CheckSendCompatibility(msgVec);
@@ -282,7 +280,7 @@ class FairMQChannel
     /// Receive a vector of messages
     /// @param msgVec message vector reference
     /// @param rcvTimeoutInMs receive timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Receive(std::vector<FairMQMessagePtr>& msgVec, int rcvTimeoutInMs = -1)
     {
         CheckReceiveCompatibility(msgVec);
@@ -292,7 +290,7 @@ class FairMQChannel
     /// Send FairMQParts
     /// @param parts FairMQParts reference
     /// @param sndTimeoutInMs send timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Send(FairMQParts& parts, int sndTimeoutInMs = -1)
     {
         return Send(parts.fParts, sndTimeoutInMs);
@@ -301,7 +299,7 @@ class FairMQChannel
     /// Receive FairMQParts
     /// @param parts FairMQParts reference
     /// @param rcvTimeoutInMs receive timeout in ms. -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Receive(FairMQParts& parts, int rcvTimeoutInMs = -1)
     {
         return Receive(parts.fParts, rcvTimeoutInMs);

fairmq/FairMQDevice.h

@@ -129,7 +129,7 @@ class FairMQDevice
     /// @param chan channel name
     /// @param i channel index
     /// @param sndTimeoutInMs send timeout in ms, -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int Send(FairMQMessagePtr& msg, const std::string& channel, const int index = 0, int sndTimeoutInMs = -1)
     {
         return GetChannel(channel, index).Send(msg, sndTimeoutInMs);
@@ -140,7 +140,7 @@ class FairMQDevice
     /// @param chan channel name
     /// @param i channel index
     /// @param rcvTimeoutInMs receive timeout in ms, -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int Receive(FairMQMessagePtr& msg, const std::string& channel, const int index = 0, int rcvTimeoutInMs = -1)
     {
         return GetChannel(channel, index).Receive(msg, rcvTimeoutInMs);
@@ -151,7 +151,7 @@ class FairMQDevice
     /// @param chan channel name
     /// @param i channel index
     /// @param sndTimeoutInMs send timeout in ms, -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot send)
-    /// @return Number of bytes that have been queued. -2 If queueing was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been queued, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Send(FairMQParts& parts, const std::string& channel, const int index = 0, int sndTimeoutInMs = -1)
     {
         return GetChannel(channel, index).Send(parts.fParts, sndTimeoutInMs);
@@ -162,7 +162,7 @@ class FairMQDevice
     /// @param chan channel name
     /// @param i channel index
     /// @param rcvTimeoutInMs receive timeout in ms, -1 will wait forever (or until interrupt (e.g. via state change)), 0 will not wait (return immediately if cannot receive)
-    /// @return Number of bytes that have been received. -2 if reading from the queue was not possible or timed out. -1 if there was an error.
+    /// @return Number of bytes that have been received, TransferResult::timeout if timed out, TransferResult::error if there was an error, TransferResult::interrupted if interrupted (e.g. by requested state change)
     int64_t Receive(FairMQParts& parts, const std::string& channel, const int index = 0, int rcvTimeoutInMs = -1)
     {
         return GetChannel(channel, index).Receive(parts.fParts, rcvTimeoutInMs);

fairmq/FairMQMessage.h

@@ -11,6 +11,7 @@
 
 #include <cstddef> // for size_t
 #include <memory> // unique_ptr
+#include <stdexcept>
 
 #include <fairmq/Transports.h>
 

fairmq/FairMQSocket.h

@@ -9,13 +9,31 @@
 #ifndef FAIRMQSOCKET_H_
 #define FAIRMQSOCKET_H_
 
+#include "FairMQMessage.h"
+
+#include <memory>
+#include <ostream>
+#include <stdexcept>
 #include <string>
 #include <vector>
-#include <memory>
 
-#include "FairMQMessage.h"
 class FairMQTransportFactory;
 
+namespace fair
+{
+namespace mq
+{
+
+enum class TransferResult : int
+{
+    error = -1,
+    timeout = -2,
+    interrupted = -3
+};
+
+} // namespace mq
+} // namespace fair
+
 class FairMQSocket
 {
   public:
@@ -37,6 +55,10 @@ class FairMQSocket
     virtual void SetOption(const std::string& option, const void* value, size_t valueSize) = 0;
     virtual void GetOption(const std::string& option, void* value, size_t* valueSize) = 0;
 
+    /// If the backend supports it, fills the unsigned integer @a events with the ZMQ_EVENTS value
+    /// DISCLAIMER: this API is experimental and unsupported and might be dropped / refactored in
+    /// the future.
+    virtual void Events(uint32_t* events) = 0;
     virtual void SetLinger(const int value) = 0;
     virtual int GetLinger() const = 0;
     virtual void SetSndBufSize(const int value) = 0;

fairmq/FairMQTransportFactory.cxx

@@ -15,6 +15,8 @@
 #include <FairMQLogger.h>
 #include <fairmq/Tools.h>
 
+#include <fairlogger/Logger.h>
+
 #include <memory>
 #include <string>
 

fairmq/FairMQTransportFactory.h

@@ -9,7 +9,6 @@
 #ifndef FAIRMQTRANSPORTFACTORY_H_
 #define FAIRMQTRANSPORTFACTORY_H_
 
-#include <FairMQLogger.h>
 #include <FairMQMessage.h>
 #include <FairMQPoller.h>
 #include <FairMQSocket.h>

fairmq/FairMQUnmanagedRegion.h

@@ -10,6 +10,7 @@
 #define FAIRMQUNMANAGEDREGION_H_
 
 #include <cstddef> // size_t
+#include <cstdint> // uint32_t
 #include <memory> // std::unique_ptr
 #include <functional> // std::function
 #include <ostream> // std::ostream
@@ -27,21 +28,24 @@ enum class FairMQRegionEvent : int
 struct FairMQRegionInfo
 {
     FairMQRegionInfo()
-        : id(0)
+        : managed(true)
+        , id(0)
         , ptr(nullptr)
         , size(0)
         , flags(0)
         , event(FairMQRegionEvent::created)
     {}
 
-    FairMQRegionInfo(uint64_t _id, void* _ptr, size_t _size, int64_t _flags, FairMQRegionEvent _event)
-        : id(_id)
+    FairMQRegionInfo(bool _managed, uint64_t _id, void* _ptr, size_t _size, int64_t _flags, FairMQRegionEvent _event)
+        : managed(_managed)
+        , id(_id)
         , ptr(_ptr)
         , size(_size)
         , flags(_flags)
-        , event (_event)
+        , event(_event)
     {}
 
+    bool managed;  // managed/unmanaged
     uint64_t id;   // id of the region
     void* ptr;     // pointer to the start of the region
     size_t size;   // region size
@@ -71,7 +75,9 @@ class FairMQUnmanagedRegion
 
     virtual void* GetData() const = 0;
     virtual size_t GetSize() const = 0;
-    virtual uint64_t GetId() const = 0;
+    virtual uint16_t GetId() const = 0;
+    virtual void SetLinger(uint32_t linger) = 0;
+    virtual uint32_t GetLinger() const = 0;
 
     FairMQTransportFactory* GetTransport() { return fTransport; }
     void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }

fairmq/JSONParser.cxx

@@ -12,16 +12,19 @@
  * Created on May 14, 2015, 5:01 PM
  */
 
-#include <fairmq/PropertyOutput.h>
 #include "JSONParser.h"
-#include "FairMQLogger.h"
-#include <fairmq/Tools.h>
+#include "FairMQChannel.h"
+#include <fairmq/PropertyOutput.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
+
+#include <fairlogger/Logger.h>
 
 #include <boost/property_tree/json_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 #include <boost/any.hpp>
 
-#include <ios>
+#include <iomanip>
 
 using namespace std;
 using namespace fair::mq;

fairmq/JSONParser.h

@@ -15,15 +15,11 @@
 #ifndef FAIR_MQ_JSONPARSER_H
 #define FAIR_MQ_JSONPARSER_H
 
-#include <string>
-#include <vector>
-#include <unordered_map>
-#include <exception>
-
+#include <fairmq/Properties.h>
 #include <boost/property_tree/ptree_fwd.hpp>
 
-#include "FairMQChannel.h"
-#include <fairmq/Properties.h>
+#include <stdexcept>
+#include <string>
 
 namespace fair
 {

fairmq/MemoryResources.h

@@ -18,11 +18,12 @@
 #include <fairmq/FairMQMessage.h>
 class FairMQTransportFactory;
 
+#include <boost/container/container_fwd.hpp>
 #include <boost/container/flat_map.hpp>
 #include <boost/container/pmr/memory_resource.hpp>
-#include <boost/container/pmr/monotonic_buffer_resource.hpp>
-#include <boost/container/pmr/polymorphic_allocator.hpp>
+
 #include <cstring>
+#include <stdexcept>
 #include <utility>
 
 namespace fair {

fairmq/PluginServices.h

@@ -17,13 +17,14 @@
 #include <boost/optional.hpp>
 #include <boost/optional/optional_io.hpp>
 
+#include <condition_variable>
 #include <functional>
+#include <map>
+#include <mutex>
+#include <stdexcept>
 #include <string>
 #include <unordered_map>
-#include <mutex>
-#include <map>
-#include <condition_variable>
-#include <stdexcept>
+#include <vector>
 
 namespace fair
 {

fairmq/Properties.h

@@ -16,6 +16,7 @@
 #include <functional>
 #include <map>
 #include <unordered_map>
+#include <stdexcept>
 #include <string>
 #include <typeindex>
 #include <typeinfo>

fairmq/StateMachine.cxx

@@ -7,7 +7,8 @@
  ********************************************************************************/
 
 #include <fairmq/StateMachine.h>
-#include <fairmq/Tools.h>
+
+#include <fairlogger/Logger.h>
 
 // Increase maximum number of boost::msm states (default is 10)
 // This #define has to be before any msm header includes

fairmq/StateMachine.h

@@ -11,8 +11,6 @@
 
 #include <fairmq/States.h>
 
-#include <fairlogger/Logger.h>
-
 #include <string>
 #include <memory>
 #include <functional>

fairmq/States.cxx

@@ -18,9 +18,10 @@ namespace fair
 namespace mq
 {
 
-array<string, 15> stateNames =
+array<string, 16> stateNames =
 {
     {
+        "UNDEFINED",
         "OK",
         "ERROR",
         "IDLE",
@@ -41,6 +42,7 @@ array<string, 15> stateNames =
 
 unordered_map<string, State> states =
 {
+    { "UNDEFINED",           State::Undefined },
     { "OK",                  State::Ok },
     { "ERROR",               State::Error },
     { "IDLE",                State::Idle },

fairmq/States.h

@@ -20,6 +20,7 @@ namespace mq
 
 enum class State : int
 {
+    Undefined = 0,
     Ok,
     Error,
     Idle,
@@ -39,7 +40,7 @@ enum class State : int
 
 enum class Transition : int
 {
-    Auto,
+    Auto = 0,
     InitDevice,
     CompleteInit,
     Bind,

fairmq/SuboptParser.cxx

@@ -12,9 +12,12 @@
 /// @brief  Parser implementation for key-value subopt format
 
 #include <fairmq/SuboptParser.h>
+#include <fairmq/JSONParser.h>
+
+#include <fairlogger/Logger.h>
 
 #include <boost/property_tree/ptree.hpp>
-#include <cstring>
+
 #include <utility> // make_pair
 
 using boost::property_tree::ptree;

fairmq/SuboptParser.h

@@ -14,7 +14,7 @@
 #ifndef FAIR_MQ_SUBOPTPARSER_H
 #define FAIR_MQ_SUBOPTPARSER_H
 
-#include <fairmq/JSONParser.h>
+#include <fairmq/Properties.h>
 
 #include <vector>
 #include <string>

fairmq/devices/FairMQBenchmarkSampler.h

@@ -17,6 +17,7 @@
 #include <chrono>
 #include <cstddef>   // size_t
 #include <cstdint>   // uint64_t
+#include <cstring>   // memset
 #include <string>
 
 /**
@@ -28,8 +29,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
   public:
     FairMQBenchmarkSampler()
         : fMultipart(false)
+        , fMemSet(false)
         , fNumParts(1)
         , fMsgSize(10000)
+        , fMsgAlignment(0)
         , fMsgRate(0)
         , fNumIterations(0)
         , fMaxIterations(0)
@@ -39,8 +42,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
     void InitTask() override
     {
         fMultipart = fConfig->GetProperty<bool>("multipart");
+        fMemSet = fConfig->GetProperty<bool>("memset");
         fNumParts = fConfig->GetProperty<size_t>("num-parts");
         fMsgSize = fConfig->GetProperty<size_t>("msg-size");
+        fMsgAlignment = fConfig->GetProperty<size_t>("msg-alignment");
         fMsgRate = fConfig->GetProperty<float>("msg-rate");
         fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
         fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
@@ -51,8 +56,6 @@ class FairMQBenchmarkSampler : public FairMQDevice
         // store the channel reference to avoid traversing the map on every loop iteration
         FairMQChannel& dataOutChannel = fChannels.at(fOutChannelName).at(0);
 
-        FairMQMessagePtr baseMsg(dataOutChannel.NewMessage(fMsgSize));
-
         LOG(info) << "Starting the benchmark with message size of " << fMsgSize << " and " << fMaxIterations << " iterations.";
         auto tStart = std::chrono::high_resolution_clock::now();
 
@@ -63,7 +66,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
                 FairMQParts parts;
 
                 for (size_t i = 0; i < fNumParts; ++i) {
-                    parts.AddPart(dataOutChannel.NewMessage(fMsgSize));
+                    parts.AddPart(dataOutChannel.NewMessage(fMsgSize, fair::mq::Alignment{fMsgAlignment}));
+                    if (fMemSet) {
+                        std::memset(parts.At(i)->GetData(), 0, parts.At(i)->GetSize());
+                    }
                 }
 
                 if (dataOutChannel.Send(parts) >= 0) {
@@ -75,7 +81,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
                     ++fNumIterations;
                 }
             } else {
-                FairMQMessagePtr msg(dataOutChannel.NewMessage(fMsgSize));
+                FairMQMessagePtr msg(dataOutChannel.NewMessage(fMsgSize, fair::mq::Alignment{fMsgAlignment}));
+                if (fMemSet) {
+                    std::memset(msg->GetData(), 0, msg->GetSize());
+                }
 
                 if (dataOutChannel.Send(msg) >= 0) {
                     if (fMaxIterations > 0) {
@@ -101,8 +110,10 @@ class FairMQBenchmarkSampler : public FairMQDevice
 
   protected:
     bool fMultipart;
+    bool fMemSet;
     size_t fNumParts;
     size_t fMsgSize;
+    size_t fMsgAlignment;
     std::atomic<int> fMsgCounter;
     float fMsgRate;
     uint64_t fNumIterations;

fairmq/devices/FairMQMerger.cxx

@@ -1,122 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQMerger.cxx
- *
- * @since 2012-12-06
- * @author D. Klein, A. Rybalchenko
- */
-
-#include "FairMQMerger.h"
-#include "../FairMQLogger.h"
-#include "../FairMQPoller.h"
-
-using namespace std;
-
-FairMQMerger::FairMQMerger()
-    : fMultipart(true)
-    , fInChannelName("data-in")
-    , fOutChannelName("data-out")
-{
-}
-
-void FairMQMerger::RegisterChannelEndpoints()
-{
-    RegisterChannelEndpoint(fInChannelName, 1, 10000);
-    RegisterChannelEndpoint(fOutChannelName, 1, 1);
-
-    PrintRegisteredChannels();
-}
-
-FairMQMerger::~FairMQMerger()
-{
-}
-
-void FairMQMerger::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fInChannelName = fConfig->GetProperty<string>("in-channel");
-    fOutChannelName = fConfig->GetProperty<string>("out-channel");
-}
-
-void FairMQMerger::Run()
-{
-    int numInputs = fChannels.at(fInChannelName).size();
-
-    vector<FairMQChannel*> chans;
-
-    for (auto& chan : fChannels.at(fInChannelName))
-    {
-        chans.push_back(&chan);
-    }
-
-    FairMQPollerPtr poller(NewPoller(chans));
-
-    if (fMultipart)
-    {
-        while (!NewStatePending())
-        {
-            poller->Poll(100);
-
-            // Loop over the data input channels.
-            for (int i = 0; i < numInputs; ++i)
-            {
-                // Check if the channel has data ready to be received.
-                if (poller->CheckInput(i))
-                {
-                    FairMQParts payload;
-
-                    if (Receive(payload, fInChannelName, i) >= 0)
-                    {
-                        if (Send(payload, fOutChannelName) < 0)
-                        {
-                            LOG(debug) << "Transfer interrupted";
-                            break;
-                        }
-                    }
-                    else
-                    {
-                        LOG(debug) << "Transfer interrupted";
-                        break;
-                    }
-                }
-            }
-        }
-    }
-    else
-    {
-        while (!NewStatePending())
-        {
-            poller->Poll(100);
-
-            // Loop over the data input channels.
-            for (int i = 0; i < numInputs; ++i)
-            {
-                // Check if the channel has data ready to be received.
-                if (poller->CheckInput(i))
-                {
-                    FairMQMessagePtr payload(fTransportFactory->CreateMessage());
-
-                    if (Receive(payload, fInChannelName, i) >= 0)
-                    {
-                        if (Send(payload, fOutChannelName) < 0)
-                        {
-                            LOG(debug) << "Transfer interrupted";
-                            break;
-                        }
-                    }
-                    else
-                    {
-                        LOG(debug) << "Transfer interrupted";
-                        break;
-                    }
-                }
-            }
-        }
-    }
-}

fairmq/devices/FairMQMerger.h

@@ -16,23 +16,100 @@
 #define FAIRMQMERGER_H_
 
 #include "FairMQDevice.h"
+#include "../FairMQPoller.h"
+#include "../FairMQLogger.h"
 
 #include <string>
+#include <vector>
 
 class FairMQMerger : public FairMQDevice
 {
   public:
-    FairMQMerger();
-    virtual ~FairMQMerger();
+    FairMQMerger()
+        : fMultipart(true)
+        , fInChannelName("data-in")
+        , fOutChannelName("data-out")
+    {}
+    ~FairMQMerger() {}
 
   protected:
     bool fMultipart;
     std::string fInChannelName;
     std::string fOutChannelName;
 
-    virtual void RegisterChannelEndpoints() override;
-    virtual void Run() override;
-    virtual void InitTask() override;
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fInChannelName = fConfig->GetProperty<std::string>("in-channel");
+        fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
+    }
+
+    void RegisterChannelEndpoints() override
+    {
+        RegisterChannelEndpoint(fInChannelName, 1, 10000);
+        RegisterChannelEndpoint(fOutChannelName, 1, 1);
+
+        PrintRegisteredChannels();
+    }
+
+    void Run() override
+    {
+        int numInputs = fChannels.at(fInChannelName).size();
+
+        std::vector<FairMQChannel*> chans;
+
+        for (auto& chan : fChannels.at(fInChannelName)) {
+            chans.push_back(&chan);
+        }
+
+        FairMQPollerPtr poller(NewPoller(chans));
+
+        if (fMultipart) {
+            while (!NewStatePending()) {
+                poller->Poll(100);
+
+                // Loop over the data input channels.
+                for (int i = 0; i < numInputs; ++i) {
+                    // Check if the channel has data ready to be received.
+                    if (poller->CheckInput(i)) {
+                        FairMQParts payload;
+
+                        if (Receive(payload, fInChannelName, i) >= 0) {
+                            if (Send(payload, fOutChannelName) < 0) {
+                                LOG(debug) << "Transfer interrupted";
+                                break;
+                            }
+                        } else {
+                            LOG(debug) << "Transfer interrupted";
+                            break;
+                        }
+                    }
+                }
+            }
+        } else {
+            while (!NewStatePending()) {
+                poller->Poll(100);
+
+                // Loop over the data input channels.
+                for (int i = 0; i < numInputs; ++i) {
+                    // Check if the channel has data ready to be received.
+                    if (poller->CheckInput(i)) {
+                        FairMQMessagePtr payload(fTransportFactory->CreateMessage());
+
+                        if (Receive(payload, fInChannelName, i) >= 0) {
+                            if (Send(payload, fOutChannelName) < 0) {
+                                LOG(debug) << "Transfer interrupted";
+                                break;
+                            }
+                        } else {
+                            LOG(debug) << "Transfer interrupted";
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    }
 };
 
 #endif /* FAIRMQMERGER_H_ */

fairmq/devices/FairMQMultiplier.cxx

@@ -1,110 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQMultiplier.h"
-
-#include "../FairMQLogger.h"
-
-using namespace std;
-
-FairMQMultiplier::FairMQMultiplier()
-    : fMultipart(true)
-    , fNumOutputs(0)
-    , fInChannelName()
-    , fOutChannelNames()
-{
-}
-
-FairMQMultiplier::~FairMQMultiplier()
-{
-}
-
-void FairMQMultiplier::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fInChannelName = fConfig->GetProperty<string>("in-channel");
-    fOutChannelNames = fConfig->GetProperty<vector<string>>("out-channel");
-    fNumOutputs = fChannels.at(fOutChannelNames.at(0)).size();
-
-    if (fMultipart)
-    {
-        OnData(fInChannelName, &FairMQMultiplier::HandleMultipartData);
-    }
-    else
-    {
-        OnData(fInChannelName, &FairMQMultiplier::HandleSingleData);
-    }
-}
-
-bool FairMQMultiplier::HandleSingleData(std::unique_ptr<FairMQMessage>& payload, int /*index*/)
-{
-    for (unsigned int i = 0; i < fOutChannelNames.size() - 1; ++i) // all except last channel
-    {
-        for (unsigned int j = 0; j < fChannels.at(fOutChannelNames.at(i)).size(); ++j) // all subChannels in a channel
-        {
-            FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
-            msgCopy->Copy(*payload);
-
-            Send(msgCopy, fOutChannelNames.at(i), j);
-        }
-    }
-
-    unsigned int lastChannelSize = fChannels.at(fOutChannelNames.back()).size();
-
-    for (unsigned int i = 0; i < lastChannelSize - 1; ++i) // iterate over all except last subChannels of the last channel
-    {
-        FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
-        msgCopy->Copy(*payload);
-
-        Send(msgCopy, fOutChannelNames.back(), i);
-    }
-
-    Send(payload, fOutChannelNames.back(), lastChannelSize - 1); // send final message to last subChannel of last channel
-
-    return true;
-}
-
-bool FairMQMultiplier::HandleMultipartData(FairMQParts& payload, int /*index*/)
-{
-    for (unsigned int i = 0; i < fOutChannelNames.size() - 1; ++i) // all except last channel
-    {
-        for (unsigned int j = 0; j < fChannels.at(fOutChannelNames.at(i)).size(); ++j) // all subChannels in a channel
-        {
-            FairMQParts parts;
-
-            for (int k = 0; k < payload.Size(); ++k)
-            {
-                FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
-                msgCopy->Copy(payload.AtRef(k));
-                parts.AddPart(std::move(msgCopy));
-            }
-
-            Send(parts, fOutChannelNames.at(i), j);
-        }
-    }
-
-    unsigned int lastChannelSize = fChannels.at(fOutChannelNames.back()).size();
-
-    for (unsigned int i = 0; i < lastChannelSize - 1; ++i) // iterate over all except last subChannels of the last channel
-    {
-        FairMQParts parts;
-
-        for (int k = 0; k < payload.Size(); ++k)
-        {
-            FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
-            msgCopy->Copy(payload.AtRef(k));
-            parts.AddPart(std::move(msgCopy));
-        }
-
-        Send(parts, fOutChannelNames.back(), i);
-    }
-
-    Send(payload, fOutChannelNames.back(), lastChannelSize - 1); // send final message to last subChannel of last channel
-
-    return true;
-}

fairmq/devices/FairMQMultiplier.h

@@ -12,12 +12,18 @@
 #include "FairMQDevice.h"
 
 #include <string>
+#include <vector>
 
 class FairMQMultiplier : public FairMQDevice
 {
   public:
-    FairMQMultiplier();
-    virtual ~FairMQMultiplier();
+    FairMQMultiplier()
+        : fMultipart(true)
+        , fNumOutputs(0)
+        , fInChannelName()
+        , fOutChannelNames()
+    {}
+    ~FairMQMultiplier() {}
 
   protected:
     bool fMultipart;
@@ -25,10 +31,80 @@ class FairMQMultiplier : public FairMQDevice
     std::string fInChannelName;
     std::vector<std::string> fOutChannelNames;
 
-    virtual void InitTask();
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fInChannelName = fConfig->GetProperty<std::string>("in-channel");
+        fOutChannelNames = fConfig->GetProperty<std::vector<std::string>>("out-channel");
+        fNumOutputs = fChannels.at(fOutChannelNames.at(0)).size();
 
-    bool HandleSingleData(std::unique_ptr<FairMQMessage>&, int);
-    bool HandleMultipartData(FairMQParts&, int);
+        if (fMultipart) {
+            OnData(fInChannelName, &FairMQMultiplier::HandleMultipartData);
+        } else {
+            OnData(fInChannelName, &FairMQMultiplier::HandleSingleData);
+        }
+    }
+
+
+    bool HandleSingleData(std::unique_ptr<FairMQMessage>& payload, int)
+    {
+        for (unsigned int i = 0; i < fOutChannelNames.size() - 1; ++i) { // all except last channel
+            for (unsigned int j = 0; j < fChannels.at(fOutChannelNames.at(i)).size(); ++j) { // all subChannels in a channel
+                FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
+                msgCopy->Copy(*payload);
+
+                Send(msgCopy, fOutChannelNames.at(i), j);
+            }
+        }
+
+        unsigned int lastChannelSize = fChannels.at(fOutChannelNames.back()).size();
+
+        for (unsigned int i = 0; i < lastChannelSize - 1; ++i) { // iterate over all except last subChannels of the last channel
+            FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
+            msgCopy->Copy(*payload);
+
+            Send(msgCopy, fOutChannelNames.back(), i);
+        }
+
+        Send(payload, fOutChannelNames.back(), lastChannelSize - 1); // send final message to last subChannel of last channel
+
+        return true;
+    }
+
+    bool HandleMultipartData(FairMQParts& payload, int)
+    {
+        for (unsigned int i = 0; i < fOutChannelNames.size() - 1; ++i) { // all except last channel
+            for (unsigned int j = 0; j < fChannels.at(fOutChannelNames.at(i)).size(); ++j) { // all subChannels in a channel
+                FairMQParts parts;
+
+                for (int k = 0; k < payload.Size(); ++k) {
+                    FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
+                    msgCopy->Copy(payload.AtRef(k));
+                    parts.AddPart(std::move(msgCopy));
+                }
+
+                Send(parts, fOutChannelNames.at(i), j);
+            }
+        }
+
+        unsigned int lastChannelSize = fChannels.at(fOutChannelNames.back()).size();
+
+        for (unsigned int i = 0; i < lastChannelSize - 1; ++i) { // iterate over all except last subChannels of the last channel
+            FairMQParts parts;
+
+            for (int k = 0; k < payload.Size(); ++k) {
+                FairMQMessagePtr msgCopy(fTransportFactory->CreateMessage());
+                msgCopy->Copy(payload.AtRef(k));
+                parts.AddPart(std::move(msgCopy));
+            }
+
+            Send(parts, fOutChannelNames.back(), i);
+        }
+
+        Send(payload, fOutChannelNames.back(), lastChannelSize - 1); // send final message to last subChannel of last channel
+
+        return true;
+    }
 };
 
 #endif /* FAIRMQMULTIPLIER_H_ */

fairmq/devices/FairMQProxy.cxx

@@ -1,81 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQProxy.cxx
- *
- * @since 2013-10-02
- * @author A. Rybalchenko
- */
-
-#include "FairMQProxy.h"
-
-#include "../FairMQLogger.h"
-
-using namespace std;
-
-FairMQProxy::FairMQProxy()
-    : fMultipart(true)
-    , fInChannelName()
-    , fOutChannelName()
-{
-}
-
-FairMQProxy::~FairMQProxy()
-{
-}
-
-void FairMQProxy::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fInChannelName = fConfig->GetProperty<string>("in-channel");
-    fOutChannelName = fConfig->GetProperty<string>("out-channel");
-}
-
-void FairMQProxy::Run()
-{
-    if (fMultipart)
-    {
-        while (!NewStatePending())
-        {
-            FairMQParts payload;
-            if (Receive(payload, fInChannelName) >= 0)
-            {
-                if (Send(payload, fOutChannelName) < 0)
-                {
-                    LOG(debug) << "Transfer interrupted";
-                    break;
-                }
-            }
-            else
-            {
-                LOG(debug) << "Transfer interrupted";
-                break;
-            }
-        }
-    }
-    else
-    {
-        while (!NewStatePending())
-        {
-            unique_ptr<FairMQMessage> payload(fTransportFactory->CreateMessage());
-            if (Receive(payload, fInChannelName) >= 0)
-            {
-                if (Send(payload, fOutChannelName) < 0)
-                {
-                    LOG(debug) << "Transfer interrupted";
-                    break;
-                }
-            }
-            else
-            {
-                LOG(debug) << "Transfer interrupted";
-                break;
-            }
-        }
-    }
-}

fairmq/devices/FairMQProxy.h

@@ -22,16 +22,55 @@
 class FairMQProxy : public FairMQDevice
 {
   public:
-    FairMQProxy();
-    virtual ~FairMQProxy();
+    FairMQProxy()
+        : fMultipart(true)
+        , fInChannelName()
+        , fOutChannelName()
+    {}
+    ~FairMQProxy() {}
 
   protected:
     bool fMultipart;
     std::string fInChannelName;
     std::string fOutChannelName;
 
-    virtual void Run();
-    virtual void InitTask();
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fInChannelName = fConfig->GetProperty<std::string>("in-channel");
+        fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
+    }
+
+    void Run() override
+    {
+        if (fMultipart) {
+            while (!NewStatePending()) {
+                FairMQParts payload;
+                if (Receive(payload, fInChannelName) >= 0) {
+                    if (Send(payload, fOutChannelName) < 0) {
+                        LOG(debug) << "Transfer interrupted";
+                        break;
+                    }
+                } else {
+                    LOG(debug) << "Transfer interrupted";
+                    break;
+                }
+            }
+        } else {
+            while (!NewStatePending()) {
+                FairMQMessagePtr payload(fTransportFactory->CreateMessage());
+                if (Receive(payload, fInChannelName) >= 0) {
+                    if (Send(payload, fOutChannelName) < 0) {
+                        LOG(debug) << "Transfer interrupted";
+                        break;
+                    }
+                } else {
+                    LOG(debug) << "Transfer interrupted";
+                    break;
+                }
+            }
+        }
+    }
 };
 
 #endif /* FAIRMQPROXY_H_ */

fairmq/devices/FairMQSink.h

@@ -32,7 +32,7 @@ class FairMQSink : public FairMQDevice   //, public OutputPolicy
         , fInChannelName()
     {}
 
-    virtual ~FairMQSink() {}
+    ~FairMQSink() {}
 
   protected:
     bool fMultipart;
@@ -40,14 +40,14 @@ class FairMQSink : public FairMQDevice   //, public OutputPolicy
     uint64_t fNumIterations;
     std::string fInChannelName;
 
-    virtual void InitTask()
+    void InitTask() override
     {
         fMultipart = fConfig->GetProperty<bool>("multipart");
         fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
         fInChannelName = fConfig->GetProperty<std::string>("in-channel");
     }
 
-    virtual void Run()
+    void Run() override
     {
         // store the channel reference to avoid traversing the map on every loop iteration
         FairMQChannel& dataInChannel = fChannels.at(fInChannelName).at(0);

fairmq/devices/FairMQSplitter.cxx

@@ -1,74 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQSplitter.cxx
- *
- * @since 2012-12-06
- * @author D. Klein, A. Rybalchenko
- */
-
-#include "FairMQSplitter.h"
-
-#include "../FairMQLogger.h"
-
-using namespace std;
-
-FairMQSplitter::FairMQSplitter()
-    : fMultipart(true)
-    , fNumOutputs(0)
-    , fDirection(0)
-    , fInChannelName()
-    , fOutChannelName()
-{
-}
-
-FairMQSplitter::~FairMQSplitter()
-{
-}
-
-void FairMQSplitter::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fInChannelName = fConfig->GetProperty<string>("in-channel");
-    fOutChannelName = fConfig->GetProperty<string>("out-channel");
-    fNumOutputs = fChannels.at(fOutChannelName).size();
-    fDirection = 0;
-
-    if (fMultipart)
-    {
-        OnData(fInChannelName, &FairMQSplitter::HandleMultipartData);
-    }
-    else
-    {
-        OnData(fInChannelName, &FairMQSplitter::HandleSingleData);
-    }
-}
-
-bool FairMQSplitter::HandleSingleData(FairMQMessagePtr& payload, int /*index*/)
-{
-    Send(payload, fOutChannelName, fDirection);
-
-    if (++fDirection >= fNumOutputs)
-    {
-        fDirection = 0;
-    }
-
-    return true;
-}
-
-bool FairMQSplitter::HandleMultipartData(FairMQParts& payload, int /*index*/)
-{
-    Send(payload, fOutChannelName, fDirection);
-
-    if (++fDirection >= fNumOutputs)
-    {
-        fDirection = 0;
-    }
-
-    return true;
-}

fairmq/devices/FairMQSplitter.h

@@ -22,8 +22,14 @@
 class FairMQSplitter : public FairMQDevice
 {
   public:
-    FairMQSplitter();
-    virtual ~FairMQSplitter();
+    FairMQSplitter()
+        : fMultipart(true)
+        , fNumOutputs(0)
+        , fDirection(0)
+        , fInChannelName()
+        , fOutChannelName()
+    {}
+    ~FairMQSplitter() {}
 
   protected:
     bool fMultipart;
@@ -32,10 +38,32 @@ class FairMQSplitter : public FairMQDevice
     std::string fInChannelName;
     std::string fOutChannelName;
 
-    virtual void InitTask();
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fInChannelName = fConfig->GetProperty<std::string>("in-channel");
+        fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
+        fNumOutputs = fChannels.at(fOutChannelName).size();
+        fDirection = 0;
 
-    bool HandleSingleData(std::unique_ptr<FairMQMessage>&, int);
-    bool HandleMultipartData(FairMQParts&, int);
+        if (fMultipart) {
+            OnData(fInChannelName, &FairMQSplitter::HandleData<FairMQParts>);
+        } else {
+            OnData(fInChannelName, &FairMQSplitter::HandleData<FairMQMessagePtr>);
+        }
+    }
+
+    template<typename T>
+    bool HandleData(T& payload, int)
+    {
+        Send(payload, fOutChannelName, fDirection);
+
+        if (++fDirection >= fNumOutputs) {
+            fDirection = 0;
+        }
+
+        return true;
+    }
 };
 
 #endif /* FAIRMQSPLITTER_H_ */

fairmq/ofi/Socket.cxx

@@ -272,7 +272,7 @@ try {
     int size(0);
     for (auto& msg : msgVec) {
         size += msg->GetSize();
-    } 
+    }
 
     fSendPushSem.wait();
     {
@@ -284,7 +284,7 @@ try {
     return size;
 } catch (const std::exception& e) {
     LOG(error) << e.what();
-    return -1;
+    return static_cast<int64_t>(TransferResult::error);
 }
 
 auto Socket::SendQueueReader() -> void
@@ -431,7 +431,7 @@ try {
     return size;
 } catch (const std::exception& e) {
     LOG(error) << e.what();
-    return -1;
+    return static_cast<int>(TransferResult::error);
 }
 
 auto Socket::Receive(std::vector<MessagePtr>& msgVec, const int /*timeout*/) -> int64_t
@@ -449,14 +449,14 @@ try {
     int64_t size(0);
     for (auto& msg : msgVec) {
         size += msg->GetSize();
-    } 
+    }
     fBytesRx += size;
     ++fMessagesRx;
 
-    return size; 
+    return size;
 } catch (const std::exception& e) {
     LOG(error) << e.what();
-    return -1;
+    return static_cast<int64_t>(TransferResult::error);
 }
 
 auto Socket::RecvControlQueueReader() -> void

fairmq/ofi/Socket.h

@@ -45,6 +45,7 @@ class Socket final : public fair::mq::Socket
 
     auto GetId() const -> std::string override { return fId; }
 
+    auto Events(uint32_t *events) -> void override { *events = 0; }
     auto Bind(const std::string& address) -> bool override;
     auto Connect(const std::string& address) -> bool override;
 

fairmq/plugins/DDS/CMakeLists.txt

@@ -17,13 +17,7 @@ set_target_properties(${plugin} PROPERTIES
   LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/fairmq
 )
 
-set(exe fairmq-dds-command-ui)
-add_executable(${exe} ${CMAKE_CURRENT_SOURCE_DIR}/runDDSCommandUI.cxx)
-target_link_libraries(${exe} FairMQ Commands SDK StateMachine)
-target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
-
-install(TARGETS ${plugin} ${exe}
+install(TARGETS ${plugin}
   EXPORT ${PROJECT_EXPORT_SET}
   LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
 )

fairmq/plugins/DDS/DDS.cxx

@@ -344,10 +344,10 @@ auto DDS::HandleCmd(const string& id, sdk::cmd::Cmd& cmd, const string& cond, ui
         case Type::change_state: {
             Transition transition = static_cast<ChangeState&>(cmd).GetTransition();
             if (ChangeDeviceState(transition)) {
-                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Ok, transition));
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Ok, transition, GetCurrentDeviceState()));
                 fDDS.Send(outCmds.Serialize(), to_string(senderId));
             } else {
-                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Failure, transition));
+                Cmds outCmds(make<TransitionStatus>(id, fDDSTaskId, Result::Failure, transition, GetCurrentDeviceState()));
                 fDDS.Send(outCmds.Serialize(), to_string(senderId));
             }
             {

fairmq/plugins/PMIx/PMIxPlugin.cxx

@@ -148,12 +148,12 @@ auto PMIxPlugin::SubscribeForCommands() -> void
                     Transition transition = static_cast<ChangeState&>(*cmd).GetTransition();
                     if (ChangeDeviceState(transition)) {
                         fCommands.Send(
-                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Ok, transition))
+                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Ok, transition, GetCurrentDeviceState()))
                                 .Serialize(Format::JSON),
                             {sender});
                     } else {
                         fCommands.Send(
-                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Failure, transition))
+                            Cmds(make<TransitionStatus>(fDeviceId, 0, Result::Failure, transition, GetCurrentDeviceState()))
                                 .Serialize(Format::JSON),
                             {sender});
                     }

fairmq/plugins/config/Config.cxx

@@ -61,22 +61,26 @@ Plugin::ProgOptions ConfigPluginProgramOptions()
     namespace po = boost::program_options;
     auto pluginOptions = po::options_description{"FairMQ device options"};
     pluginOptions.add_options()
-        ("id",                  po::value<string        >()->default_value(""),         "Device ID.")
-        ("io-threads",          po::value<int           >()->default_value(1),          "Number of I/O threads.")
-        ("transport",           po::value<string        >()->default_value("zeromq"),   "Transport ('zeromq'/'shmem').")
-        ("network-interface",   po::value<string        >()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
-        ("init-timeout",        po::value<int           >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
-        ("max-run-time",        po::value<uint64_t      >()->default_value(0),          "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
-        ("print-channels",      po::value<bool          >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
-        ("shm-segment-size",    po::value<size_t        >()->default_value(2ULL << 30), "Shared memory: size of the shared memory segment (in bytes).")
-        ("shm-throw-bad-alloc", po::value<bool          >()->default_value(true),       "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).")
-        ("shm-monitor",         po::value<bool          >()->default_value(true),       "Shared memory: run monitor daemon.")
-        ("ofi-size-hint",       po::value<size_t        >()->default_value(0),          "EXPERIMENTAL: OFI size hint for the allocator.")
-        ("rate",                po::value<float         >()->default_value(0.),         "Rate for conditional run loop (Hz).")
-        ("session",             po::value<string        >()->default_value("default"),  "Session name.")
-        ("config-key",          po::value<string        >(),                            "Use provided value instead of device id for fetching the configuration from JSON file.")
-        ("mq-config",           po::value<string        >(),                            "JSON input as file.")
-        ("channel-config",      po::value<vector<string>>()->multitoken()->composing(), "Configuration of single or multiple channel(s) by comma separated key=value list");
+        ("id",                  po::value<string        >()->default_value(""),                "Device ID.")
+        ("io-threads",          po::value<int           >()->default_value(1),                 "Number of I/O threads.")
+        ("transport",           po::value<string        >()->default_value("zeromq"),          "Transport ('zeromq'/'shmem').")
+        ("network-interface",   po::value<string        >()->default_value("default"),         "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
+        ("init-timeout",        po::value<int           >()->default_value(120),               "Timeout for the initialization in seconds (when expecting dynamic initialization).")
+        ("max-run-time",        po::value<uint64_t      >()->default_value(0),                 "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
+        ("print-channels",      po::value<bool          >()->implicit_value(true),             "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
+        ("shm-segment-size",    po::value<size_t        >()->default_value(2ULL << 30),        "Shared memory: size of the shared memory segment (in bytes).")
+        ("shm-allocation",      po::value<string        >()->default_value("rbtree_best_fit"), "Shared memory allocation algorithm: rbtree_best_fit/simple_seq_fit.")
+        ("shm-segment-id",      po::value<uint16_t      >()->default_value(0),                 "EXPERIMENTAL: Shared memory segment id for message creation.")
+        ("shm-mlock-segment",   po::value<bool          >()->default_value(false),             "Shared memory: mlock the shared memory segment after initialization.")
+        ("shm-zero-segment",    po::value<bool          >()->default_value(false),             "Shared memory: zero the shared memory segment memory after initialization.")
+        ("shm-throw-bad-alloc", po::value<bool          >()->default_value(true),              "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).")
+        ("shm-monitor",         po::value<bool          >()->default_value(true),              "Shared memory: run monitor daemon.")
+        ("ofi-size-hint",       po::value<size_t        >()->default_value(0),                 "EXPERIMENTAL: OFI size hint for the allocator.")
+        ("rate",                po::value<float         >()->default_value(0.),                "Rate for conditional run loop (Hz).")
+        ("session",             po::value<string        >()->default_value("default"),         "Session name.")
+        ("config-key",          po::value<string        >(),                                   "Use provided value instead of device id for fetching the configuration from JSON file.")
+        ("mq-config",           po::value<string        >(),                                   "JSON input as file.")
+        ("channel-config",      po::value<vector<string>>()->multitoken()->composing(),        "Configuration of single or multiple channel(s) by comma separated key=value list");
     return pluginOptions;
 }
 

fairmq/run/runBenchmarkSampler.cxx

@@ -15,10 +15,11 @@ void addCustomOptions(bpo::options_description& options)
 {
     options.add_options()
         ("out-channel", bpo::value<std::string>()->default_value("data"), "Name of the output channel")
-        ("same-msg", bpo::value<bool>()->default_value(false), "Re-send the same message, or recreate for each iteration")
         ("multipart", bpo::value<bool>()->default_value(false), "Handle multipart payloads")
+        ("memset", bpo::value<bool>()->default_value(false), "Memset allocated buffers to 0")
         ("num-parts", bpo::value<size_t>()->default_value(1), "Number of parts to send. 1 will send single messages, not parts")
         ("msg-size", bpo::value<size_t>()->default_value(1000000), "Message size in bytes")
+        ("msg-alignment", bpo::value<size_t>()->default_value(0), "Message alignment")
         ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Number of run iterations (0 - infinite)")
         ("msg-rate", bpo::value<float>()->default_value(0), "Msg rate limit in maximum number of messages per second");
 }

fairmq/sdk/CMakeLists.txt

@@ -84,10 +84,19 @@ target_link_libraries(fairmq
   Boost::program_options
 )
 
+add_executable(fairmq-dds-command-ui ${CMAKE_CURRENT_SOURCE_DIR}/runDDSCommandUI.cxx)
+target_link_libraries(fairmq-dds-command-ui
+  FairMQ
+  Commands
+  SDK
+  StateMachine
+)
+
 install(
   TARGETS
   SDK
   fairmq
+  fairmq-dds-command-ui
 
   EXPORT ${PROJECT_EXPORT_SET}
   RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}

fairmq/sdk/Error.h

@@ -25,11 +25,6 @@ struct RuntimeError : ::std::runtime_error
     {}
 };
 
-struct MixedStateError : RuntimeError
-{
-    using RuntimeError::RuntimeError;
-};
-
 } /* namespace sdk */
 
 enum class ErrorCode

fairmq/sdk/Topology.h

@@ -70,6 +70,65 @@ const std::map<DeviceTransition, DeviceState> expectedState =
     { DeviceTransition::End,          DeviceState::Exiting }
 };
 
+// mirrors DeviceState, but adds a "Mixed" state that represents a topology where devices are currently not in the same state.
+enum class AggregatedTopologyState : int
+{
+    Undefined = static_cast<int>(fair::mq::State::Undefined),
+    Ok = static_cast<int>(fair::mq::State::Ok),
+    Error = static_cast<int>(fair::mq::State::Error),
+    Idle = static_cast<int>(fair::mq::State::Idle),
+    InitializingDevice = static_cast<int>(fair::mq::State::InitializingDevice),
+    Initialized = static_cast<int>(fair::mq::State::Initialized),
+    Binding = static_cast<int>(fair::mq::State::Binding),
+    Bound = static_cast<int>(fair::mq::State::Bound),
+    Connecting = static_cast<int>(fair::mq::State::Connecting),
+    DeviceReady = static_cast<int>(fair::mq::State::DeviceReady),
+    InitializingTask = static_cast<int>(fair::mq::State::InitializingTask),
+    Ready = static_cast<int>(fair::mq::State::Ready),
+    Running = static_cast<int>(fair::mq::State::Running),
+    ResettingTask = static_cast<int>(fair::mq::State::ResettingTask),
+    ResettingDevice = static_cast<int>(fair::mq::State::ResettingDevice),
+    Exiting = static_cast<int>(fair::mq::State::Exiting),
+    Mixed
+};
+
+inline auto operator==(DeviceState lhs, AggregatedTopologyState rhs) -> bool
+{
+    return static_cast<int>(lhs) == static_cast<int>(rhs);
+}
+
+inline auto operator==(AggregatedTopologyState lhs, DeviceState rhs) -> bool
+{
+    return static_cast<int>(lhs) == static_cast<int>(rhs);
+}
+
+inline std::ostream& operator<<(std::ostream& os, const AggregatedTopologyState& state)
+{
+    if (state == AggregatedTopologyState::Mixed) {
+        return os << "MIXED";
+    } else {
+        return os << static_cast<DeviceState>(state);
+    }
+}
+
+inline std::string GetAggregatedTopologyStateName(AggregatedTopologyState s)
+{
+    if (s == AggregatedTopologyState::Mixed) {
+        return "MIXED";
+    } else {
+        return GetStateName(static_cast<State>(s));
+    }
+}
+
+inline AggregatedTopologyState GetAggregatedTopologyState(const std::string& state)
+{
+    if (state == "MIXED") {
+        return AggregatedTopologyState::Mixed;
+    } else {
+        return static_cast<AggregatedTopologyState>(GetState(state));
+    }
+}
+
 struct DeviceStatus
 {
     bool subscribed_to_state_changes;
@@ -100,22 +159,22 @@ using TopologyStateByTask = std::unordered_map<DDSTask::Id, DeviceStatus>;
 using TopologyStateByCollection = std::unordered_map<DDSCollection::Id, std::vector<DeviceStatus>>;
 using TopologyTransition = fair::mq::Transition;
 
-inline DeviceState AggregateState(const TopologyState& topologyState)
+inline AggregatedTopologyState AggregateState(const TopologyState& topologyState)
 {
     DeviceState first = topologyState.begin()->state;
 
     if (std::all_of(topologyState.cbegin(), topologyState.cend(), [&](TopologyState::value_type i) {
             return i.state == first;
         })) {
-        return first;
+        return static_cast<AggregatedTopologyState>(first);
     }
 
-    throw MixedStateError("State is not uniform");
+    return AggregatedTopologyState::Mixed;
 }
 
 inline bool StateEqualsTo(const TopologyState& topologyState, DeviceState state)
 {
-    return AggregateState(topologyState) == state;
+    return AggregateState(topologyState) == static_cast<AggregatedTopologyState>(state);
 }
 
 inline TopologyStateByCollection GroupByCollectionId(const TopologyState& topologyState)
@@ -362,13 +421,16 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     auto HandleCmd(cmd::TransitionStatus const& cmd) -> void
     {
         if (cmd.GetResult() != cmd::Result::Ok) {
-            FAIR_LOG(error) << cmd.GetTransition() << " transition failed for " << cmd.GetDeviceId();
             DDSTask::Id taskId(cmd.GetTaskId());
             std::lock_guard<std::mutex> lk(fMtx);
             for (auto& op : fChangeStateOps) {
-                if (!op.second.IsCompleted() && op.second.ContainsTask(taskId) &&
-                    fStateData.at(fStateIndex.at(taskId)).state != op.second.GetTargetState()) {
-                    op.second.Complete(MakeErrorCode(ErrorCode::DeviceChangeStateFailed));
+                if (!op.second.IsCompleted() && op.second.ContainsTask(taskId)) {
+                    if (fStateData.at(fStateIndex.at(taskId)).state != op.second.GetTargetState()) {
+                        FAIR_LOG(error) << cmd.GetTransition() << " transition failed for " << cmd.GetDeviceId() << ", device is in " << cmd.GetCurrentState() << " state.";
+                        op.second.Complete(MakeErrorCode(ErrorCode::DeviceChangeStateFailed));
+                    } else {
+                        FAIR_LOG(debug) << cmd.GetTransition() << " transition failed for " << cmd.GetDeviceId() << ", device is already in " << cmd.GetCurrentState() << " state.";
+                    }
                 }
             }
         }
@@ -402,7 +464,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         }
     }
 
-    using Duration = std::chrono::milliseconds;
+    using Duration = std::chrono::microseconds;
     using ChangeStateCompletionSignature = void(std::error_code, TopologyState);
 
   private:
@@ -439,6 +501,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (fTasks.empty()) {
+                FAIR_LOG(warn) << "ChangeState initiated on an empty set of tasks, check the path argument.";
+            }
         }
         ChangeStateOp() = delete;
         ChangeStateOp(const ChangeStateOp&) = delete;
@@ -741,6 +806,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (fTasks.empty()) {
+                FAIR_LOG(warn) << "WaitForState initiated on an empty set of tasks, check the path argument.";
+            }
         }
         WaitForStateOp() = delete;
         WaitForStateOp(const WaitForStateOp&) = delete;
@@ -754,9 +822,8 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         {
             fCount = std::count_if(stateIndex.cbegin(), stateIndex.cend(), [=](const auto& s) {
                 if (ContainsTask(stateData.at(s.second).taskId)) {
-                    return stateData.at(s.second).state == fTargetCurrentState
-                           &&
-                           (stateData.at(s.second).lastState == fTargetLastState || fTargetLastState == DeviceState::Ok);
+                    return stateData.at(s.second).state == fTargetCurrentState &&
+                          (stateData.at(s.second).lastState == fTargetLastState || fTargetLastState == DeviceState::Undefined);
                 } else {
                     return false;
                 }
@@ -768,8 +835,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         {
             if (!fOp.IsCompleted() && ContainsTask(taskId)) {
                 if (currentState == fTargetCurrentState &&
-                    (lastState == fTargetLastState ||
-                     fTargetLastState == DeviceState::Ok)) {
+                    (lastState == fTargetLastState || fTargetLastState == DeviceState::Undefined)) {
                     ++fCount;
                 }
                 TryCompletion();
@@ -873,7 +939,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     template<typename CompletionToken>
     auto AsyncWaitForState(const DeviceState targetCurrentState, CompletionToken&& token)
     {
-        return AsyncWaitForState(DeviceState::Ok, targetCurrentState, "", Duration(0), std::move(token));
+        return AsyncWaitForState(DeviceState::Undefined, targetCurrentState, "", Duration(0), std::move(token));
     }
 
     /// @brief Wait for selected FairMQ devices to reach given last & current state in this topology
@@ -903,7 +969,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
     auto WaitForState(const DeviceState targetCurrentState, const std::string& path = "", Duration timeout = Duration(0))
         -> std::error_code
     {
-        return WaitForState(DeviceState::Ok, targetCurrentState, path, timeout);
+        return WaitForState(DeviceState::Undefined, targetCurrentState, path, timeout);
     }
 
     using GetPropertiesCompletionSignature = void(std::error_code, GetPropertiesResult);
@@ -938,6 +1004,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (expectedCount == 0) {
+                FAIR_LOG(warn) << "GetProperties initiated on an empty set of tasks, check the path argument.";
+            }
             // FAIR_LOG(debug) << "GetProperties " << fId << " with expected count of " << fExpectedCount << " started.";
         }
         GetPropertiesOp() = delete;
@@ -1093,6 +1162,9 @@ class BasicTopology : public AsioBase<Executor, Allocator>
                     }
                 });
             }
+            if (expectedCount == 0) {
+                FAIR_LOG(warn) << "SetProperties initiated on an empty set of tasks, check the path argument.";
+            }
             // FAIR_LOG(debug) << "SetProperties " << fId << " with expected count of " << fExpectedCount << " started.";
         }
         SetPropertiesOp() = delete;
@@ -1242,7 +1314,7 @@ class BasicTopology : public AsioBase<Executor, Allocator>
         int index = 0;
 
         for (const auto& task : fDDSTopo.GetTasks()) {
-            fStateData.push_back(DeviceStatus{false, DeviceState::Ok, DeviceState::Ok, task.GetId(), task.GetCollectionId()});
+            fStateData.push_back(DeviceStatus{false, DeviceState::Undefined, DeviceState::Undefined, task.GetId(), task.GetCollectionId()});
             fStateIndex.emplace(task.GetId(), index);
             index++;
         }

fairmq/sdk/commands/Commands.cxx

@@ -70,9 +70,10 @@ array<string, 17> typeNames =
     }
 };
 
-array<fair::mq::State, 15> fbStateToMQState =
+array<fair::mq::State, 16> fbStateToMQState =
 {
     {
+        fair::mq::State::Undefined,
         fair::mq::State::Ok,
         fair::mq::State::Error,
         fair::mq::State::Idle,
@@ -91,9 +92,10 @@ array<fair::mq::State, 15> fbStateToMQState =
     }
 };
 
-array<sdk::cmd::FBState, 15> mqStateToFBState =
+array<sdk::cmd::FBState, 16> mqStateToFBState =
 {
     {
+        sdk::cmd::FBState_Undefined,
         sdk::cmd::FBState_Ok,
         sdk::cmd::FBState_Error,
         sdk::cmd::FBState_Idle,
@@ -288,6 +290,7 @@ string Cmds::Serialize(const Format type) const
                 cmdBuilder->add_task_id(_cmd.GetTaskId());
                 cmdBuilder->add_result(GetFBResult(_cmd.GetResult()));
                 cmdBuilder->add_transition(GetFBTransition(_cmd.GetTransition()));
+                cmdBuilder->add_current_state(GetFBState(_cmd.GetCurrentState()));
             }
             break;
             case Type::config: {
@@ -443,7 +446,7 @@ void Cmds::Deserialize(const string& str, const Format type)
                 fCmds.emplace_back(make<CurrentState>(cmdPtr.device_id()->str(), GetMQState(cmdPtr.current_state())));
             break;
             case FBCmd_transition_status:
-                fCmds.emplace_back(make<TransitionStatus>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result()), GetMQTransition(cmdPtr.transition())));
+                fCmds.emplace_back(make<TransitionStatus>(cmdPtr.device_id()->str(), cmdPtr.task_id(), GetResult(cmdPtr.result()), GetMQTransition(cmdPtr.transition()), GetMQState(cmdPtr.current_state())));
             break;
             case FBCmd_config:
                 fCmds.emplace_back(make<Config>(cmdPtr.device_id()->str(), cmdPtr.config_string()->str()));

fairmq/sdk/commands/Commands.h

@@ -50,7 +50,7 @@ enum class Type : int
     subscription_heartbeat,        // args: { interval }
 
     current_state,                 // args: { device_id, current_state }
-    transition_status,             // args: { device_id, task_id, Result, transition }
+    transition_status,             // args: { device_id, task_id, Result, transition, current_state }
     config,                        // args: { device_id, config_string }
     state_change_subscription,     // args: { device_id, task_id, Result }
     state_change_unsubscription,   // args: { device_id, task_id, Result }
@@ -188,12 +188,13 @@ struct CurrentState : Cmd
 
 struct TransitionStatus : Cmd
 {
-    explicit TransitionStatus(const std::string& deviceId, const uint64_t taskId, const Result result, const Transition transition)
+    explicit TransitionStatus(const std::string& deviceId, const uint64_t taskId, const Result result, const Transition transition, State currentState)
         : Cmd(Type::transition_status)
         , fDeviceId(deviceId)
         , fTaskId(taskId)
         , fResult(result)
         , fTransition(transition)
+        , fCurrentState(currentState)
     {}
 
     std::string GetDeviceId() const { return fDeviceId; }
@@ -204,12 +205,15 @@ struct TransitionStatus : Cmd
     void SetResult(const Result result) { fResult = result; }
     Transition GetTransition() const { return fTransition; }
     void SetTransition(const Transition transition) { fTransition = transition; }
+    fair::mq::State GetCurrentState() const { return fCurrentState; }
+    void SetCurrentState(fair::mq::State state) { fCurrentState = state; }
 
   private:
     std::string fDeviceId;
     uint64_t fTaskId;
     Result fResult;
     Transition fTransition;
+    fair::mq::State fCurrentState;
 };
 
 struct Config : Cmd

fairmq/sdk/commands/CommandsFormat.fbs

@@ -6,6 +6,7 @@ enum FBResult:byte {
 }
 
 enum FBState:byte {
+    Undefined,
     Ok,
     Error,
     Idle,
@@ -55,7 +56,7 @@ enum FBCmd:byte {
     subscription_heartbeat,        // args: { interval }
 
     current_state,                 // args: { device_id, current_state }
-    transition_status,             // args: { device_id, task_id, Result, transition }
+    transition_status,             // args: { device_id, task_id, Result, transition, current_state }
     config,                        // args: { device_id, config_string }
     state_change_subscription,     // args: { device_id, task_id, Result }
     state_change_unsubscription,   // args: { device_id, task_id, Result }

fairmq/sdk/runDDSCommandUI.cxx

@@ -58,14 +58,17 @@ void printControlsHelp()
 
 void handleCommand(const string& command, const string& path, unsigned int timeout, Topology& topo, const string& pKey, const string& pVal)
 {
+    std::pair<std::error_code, fair::mq::sdk::TopologyState> changeStateResult;
+
     if (command == "c") {
         cout << "> checking state of the devices" << endl;
         auto const result = topo.GetCurrentState();
         for (const auto& d : result) {
             cout << d.taskId << " : " << d.state << endl;
         }
+        return;
     } else if (command == "o") {
-        cout << "> dumping config of the devices (" << path << ")" << endl;
+        cout << "> dumping config of " << (path == "" ? "all" : path) << endl;
         // TODO: extend this regex to return all properties, once command size limitation is removed.
         auto const result = topo.GetProperties("^(session|id)$", path, std::chrono::milliseconds(timeout));
         for (const auto& d : result.second.devices) {
@@ -73,52 +76,59 @@ void handleCommand(const string& command, const string& path, unsigned int timeo
                 cout << d.first << ": " << p.first << " : " << p.second << endl;
             }
         }
+        return;
     } else if (command == "p") {
         if (pKey == "" || pVal == "") {
             cout << "cannot send property with empty key and/or value! given key: '" << pKey << "', value: '" << pVal << "'." << endl;
             return;
         }
         const DeviceProperties props{{pKey, pVal}};
-        cout << "> sending property (" << path << ")" << endl;
+        cout << "> setting properties --> " << (path == "" ? "all" : path) << endl;
         topo.SetProperties(props, path);
         // give dds time to complete request
         this_thread::sleep_for(chrono::milliseconds(100));
+        return;
     } else if (command == "i") {
-        cout << "> init devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::InitDevice, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating InitDevice transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::InitDevice, path, std::chrono::milliseconds(timeout));
     } else if (command == "k") {
-        cout << "> complete init (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::CompleteInit, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating CompleteInit transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::CompleteInit, path, std::chrono::milliseconds(timeout));
     } else if (command == "b") {
-        cout << "> bind devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::Bind, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating Bind transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::Bind, path, std::chrono::milliseconds(timeout));
     } else if (command == "x") {
-        cout << "> connect devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::Connect, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating Connect transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::Connect, path, std::chrono::milliseconds(timeout));
     } else if (command == "j") {
-        cout << "> init tasks (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::InitTask, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating InitTask transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::InitTask, path, std::chrono::milliseconds(timeout));
     } else if (command == "r") {
-        cout << "> run tasks (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::Run, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating Run transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::Run, path, std::chrono::milliseconds(timeout));
     } else if (command == "s") {
-        cout << "> stop devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::Stop, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating Stop transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::Stop, path, std::chrono::milliseconds(timeout));
     } else if (command == "t") {
-        cout << "> reset tasks (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::ResetTask, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating ResetTask transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::ResetTask, path, std::chrono::milliseconds(timeout));
     } else if (command == "d") {
-        cout << "> reset devices (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::ResetDevice, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating ResetDevice transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::ResetDevice, path, std::chrono::milliseconds(timeout));
     } else if (command == "q") {
-        cout << "> end (" << path << ")" << endl;
-        topo.ChangeState(TopologyTransition::End, path, std::chrono::milliseconds(timeout));
+        cout << "> initiating End transition --> " << (path == "" ? "all" : path) << endl;
+        changeStateResult = topo.ChangeState(TopologyTransition::End, path, std::chrono::milliseconds(timeout));
     } else if (command == "h") {
         cout << "> help" << endl;
         printControlsHelp();
+        return;
     } else {
         cout << "\033[01;32mInvalid input: [" << command << "]\033[0m" << endl;
         printControlsHelp();
+        return;
+    }
+    if (changeStateResult.first != std::error_code()) {
+        cout << "ERROR: ChangeState failed for '" << path << "': " << changeStateResult.first.message() << endl;
     }
 }
 
@@ -206,6 +216,7 @@ try {
             sendCommand(command, path, timeout, topo, pKey, pVal);
         }
         size_t pos = targetState.find("->");
+        cout << "> waiting for " << (path == "" ? "all" : path) << " to reach " << targetState << endl;
         if (pos == string::npos) {
             /* auto ec =  */topo.WaitForState(GetState(targetState), path, std::chrono::milliseconds(timeout));
             // cout << "WaitForState(" << targetState << ") result: " << ec.message() << endl;

fairmq/shmem/Common.h

@@ -12,14 +12,18 @@
 
 #include <atomic>
 #include <string>
-#include <unordered_map>
+#include <functional> // std::equal_to
 
-#include <boost/interprocess/managed_shared_memory.hpp>
+#include <boost/functional/hash.hpp>
 #include <boost/interprocess/allocators/allocator.hpp>
 #include <boost/interprocess/containers/map.hpp>
 #include <boost/interprocess/containers/string.hpp>
 #include <boost/interprocess/containers/vector.hpp>
-#include <boost/functional/hash.hpp>
+#include <boost/interprocess/indexes/null_index.hpp>
+#include <boost/interprocess/managed_shared_memory.hpp>
+#include <boost/interprocess/mem_algo/simple_seq_fit.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/variant.hpp>
 
 #include <unistd.h>
 #include <sys/types.h>
@@ -31,6 +35,17 @@ namespace mq
 namespace shmem
 {
 
+struct SharedMemoryError : std::runtime_error { using std::runtime_error::runtime_error; };
+
+using SimpleSeqFitSegment = boost::interprocess::basic_managed_shared_memory<char,
+    boost::interprocess::simple_seq_fit<boost::interprocess::mutex_family>,
+    boost::interprocess::null_index>;
+    // boost::interprocess::iset_index>;
+using RBTreeBestFitSegment = boost::interprocess::basic_managed_shared_memory<char,
+    boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family>,
+    boost::interprocess::null_index>;
+    // boost::interprocess::iset_index>;
+
 using SegmentManager = boost::interprocess::managed_shared_memory::segment_manager;
 using VoidAlloc      = boost::interprocess::allocator<void, SegmentManager>;
 using CharAlloc      = boost::interprocess::allocator<char, SegmentManager>;
@@ -38,6 +53,12 @@ using Str            = boost::interprocess::basic_string<char, std::char_traits<
 using StrAlloc       = boost::interprocess::allocator<Str, SegmentManager>;
 using StrVector      = boost::interprocess::vector<Str, StrAlloc>;
 
+enum class AllocationAlgorithm : int
+{
+    rbtree_best_fit,
+    simple_seq_fit
+};
+
 struct RegionInfo
 {
     RegionInfo(const VoidAlloc& alloc)
@@ -60,8 +81,22 @@ struct RegionInfo
     bool fDestroyed;
 };
 
-using Uint64RegionInfoPairAlloc = boost::interprocess::allocator<std::pair<const uint64_t, RegionInfo>, SegmentManager>;
-using Uint64RegionInfoMap = boost::interprocess::map<uint64_t, RegionInfo, std::less<uint64_t>, Uint64RegionInfoPairAlloc>;
+using Uint16RegionInfoPairAlloc = boost::interprocess::allocator<std::pair<const uint16_t, RegionInfo>, SegmentManager>;
+using Uint16RegionInfoMap = boost::interprocess::map<uint16_t, RegionInfo, std::less<uint16_t>, Uint16RegionInfoPairAlloc>;
+using Uint16RegionInfoHashMap = boost::unordered_map<uint16_t, RegionInfo, boost::hash<uint16_t>, std::equal_to<uint16_t>, Uint16RegionInfoPairAlloc>;
+
+struct SegmentInfo
+{
+    SegmentInfo(AllocationAlgorithm aa)
+        : fAllocationAlgorithm(aa)
+    {}
+
+    AllocationAlgorithm fAllocationAlgorithm;
+};
+
+using Uint16SegmentInfoPairAlloc = boost::interprocess::allocator<std::pair<const uint16_t, SegmentInfo>, SegmentManager>;
+using Uint16SegmentInfoHashMap = boost::unordered_map<uint16_t, SegmentInfo, boost::hash<uint16_t>, std::equal_to<uint16_t>, Uint16SegmentInfoPairAlloc>;
+// using Uint16SegmentInfoMap = boost::interprocess::map<uint16_t, SegmentInfo, std::less<uint16_t>, Uint16SegmentInfoPairAlloc>;
 
 struct DeviceCounter
 {
@@ -74,21 +109,65 @@ struct DeviceCounter
 
 struct RegionCounter
 {
-    RegionCounter(uint64_t c)
+    RegionCounter(uint16_t c)
         : fCount(c)
     {}
 
-    std::atomic<uint64_t> fCount;
+    std::atomic<uint16_t> fCount;
 };
 
 struct MetaHeader
 {
     size_t fSize;
-    size_t fRegionId;
     size_t fHint;
+    uint16_t fRegionId;
+    uint16_t fSegmentId;
     boost::interprocess::managed_shared_memory::handle_t fHandle;
 };
 
+#ifdef FAIRMQ_DEBUG_MODE
+struct MsgCounter
+{
+    MsgCounter()
+        : fCount(0)
+    {}
+
+    MsgCounter(unsigned int c)
+        : fCount(c)
+    {}
+
+    std::atomic<unsigned int> fCount;
+};
+
+using Uint16MsgCounterPairAlloc = boost::interprocess::allocator<std::pair<const uint16_t, MsgCounter>, SegmentManager>;
+using Uint16MsgCounterHashMap = boost::unordered_map<uint16_t, MsgCounter, boost::hash<uint16_t>, std::equal_to<uint16_t>, Uint16MsgCounterPairAlloc>;
+
+struct MsgDebug
+{
+    MsgDebug()
+        : fPid(0)
+        , fSize(0)
+        , fCreationTime(0)
+    {}
+
+    MsgDebug(pid_t pid, size_t size, const uint64_t creationTime)
+        : fPid(pid)
+        , fSize(size)
+        , fCreationTime(creationTime)
+    {}
+
+    pid_t fPid;
+    size_t fSize;
+    uint64_t fCreationTime;
+};
+
+using SizetMsgDebugPairAlloc = boost::interprocess::allocator<std::pair<const size_t, MsgDebug>, SegmentManager>;
+// using SizetMsgDebugHashMap = boost::unordered_map<size_t, MsgDebug, boost::hash<size_t>, std::equal_to<size_t>, SizetMsgDebugPairAlloc>;
+using SizetMsgDebugMap = boost::interprocess::map<size_t, MsgDebug, std::less<size_t>, SizetMsgDebugPairAlloc>;
+using Uint16MsgDebugMapPairAlloc = boost::interprocess::allocator<std::pair<const uint16_t, SizetMsgDebugMap>, SegmentManager>;
+using Uint16MsgDebugMapHashMap = boost::unordered_map<uint16_t, SizetMsgDebugMap, boost::hash<uint16_t>, std::equal_to<uint16_t>, Uint16MsgDebugMapPairAlloc>;
+#endif
+
 struct RegionBlock
 {
     RegionBlock()
@@ -113,11 +192,109 @@ struct RegionBlock
 inline std::string buildShmIdFromSessionIdAndUserId(const std::string& sessionId)
 {
     std::string seed((std::to_string(geteuid()) + sessionId));
-    std::string shmId = picosha2::hash256_hex_string(seed);
-    shmId.resize(10, '_');
+    // generate a 8-digit hex value out of sha256 hash
+    std::vector<unsigned char> hash(4);
+    picosha2::hash256(seed.begin(), seed.end(), hash.begin(), hash.end());
+    std::string shmId = picosha2::bytes_to_hex_string(hash.begin(), hash.end());
+
     return shmId;
 }
 
+struct SegmentSize : public boost::static_visitor<size_t>
+{
+    template<typename S>
+    size_t operator()(S& s) const { return s.get_size(); }
+};
+
+struct SegmentAddress : public boost::static_visitor<void*>
+{
+    template<typename S>
+    void* operator()(S& s) const { return s.get_address(); }
+};
+
+struct SegmentMemoryZeroer : public boost::static_visitor<>
+{
+    template<typename S>
+    void operator()(S& s) const { s.zero_free_memory(); }
+};
+
+struct SegmentFreeMemory : public boost::static_visitor<size_t>
+{
+    template<typename S>
+    size_t operator()(S& s) const { return s.get_free_memory(); }
+};
+
+struct SegmentHandleFromAddress : public boost::static_visitor<boost::interprocess::managed_shared_memory::handle_t>
+{
+    SegmentHandleFromAddress(const void* _ptr) : ptr(_ptr) {}
+
+    template<typename S>
+    boost::interprocess::managed_shared_memory::handle_t operator()(S& s) const { return s.get_handle_from_address(ptr); }
+
+    const void* ptr;
+};
+
+struct SegmentAddressFromHandle : public boost::static_visitor<void*>
+{
+    SegmentAddressFromHandle(const boost::interprocess::managed_shared_memory::handle_t _handle) : handle(_handle) {}
+
+    template<typename S>
+    void* operator()(S& s) const { return s.get_address_from_handle(handle); }
+
+    const boost::interprocess::managed_shared_memory::handle_t handle;
+};
+
+struct SegmentAllocate : public boost::static_visitor<void*>
+{
+    SegmentAllocate(const size_t _size) : size(_size) {}
+
+    template<typename S>
+    void* operator()(S& s) const { return s.allocate(size); }
+
+    const size_t size;
+};
+
+struct SegmentAllocateAligned : public boost::static_visitor<void*>
+{
+    SegmentAllocateAligned(const size_t _size, const size_t _alignment) : size(_size), alignment(_alignment) {}
+
+    template<typename S>
+    void* operator()(S& s) const { return s.allocate_aligned(size, alignment); }
+
+    const size_t size;
+    const size_t alignment;
+};
+
+struct SegmentBufferShrink : public boost::static_visitor<char*>
+{
+    SegmentBufferShrink(const size_t _old_size, const size_t _new_size, char* _local_ptr)
+        : old_size(_old_size)
+        , new_size(_new_size)
+        , local_ptr(_local_ptr)
+    {}
+
+    template<typename S>
+    char* operator()(S& s) const
+    {
+        boost::interprocess::managed_shared_memory::size_type shrunk_size = new_size;
+        return s.template allocation_command<char>(boost::interprocess::shrink_in_place, old_size + 128, shrunk_size, local_ptr);
+    }
+
+    const size_t old_size;
+    const size_t new_size;
+    mutable char* local_ptr;
+};
+
+struct SegmentDeallocate : public boost::static_visitor<>
+{
+    SegmentDeallocate(void* _ptr) : ptr(_ptr) {}
+
+    template<typename S>
+    void operator()(S& s) const { return s.deallocate(ptr); }
+
+    void* ptr;
+};
+
 } // namespace shmem
 } // namespace mq
 } // namespace fair

fairmq/shmem/Manager.h

@@ -17,22 +17,28 @@
 
 #include "Common.h"
 #include "Region.h"
+#include "Monitor.h"
 
 #include <FairMQLogger.h>
 #include <FairMQMessage.h>
+#include <fairmq/ProgOptions.h>
 #include <fairmq/tools/CppSTL.h>
 #include <fairmq/tools/Strings.h>
 
-#include <boost/process.hpp>
-#include <boost/filesystem.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/interprocess/ipc/message_queue.hpp>
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/sync/named_mutex.hpp>
-#include <boost/interprocess/ipc/message_queue.hpp>
+#include <boost/process.hpp>
+#include <boost/variant.hpp>
 
 #include <cstdlib> // getenv
+#include <condition_variable>
+#include <mutex>
 #include <set>
+#include <sstream>
 #include <stdexcept>
 #include <string>
 #include <thread>
@@ -40,6 +46,8 @@
 #include <utility> // pair
 #include <vector>
 
+#include <sys/mman.h> // mlock
+
 namespace fair
 {
 namespace mq
@@ -47,49 +55,130 @@ namespace mq
 namespace shmem
 {
 
-struct SharedMemoryError : std::runtime_error { using std::runtime_error::runtime_error; };
-
 class Manager
 {
   public:
-    Manager(std::string id, std::string deviceId, size_t size, bool throwOnBadAlloc)
-        : fShmId(std::move(id))
+    Manager(std::string shmId, std::string deviceId, size_t size, const ProgOptions* config)
+        : fShmId(std::move(shmId))
+        , fSegmentId(config ? config->GetProperty<uint16_t>("shm-segment-id", 0) : 0)
         , fDeviceId(std::move(deviceId))
-        , fSegmentName("fmq_" + fShmId + "_main")
-        , fManagementSegmentName("fmq_" + fShmId + "_mng")
-        , fSegment(boost::interprocess::open_or_create, fSegmentName.c_str(), size)
-        , fManagementSegment(boost::interprocess::open_or_create, fManagementSegmentName.c_str(), 655360)
+        , fSegments()
+        , fManagementSegment(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_mng").c_str(), 6553600)
         , fShmVoidAlloc(fManagementSegment.get_segment_manager())
         , fShmMtx(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_mtx").c_str())
         , fRegionEventsCV(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_cv").c_str())
         , fRegionEventsSubscriptionActive(false)
         , fDeviceCounter(nullptr)
-        , fRegionInfos(nullptr)
+        , fShmSegments(nullptr)
+        , fShmRegions(nullptr)
         , fInterrupted(false)
         , fMsgCounter(0)
+#ifdef FAIRMQ_DEBUG_MODE
+        , fMsgDebug(nullptr)
+        , fShmMsgCounters(nullptr)
+#endif
         , fHeartbeatThread()
         , fSendHeartbeats(true)
-        , fThrowOnBadAlloc(throwOnBadAlloc)
+        , fThrowOnBadAlloc(config ? config->GetProperty<bool>("shm-throw-bad-alloc", true) : true)
     {
         using namespace boost::interprocess;
-        LOG(debug) << "created/opened shared memory segment '" << "fmq_" << fShmId << "_main" << "' of " << size << " bytes. Available are " << fSegment.get_free_memory() << " bytes.";
 
-        fRegionInfos = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(unique_instance)(fShmVoidAlloc);
-        // store info about the managed segment as region with id 0
-        fRegionInfos->emplace(0, RegionInfo("", 0, 0, fShmVoidAlloc));
-
-        boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
-
-        fDeviceCounter = fManagementSegment.find<DeviceCounter>(unique_instance).first;
-
-        if (fDeviceCounter) {
-            LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
-            (fDeviceCounter->fCount)++;
-            LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
+        bool mlockSegment = false;
+        bool zeroSegment = false;
+        bool autolaunchMonitor = false;
+        std::string allocationAlgorithm("rbtree_best_fit");
+        if (config) {
+            mlockSegment = config->GetProperty<bool>("shm-mlock-segment", mlockSegment);
+            zeroSegment = config->GetProperty<bool>("shm-zero-segment", zeroSegment);
+            autolaunchMonitor = config->GetProperty<bool>("shm-monitor", autolaunchMonitor);
+            allocationAlgorithm = config->GetProperty<std::string>("shm-allocation", allocationAlgorithm);
         } else {
-            LOG(debug) << "no device counter found, creating one and initializing with 1";
-            fDeviceCounter = fManagementSegment.construct<DeviceCounter>(unique_instance)(1);
-            LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
+            LOG(debug) << "ProgOptions not available! Using defaults.";
+        }
+
+        if (autolaunchMonitor) {
+            StartMonitor(fShmId);
+        }
+
+        {
+            std::stringstream ss;
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+
+            fShmSegments = fManagementSegment.find_or_construct<Uint16SegmentInfoHashMap>(unique_instance)(fShmVoidAlloc);
+
+            try {
+                auto it = fShmSegments->find(fSegmentId);
+                if (it == fShmSegments->end()) {
+                    // no segment with given id exists, creating
+                    if (allocationAlgorithm == "rbtree_best_fit") {
+                        fSegments.emplace(fSegmentId, RBTreeBestFitSegment(create_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId)).c_str(), size));
+                        fShmSegments->emplace(fSegmentId, AllocationAlgorithm::rbtree_best_fit);
+                    } else if (allocationAlgorithm == "simple_seq_fit") {
+                        fSegments.emplace(fSegmentId, SimpleSeqFitSegment(create_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId)).c_str(), size));
+                        fShmSegments->emplace(fSegmentId, AllocationAlgorithm::simple_seq_fit);
+                    }
+                    ss << "Created ";
+                } else {
+                    // found segment with the given id, opening
+                    if (it->second.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
+                        fSegments.emplace(fSegmentId, RBTreeBestFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId)).c_str()));
+                        if (allocationAlgorithm != "rbtree_best_fit") {
+                            LOG(warn) << "Allocation algorithm of the opened segment is rbtree_best_fit, but requested is " << allocationAlgorithm << ". Ignoring requested setting.";
+                            allocationAlgorithm = "rbtree_best_fit";
+                        }
+                    } else {
+                        fSegments.emplace(fSegmentId, SimpleSeqFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId)).c_str()));
+                        if (allocationAlgorithm != "simple_seq_fit") {
+                            LOG(warn) << "Allocation algorithm of the opened segment is simple_seq_fit, but requested is " << allocationAlgorithm << ". Ignoring requested setting.";
+                            allocationAlgorithm = "simple_seq_fit";
+                        }
+                    }
+                    ss << "Opened ";
+                }
+                ss << "shared memory segment '" << "fmq_" << fShmId << "_m_" << fSegmentId << "'."
+                << " Size: " << boost::apply_visitor(SegmentSize{}, fSegments.at(fSegmentId)) << " bytes."
+                << " Available: " << boost::apply_visitor(SegmentFreeMemory{}, fSegments.at(fSegmentId)) << " bytes."
+                << " Allocation algorithm: " << allocationAlgorithm;
+                LOG(debug) << ss.str();
+            } catch(interprocess_exception& bie) {
+                LOG(error) << "something went wrong: " << bie.what();
+            }
+        }
+
+        if (mlockSegment) {
+            LOG(debug) << "Locking the managed segment memory pages...";
+            if (mlock(boost::apply_visitor(SegmentAddress{}, fSegments.at(fSegmentId)), boost::apply_visitor(SegmentSize{}, fSegments.at(fSegmentId))) == -1) {
+                LOG(error) << "Could not lock the managed segment memory. Code: " << errno << ", reason: " << strerror(errno);
+            }
+            LOG(debug) << "Successfully locked the managed segment memory pages.";
+        }
+        if (zeroSegment) {
+            LOG(debug) << "Zeroing the managed segment free memory...";
+            boost::apply_visitor(SegmentMemoryZeroer{}, fSegments.at(fSegmentId));
+            LOG(debug) << "Successfully zeroed the managed segment free memory.";
+        }
+
+        {
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+
+            fShmRegions = fManagementSegment.find_or_construct<Uint16RegionInfoHashMap>(unique_instance)(fShmVoidAlloc);
+
+            fDeviceCounter = fManagementSegment.find<DeviceCounter>(unique_instance).first;
+
+            if (fDeviceCounter) {
+                LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
+                (fDeviceCounter->fCount)++;
+                LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
+            } else {
+                LOG(debug) << "no device counter found, creating one and initializing with 1";
+                fDeviceCounter = fManagementSegment.construct<DeviceCounter>(unique_instance)(1);
+                LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
+            }
+
+#ifdef FAIRMQ_DEBUG_MODE
+            fMsgDebug = fManagementSegment.find_or_construct<Uint16MsgDebugMapHashMap>(unique_instance)(fShmVoidAlloc);
+            fShmMsgCounters = fManagementSegment.find_or_construct<Uint16MsgCounterHashMap>(unique_instance)(fShmVoidAlloc);
+#endif
         }
 
         fHeartbeatThread = std::thread(&Manager::SendHeartbeats, this);
@@ -100,42 +189,6 @@ class Manager
     Manager(const Manager&) = delete;
     Manager operator=(const Manager&) = delete;
 
-    ~Manager()
-    {
-        using namespace boost::interprocess;
-        bool lastRemoved = false;
-
-        UnsubscribeFromRegionEvents();
-
-        fSendHeartbeats = false;
-        fHeartbeatThread.join();
-
-        try {
-            boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
-
-            (fDeviceCounter->fCount)--;
-
-            if (fDeviceCounter->fCount == 0) {
-                LOG(debug) << "last segment user, removing segment.";
-
-                RemoveSegments();
-                lastRemoved = true;
-            } else {
-                LOG(debug) << "other segment users present (" << fDeviceCounter->fCount << "), not removing it.";
-            }
-        } catch(interprocess_exception& e) {
-            LOG(error) << "error while acquiring lock in Manager destructor: " << e.what();
-        }
-
-        if (lastRemoved) {
-            named_mutex::remove(std::string("fmq_" + fShmId + "_mtx").c_str());
-            named_condition::remove(std::string("fmq_" + fShmId + "_cv").c_str());
-        }
-    }
-
-    boost::interprocess::managed_shared_memory& Segment() { return fSegment; }
-    boost::interprocess::managed_shared_memory& ManagementSegment() { return fManagementSegment; }
-
     static void StartMonitor(const std::string& id)
     {
         using namespace boost::interprocess;
@@ -187,7 +240,7 @@ class Manager
     }
     bool Interrupted() { return fInterrupted.load(); }
 
-    std::pair<boost::interprocess::mapped_region*, uint64_t> CreateRegion(const size_t size,
+    std::pair<boost::interprocess::mapped_region*, uint16_t> CreateRegion(const size_t size,
                                                                           const int64_t userFlags,
                                                                           RegionCallback callback,
                                                                           RegionBulkCallback bulkCallback,
@@ -196,10 +249,10 @@ class Manager
     {
         using namespace boost::interprocess;
         try {
-            std::pair<mapped_region*, uint64_t> result;
+            std::pair<mapped_region*, uint16_t> result;
 
             {
-                uint64_t id = 0;
+                uint16_t id = 0;
                 boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
 
                 RegionCounter* rc = fManagementSegment.find<RegionCounter>(unique_instance).first;
@@ -223,7 +276,7 @@ class Manager
                 }
 
                 // create region info
-                fRegionInfos->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
+                fShmRegions->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
 
                 auto r = fRegions.emplace(id, tools::make_unique<Region>(fShmId, id, size, false, callback, bulkCallback, path, flags));
                 // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
@@ -243,13 +296,13 @@ class Manager
         }
     }
 
-    Region* GetRegion(const uint64_t id)
+    Region* GetRegion(const uint16_t id)
     {
         boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
         return GetRegionUnsafe(id);
     }
 
-    Region* GetRegionUnsafe(const uint64_t id)
+    Region* GetRegionUnsafe(const uint16_t id)
     {
         // remote region could actually be a local one if a message originates from this device (has been sent out and returned)
         auto it = fRegions.find(id);
@@ -258,26 +311,30 @@ class Manager
         } else {
             try {
                 // get region info
-                RegionInfo regionInfo = fRegionInfos->at(id);
+                RegionInfo regionInfo = fShmRegions->at(id);
                 std::string path = regionInfo.fPath.c_str();
                 int flags = regionInfo.fFlags;
                 // LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
 
                 auto r = fRegions.emplace(id, tools::make_unique<Region>(fShmId, id, 0, true, nullptr, nullptr, path, flags));
                 return r.first->second.get();
+            } catch (std::out_of_range& oor) {
+                LOG(error) << "Could not get remote region with id '" << id << "'. Does the region creator run with the same session id?";
+                LOG(error) << oor.what();
+                return nullptr;
             } catch (boost::interprocess::interprocess_exception& e) {
-                LOG(warn) << "Could not get remote region for id: " << id;
+                LOG(warn) << "Could not get remote region for id '" << id << "'";
                 return nullptr;
             }
         }
     }
 
-    void RemoveRegion(const uint64_t id)
+    void RemoveRegion(const uint16_t id)
     {
+        fRegions.erase(id);
         {
             boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
-            fRegions.erase(id);
-            fRegionInfos->at(id).fDestroyed = true;
+            fShmRegions->at(id).fDestroyed = true;
         }
         fRegionEventsCV.notify_all();
     }
@@ -292,31 +349,31 @@ class Manager
     {
         std::vector<fair::mq::RegionInfo> result;
 
-        for (const auto& e : *fRegionInfos) {
+        for (const auto& e : *fShmRegions) {
             fair::mq::RegionInfo info;
+            info.managed = false;
             info.id = e.first;
             info.flags = e.second.fUserFlags;
             info.event = e.second.fDestroyed ? RegionEvent::destroyed : RegionEvent::created;
-            if (info.id != 0) {
-                if (!e.second.fDestroyed) {
-                    auto region = GetRegionUnsafe(info.id);
-                    info.ptr = region->fRegion.get_address();
-                    info.size = region->fRegion.get_size();
-                } else {
-                    info.ptr = nullptr;
-                    info.size = 0;
-                }
-                result.push_back(info);
+            if (!e.second.fDestroyed) {
+                auto region = GetRegionUnsafe(info.id);
+                info.ptr = region->fRegion.get_address();
+                info.size = region->fRegion.get_size();
             } else {
-                if (!e.second.fDestroyed) {
-                    info.ptr = fSegment.get_address();
-                    info.size = fSegment.get_size();
-                } else {
-                    info.ptr = nullptr;
-                    info.size = 0;
-                }
-                result.push_back(info);
+                info.ptr = nullptr;
+                info.size = 0;
             }
+            result.push_back(info);
+        }
+
+        for (const auto& e : *fShmSegments) {
+            fair::mq::RegionInfo info;
+            info.managed = true;
+            info.id = e.first;
+            info.event = RegionEvent::created;
+            info.ptr = boost::apply_visitor(SegmentAddress{}, fSegments.at(e.first));
+            info.size = boost::apply_visitor(SegmentSize{}, fSegments.at(e.first));
+            result.push_back(info);
         }
 
         return result;
@@ -378,52 +435,181 @@ class Manager
         }
     }
 
-    void IncrementMsgCounter() { ++fMsgCounter; }
-    void DecrementMsgCounter() { --fMsgCounter; }
+    void IncrementMsgCounter() { fMsgCounter.fetch_add(1, std::memory_order_relaxed); }
+    void DecrementMsgCounter() { fMsgCounter.fetch_sub(1, std::memory_order_relaxed); }
 
-    void RemoveSegments()
-    {
-        using namespace boost::interprocess;
-        if (shared_memory_object::remove(fSegmentName.c_str())) {
-            LOG(debug) << "successfully removed '" << fSegmentName << "' segment after the device has stopped.";
-        } else {
-            LOG(debug) << "did not remove " << fSegmentName << " segment after the device stopped. Already removed?";
-        }
+#ifdef FAIRMQ_DEBUG_MODE
+    void IncrementShmMsgCounter(uint16_t segmentId) { ++((*fShmMsgCounters)[segmentId].fCount); }
+    void DecrementShmMsgCounter(uint16_t segmentId) { --((*fShmMsgCounters)[segmentId].fCount); }
+#endif
 
-        if (shared_memory_object::remove(fManagementSegmentName.c_str())) {
-            LOG(debug) << "successfully removed '" << fManagementSegmentName << "' segment after the device has stopped.";
-        } else {
-            LOG(debug) << "did not remove '" << fManagementSegmentName << "' segment after the device stopped. Already removed?";
-        }
-    }
+    boost::interprocess::named_mutex& GetMtx() { return fShmMtx; }
 
     void SendHeartbeats()
     {
         std::string controlQueueName("fmq_" + fShmId + "_cq");
+        std::unique_lock<std::mutex> lock(fHeartbeatsMtx);
         while (fSendHeartbeats) {
             try {
                 boost::interprocess::message_queue mq(boost::interprocess::open_only, controlQueueName.c_str());
                 boost::posix_time::ptime sndTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(100);
                 if (mq.timed_send(fDeviceId.c_str(), fDeviceId.size(), 0, sndTill)) {
-                    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+                    fHeartbeatsCV.wait_for(lock, std::chrono::milliseconds(100), [&]() { return !fSendHeartbeats; });
                 } else {
                     LOG(debug) << "control queue timeout";
                 }
             } catch (boost::interprocess::interprocess_exception& ie) {
-                std::this_thread::sleep_for(std::chrono::milliseconds(500));
-                // LOG(warn) << "no " << controlQueueName << " found";
+                fHeartbeatsCV.wait_for(lock, std::chrono::milliseconds(500), [&]() { return !fSendHeartbeats; });
+                // LOG(debug) << "no " << controlQueueName << " found";
             }
         }
     }
 
     bool ThrowingOnBadAlloc() const { return fThrowOnBadAlloc; }
 
+    void GetSegment(uint16_t id)
+    {
+        auto it = fSegments.find(id);
+        if (it == fSegments.end()) {
+            try {
+                // get region info
+                SegmentInfo segmentInfo = fShmSegments->at(id);
+                LOG(info) << "LOCATED SEGMENT WITH ID '" << id << "'";
+
+                using namespace boost::interprocess;
+
+                if (segmentInfo.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
+                    fSegments.emplace(id, RBTreeBestFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(id)).c_str()));
+                } else {
+                    fSegments.emplace(id, SimpleSeqFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(id)).c_str()));
+                }
+            } catch (std::out_of_range& oor) {
+                LOG(error) << "Could not get segment with id '" << id << "': " << oor.what();
+            } catch (boost::interprocess::interprocess_exception& bie) {
+                LOG(error) << "Could not get segment with id '" << id << "': " << bie.what();
+            }
+        }
+    }
+
+    boost::interprocess::managed_shared_memory::handle_t GetHandleFromAddress(const void* ptr, uint16_t segmentId) const
+    {
+        return boost::apply_visitor(SegmentHandleFromAddress{ptr}, fSegments.at(segmentId));
+    }
+    void* GetAddressFromHandle(const boost::interprocess::managed_shared_memory::handle_t handle, uint16_t segmentId) const
+    {
+        return boost::apply_visitor(SegmentAddressFromHandle{handle}, fSegments.at(segmentId));
+    }
+
+    char* Allocate(const size_t size, size_t alignment = 0)
+    {
+        char* ptr = nullptr;
+        // tools::RateLimiter rateLimiter(20);
+
+        while (ptr == nullptr) {
+            try {
+                // boost::interprocess::managed_shared_memory::size_type actualSize = size;
+                // char* hint = 0; // unused for boost::interprocess::allocate_new
+                // ptr = fSegments.at(fSegmentId).allocation_command<char>(boost::interprocess::allocate_new, size, actualSize, hint);
+                size_t segmentSize = boost::apply_visitor(SegmentSize{}, fSegments.at(fSegmentId));
+                if (size > segmentSize) {
+                    throw MessageBadAlloc(tools::ToString("Requested message size (", size, ") exceeds segment size (", segmentSize, ")"));
+                }
+                if (alignment == 0) {
+                    ptr = reinterpret_cast<char*>(boost::apply_visitor(SegmentAllocate{size}, fSegments.at(fSegmentId)));
+                } else {
+                    ptr = reinterpret_cast<char*>(boost::apply_visitor(SegmentAllocateAligned{size, alignment}, fSegments.at(fSegmentId)));
+                }
+            } catch (boost::interprocess::bad_alloc& ba) {
+                // LOG(warn) << "Shared memory full...";
+                if (ThrowingOnBadAlloc()) {
+                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size, ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default", ", free memory: ", boost::apply_visitor(SegmentFreeMemory{}, fSegments.at(fSegmentId))));
+                }
+                // rateLimiter.maybe_sleep();
+                std::this_thread::sleep_for(std::chrono::milliseconds(50));
+                if (Interrupted()) {
+                    return ptr;
+                } else {
+                    continue;
+                }
+            }
+#ifdef FAIRMQ_DEBUG_MODE
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+            IncrementShmMsgCounter(fSegmentId);
+            if (fMsgDebug->count(fSegmentId) == 0) {
+                (*fMsgDebug).emplace(fSegmentId, fShmVoidAlloc);
+            }
+            (*fMsgDebug).at(fSegmentId).emplace(
+                static_cast<size_t>(GetHandleFromAddress(ptr, fSegmentId)),
+                MsgDebug(getpid(), size, std::chrono::system_clock::now().time_since_epoch().count())
+            );
+#endif
+        }
+
+        return ptr;
+    }
+
+    void Deallocate(boost::interprocess::managed_shared_memory::handle_t handle, uint16_t segmentId)
+    {
+        boost::apply_visitor(SegmentDeallocate{GetAddressFromHandle(handle, segmentId)}, fSegments.at(segmentId));
+#ifdef FAIRMQ_DEBUG_MODE
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        DecrementShmMsgCounter(segmentId);
+        try {
+            (*fMsgDebug).at(segmentId).erase(handle);
+        } catch(const std::out_of_range& oor) {
+            LOG(debug) << "could not locate debug container for " << segmentId << ": " << oor.what();
+        }
+#endif
+    }
+
+    char* ShrinkInPlace(size_t oldSize, size_t newSize, char* localPtr, uint16_t segmentId)
+    {
+        return boost::apply_visitor(SegmentBufferShrink{oldSize, newSize, localPtr}, fSegments.at(segmentId));
+    }
+
+    uint16_t GetSegmentId() const { return fSegmentId; }
+
+    ~Manager()
+    {
+        using namespace boost::interprocess;
+        bool lastRemoved = false;
+
+        UnsubscribeFromRegionEvents();
+
+        {
+            std::unique_lock<std::mutex> lock(fHeartbeatsMtx);
+            fSendHeartbeats = false;
+        }
+        fHeartbeatsCV.notify_one();
+        if (fHeartbeatThread.joinable()) {
+            fHeartbeatThread.join();
+        }
+
+        try {
+            boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
+
+            (fDeviceCounter->fCount)--;
+
+            if (fDeviceCounter->fCount == 0) {
+                LOG(debug) << "Last segment user, removing segment.";
+                lastRemoved = true;
+            } else {
+                LOG(debug) << "Other segment users present (" << fDeviceCounter->fCount << "), skipping removal.";
+            }
+        } catch (interprocess_exception& e) {
+            LOG(error) << "Manager could not acquire lock: " << e.what();
+        }
+
+        if (lastRemoved) {
+            Monitor::Cleanup(ShmId{fShmId});
+        }
+    }
+
   private:
     std::string fShmId;
+    uint16_t fSegmentId;
     std::string fDeviceId;
-    std::string fSegmentName;
-    std::string fManagementSegmentName;
-    boost::interprocess::managed_shared_memory fSegment;
+    std::unordered_map<uint16_t, boost::variant<RBTreeBestFitSegment, SimpleSeqFitSegment>> fSegments;
     boost::interprocess::managed_shared_memory fManagementSegment;
     VoidAlloc fShmVoidAlloc;
     boost::interprocess::named_mutex fShmMtx;
@@ -432,17 +618,25 @@ class Manager
     std::thread fRegionEventThread;
     bool fRegionEventsSubscriptionActive;
     std::function<void(fair::mq::RegionInfo)> fRegionEventCallback;
-    std::unordered_map<uint64_t, RegionEvent> fObservedRegionEvents;
+    std::unordered_map<uint16_t, RegionEvent> fObservedRegionEvents;
 
     DeviceCounter* fDeviceCounter;
-    Uint64RegionInfoMap* fRegionInfos;
-    std::unordered_map<uint64_t, std::unique_ptr<Region>> fRegions;
+    Uint16SegmentInfoHashMap* fShmSegments;
+    Uint16RegionInfoHashMap* fShmRegions;
+    std::unordered_map<uint16_t, std::unique_ptr<Region>> fRegions;
 
     std::atomic<bool> fInterrupted;
     std::atomic<int32_t> fMsgCounter; // TODO: find a better lifetime solution instead of the counter
+#ifdef FAIRMQ_DEBUG_MODE
+    Uint16MsgDebugMapHashMap* fMsgDebug;
+    Uint16MsgCounterHashMap* fShmMsgCounters;
+#endif
 
     std::thread fHeartbeatThread;
-    std::atomic<bool> fSendHeartbeats;
+    bool fSendHeartbeats;
+    std::mutex fHeartbeatsMtx;
+    std::condition_variable fHeartbeatsCV;
+
     bool fThrowOnBadAlloc;
 };
 

fairmq/shmem/Message.h

@@ -22,6 +22,9 @@
 #include <cstddef> // size_t
 #include <atomic>
 
+#include <sys/types.h> // getpid
+#include <unistd.h> // pid_t
+
 namespace fair
 {
 namespace mq
@@ -40,7 +43,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -51,7 +54,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -62,7 +65,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -74,7 +77,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -86,7 +89,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{0, 0, 0, -1}
+        , fMeta{0, 0, 0, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(nullptr)
     {
@@ -105,7 +108,7 @@ class Message final : public fair::mq::Message
         : fair::mq::Message(factory)
         , fManager(manager)
         , fQueued(false)
-        , fMeta{size, static_cast<UnmanagedRegion*>(region.get())->fRegionId, reinterpret_cast<size_t>(hint), -1}
+        , fMeta{size, reinterpret_cast<size_t>(hint), static_cast<UnmanagedRegion*>(region.get())->fRegionId, fManager.GetSegmentId(), -1}
         , fRegionPtr(nullptr)
         , fLocalPtr(static_cast<char*>(data))
     {
@@ -166,7 +169,8 @@ class Message final : public fair::mq::Message
         if (!fLocalPtr) {
             if (fMeta.fRegionId == 0) {
                 if (fMeta.fSize > 0) {
-                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+                    fManager.GetSegment(fMeta.fSegmentId);
+                    fLocalPtr = reinterpret_cast<char*>(fManager.GetAddressFromHandle(fMeta.fHandle, fMeta.fSegmentId));
                 } else {
                     fLocalPtr = nullptr;
                 }
@@ -186,15 +190,14 @@ class Message final : public fair::mq::Message
 
     size_t GetSize() const override { return fMeta.fSize; }
 
-    bool SetUsedSize(const size_t size) override
+    bool SetUsedSize(const size_t newSize) override
     {
-        if (size == fMeta.fSize) {
+        if (newSize == fMeta.fSize) {
             return true;
-        } else if (size <= fMeta.fSize) {
+        } else if (newSize <= fMeta.fSize) {
             try {
-                boost::interprocess::managed_shared_memory::size_type shrunkSize = size;
-                fLocalPtr = fManager.Segment().allocation_command<char>(boost::interprocess::shrink_in_place, fMeta.fSize + 128, shrunkSize, fLocalPtr);
-                fMeta.fSize = size;
+                fLocalPtr = fManager.ShrinkInPlace(fMeta.fSize, newSize, fLocalPtr, fMeta.fSegmentId);
+                fMeta.fSize = newSize;
                 return true;
             } catch (boost::interprocess::interprocess_exception& e) {
                 LOG(info) << "could not set used size: " << e.what();
@@ -242,44 +245,22 @@ class Message final : public fair::mq::Message
     mutable Region* fRegionPtr;
     mutable char* fLocalPtr;
 
-    bool InitializeChunk(const size_t size, size_t alignment = 0)
+    char* InitializeChunk(const size_t size, size_t alignment = 0)
     {
-        tools::RateLimiter rateLimiter(20);
-
-        while (fMeta.fHandle < 0) {
-            try {
-                // boost::interprocess::managed_shared_memory::size_type actualSize = size;
-                // char* hint = 0; // unused for boost::interprocess::allocate_new
-                // fLocalPtr = fManager.Segment().allocation_command<char>(boost::interprocess::allocate_new, size, actualSize, hint);
-                if (alignment == 0) {
-                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().allocate(size));
-                } else {
-                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().allocate_aligned(size, alignment));
-                }
-            } catch (boost::interprocess::bad_alloc& ba) {
-                // LOG(warn) << "Shared memory full...";
-                if (fManager.ThrowingOnBadAlloc()) {
-                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size, ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default"));
-                }
-                rateLimiter.maybe_sleep();
-                if (fManager.Interrupted()) {
-                    return false;
-                } else {
-                    continue;
-                }
-            }
-            fMeta.fHandle = fManager.Segment().get_handle_from_address(fLocalPtr);
+        fLocalPtr = fManager.Allocate(size, alignment);
+        if (fLocalPtr) {
+            fMeta.fHandle = fManager.GetHandleFromAddress(fLocalPtr, fMeta.fSegmentId);
+            fMeta.fSize = size;
         }
-
-        fMeta.fSize = size;
-        return true;
+        return fLocalPtr;
     }
 
     void CloseMessage()
     {
         if (fMeta.fHandle >= 0 && !fQueued) {
             if (fMeta.fRegionId == 0) {
-                fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+                fManager.GetSegment(fMeta.fSegmentId);
+                fManager.Deallocate(fMeta.fHandle, fMeta.fSegmentId);
                 fMeta.fHandle = -1;
             } else {
                 if (!fRegionPtr) {
@@ -293,6 +274,8 @@ class Message final : public fair::mq::Message
                 }
             }
         }
+        fLocalPtr = nullptr;
+        fMeta.fSize = 0;
 
         fManager.DecrementMsgCounter();
     }

fairmq/shmem/Monitor.cxx

@@ -10,6 +10,7 @@
 #include "Common.h"
 
 #include <fairmq/Tools.h>
+#include <fairlogger/Logger.h>
 
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/file_mapping.hpp>
@@ -22,6 +23,11 @@
 #include <csignal>
 #include <iostream>
 #include <iomanip>
+#include <chrono>
+#include <ctime>
+#include <time.h>
+#include <iomanip>
+#include <sstream>
 
 #include <termios.h>
 #include <poll.h>
@@ -48,7 +54,7 @@ void signalHandler(int signal)
     gSignalStatus = signal;
 }
 
-Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit)
+Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, unsigned int intervalInMS, bool runAsDaemon, bool cleanOnExit)
     : fSelfDestruct(selfDestruct)
     , fInteractive(interactive)
     , fViewOnly(viewOnly)
@@ -56,15 +62,15 @@ Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool
     , fSeenOnce(false)
     , fCleanOnExit(cleanOnExit)
     , fTimeoutInMS(timeoutInMS)
+    , fIntervalInMS(intervalInMS)
     , fShmId(shmId)
-    , fSegmentName("fmq_" + fShmId + "_main")
+    , fSegmentName("fmq_" + fShmId + "_m_0")
     , fManagementSegmentName("fmq_" + fShmId + "_mng")
     , fControlQueueName("fmq_" + fShmId + "_cq")
     , fTerminating(false)
     , fHeartbeatTriggered(false)
     , fLastHeartbeat(chrono::high_resolution_clock::now())
     , fSignalThread()
-    , fManagementSegment(bipc::open_or_create, fManagementSegmentName.c_str(), 65536)
     , fDeviceHeartbeats()
 {
     if (!fViewOnly) {
@@ -75,6 +81,10 @@ Monitor::Monitor(const string& shmId, bool selfDestruct, bool interactive, bool
             throw DaemonPresent(tools::ToString("fairmq-shmmonitor for shared memory id ", fShmId, " already started or not properly exited."));
         }
     }
+
+    Logger::SetConsoleColor(false);
+    Logger::DefineVerbosity(Verbosity::user1, VerbositySpec::Make(VerbositySpec::Info::timestamp_us));
+    Logger::SetVerbosity(Verbosity::verylow);
 }
 
 void Monitor::CatchSignals()
@@ -111,7 +121,7 @@ void Monitor::Run()
         Interactive();
     } else {
         while (!fTerminating) {
-            this_thread::sleep_for(chrono::milliseconds(100));
+            this_thread::sleep_for(chrono::milliseconds(fIntervalInMS));
             CheckSegment();
         }
     }
@@ -180,11 +190,9 @@ void Monitor::Interactive()
 
     cout << endl;
     PrintHelp();
-    cout << endl;
-    PrintHeader();
 
     while (!fTerminating) {
-        if (poll(cinfd, 1, 100)) {
+        if (poll(cinfd, 1, fIntervalInMS)) {
             if (fTerminating || gSignalStatus != 0) {
                 break;
             }
@@ -196,14 +204,10 @@ void Monitor::Interactive()
                     cout << "\n[q] --> quitting." << endl;
                     fTerminating = true;
                     break;
-                case 'p':
-                    cout << "\n[p] --> active queues:" << endl;
-                    PrintQueues();
-                    break;
                 case 'x':
                     cout << "\n[x] --> closing shared memory:" << endl;
                     if (!fViewOnly) {
-                        Cleanup(fShmId);
+                        Cleanup(ShmId{fShmId});
                     } else {
                         cout << "cannot close because in view only mode" << endl;
                     }
@@ -216,6 +220,9 @@ void Monitor::Interactive()
                 case '\n':
                     cout << "\n[\\n] --> invalid input." << endl;
                     break;
+                case 'b':
+                    PrintDebugInfo(ShmId{fShmId});
+                    break;
                 default:
                     cout << "\n[" << c << "] --> invalid input." << endl;
                     break;
@@ -224,8 +231,6 @@ void Monitor::Interactive()
             if (fTerminating) {
                 break;
             }
-
-            PrintHeader();
         }
 
         if (fTerminating) {
@@ -242,45 +247,43 @@ void Monitor::Interactive()
 
 void Monitor::CheckSegment()
 {
-    char c = '#';
-
-    if (fInteractive) {
-        static uint64_t counter = 0;
-        int mod = counter++ % 5;
-        switch (mod) {
-            case 0:
-                c = '-';
-                break;
-            case 1:
-                c = '\\';
-                break;
-            case 2:
-                c = '|';
-                break;
-            case 3:
-                c = '-';
-                break;
-            case 4:
-                c = '/';
-                break;
-            default:
-                break;
-        }
-    }
+    using namespace boost::interprocess;
 
     try {
-        bipc::managed_shared_memory segment(bipc::open_only, fSegmentName.c_str());
-        bipc::managed_shared_memory managementSegment(bipc::open_only, fManagementSegmentName.c_str());
+        managed_shared_memory managementSegment(open_only, fManagementSegmentName.c_str());
+        VoidAlloc allocInstance(managementSegment.get_segment_manager());
+
+        Uint16SegmentInfoHashMap* segmentInfos = managementSegment.find<Uint16SegmentInfoHashMap>(unique_instance).first;
+        std::unordered_map<uint16_t, boost::variant<RBTreeBestFitSegment, SimpleSeqFitSegment>> segments;
+
+        if (!segmentInfos) {
+            cout << "Found management segment, but cannot locate segment info, something went wrong..." << endl;
+            return;
+        }
+
+        for (const auto& s : *segmentInfos) {
+            if (s.second.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
+                segments.emplace(s.first, RBTreeBestFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + to_string(s.first)).c_str()));
+            } else {
+                segments.emplace(s.first, SimpleSeqFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + to_string(s.first)).c_str()));
+            }
+        }
 
         fSeenOnce = true;
 
         unsigned int numDevices = 0;
+#ifdef FAIRMQ_DEBUG_MODE
+        Uint16MsgCounterHashMap* msgCounters = nullptr;
+#endif
 
-        if (fInteractive) {
-            DeviceCounter* dc = managementSegment.find<DeviceCounter>(bipc::unique_instance).first;
+        if (fInteractive || fViewOnly) {
+            DeviceCounter* dc = managementSegment.find<DeviceCounter>(unique_instance).first;
             if (dc) {
                 numDevices = dc->fCount;
             }
+#ifdef FAIRMQ_DEBUG_MODE
+            msgCounters = managementSegment.find_or_construct<Uint16MsgCounterHashMap>(unique_instance)(allocInstance);
+#endif
         }
 
         auto now = chrono::high_resolution_clock::now();
@@ -288,7 +291,7 @@ void Monitor::CheckSegment()
 
         if (fHeartbeatTriggered && duration > fTimeoutInMS) {
             cout << "no heartbeats since over " << fTimeoutInMS << " milliseconds, cleaning..." << endl;
-            Cleanup(fShmId);
+            Cleanup(ShmId{fShmId});
             fHeartbeatTriggered = false;
             if (fSelfDestruct) {
                 cout << "\nself destructing" << endl;
@@ -296,32 +299,43 @@ void Monitor::CheckSegment()
             }
         }
 
-        if (fInteractive) {
-            cout << "| "
-                 << setw(18) << fSegmentName                                    << " | "
-                 << setw(10) << segment.get_size()                              << " | "
-                 << setw(10) << segment.get_free_memory()                       << " | "
-                 << setw(8)  << numDevices                                      << " | "
-                 << setw(10) << (fViewOnly ? "view only" : to_string(duration)) << " |"
-                 << c << flush;
+        if (fInteractive || fViewOnly) {
+            stringstream ss;
+            size_t mfree = managementSegment.get_free_memory();
+            size_t mtotal = managementSegment.get_size();
+            size_t mused = mtotal - mfree;
+
+            ss << "shm id: " << fShmId
+               << ", devices: " << numDevices << ", segments:\n";
+            for (const auto& s : segments) {
+                size_t free = boost::apply_visitor(SegmentFreeMemory{}, s.second);
+                size_t total = boost::apply_visitor(SegmentSize{}, s.second);
+                size_t used = total - free;
+                ss << "   [" << s.first
+                   << "]: total: " << total
+#ifdef FAIRMQ_DEBUG_MODE
+                   << ", msgs: " << (*msgCounters)[s.first].fCount
+#else
+                   << ", msgs: NODEBUG"
+#endif
+                   << ", free: " << free
+                   << ", used: " << used
+                   << "\n";
+            }
+            ss << "   [m]: "
+               << "total: " << mtotal
+               << ", free: " << mfree
+               << ", used: " << mused;
+            LOGV(info, user1) << ss.str();
         }
     } catch (bie&) {
         fHeartbeatTriggered = false;
-        if (fInteractive) {
-            cout << "| "
-                 << setw(18) << "-" << " | "
-                 << setw(10) << "-" << " | "
-                 << setw(10) << "-" << " | "
-                 << setw(8)  << "-" << " | "
-                 << setw(10) << "-" << " |"
-                 << c << flush;
-        }
 
         auto now = chrono::high_resolution_clock::now();
         unsigned int duration = chrono::duration_cast<chrono::milliseconds>(now - fLastHeartbeat).count();
 
         if (fIsDaemon && duration > fTimeoutInMS * 2) {
-            Cleanup(fShmId);
+            Cleanup(ShmId{fShmId});
             fHeartbeatTriggered = false;
             if (fSelfDestruct) {
                 cout << "\nself destructing" << endl;
@@ -338,150 +352,214 @@ void Monitor::CheckSegment()
     }
 }
 
-void Monitor::PrintQueues()
+void Monitor::PrintDebugInfo(const ShmId& shmId __attribute__((unused)))
 {
-    cout << '\n';
-
+#ifdef FAIRMQ_DEBUG_MODE
+    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
     try {
-        bipc::managed_shared_memory segment(bipc::open_only, fSegmentName.c_str());
-        StrVector* queues = segment.find<StrVector>(string("fmq_" + fShmId + "_qs").c_str()).first;
-        if (queues) {
-            cout << "found " << queues->size() << " queue(s):" << endl;
+        bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
+        boost::interprocess::named_mutex mtx(boost::interprocess::open_only, string("fmq_" + shmId.shmId + "_mtx").c_str());
+        boost::interprocess::scoped_lock<bipc::named_mutex> lock(mtx);
 
-            for (const auto& queue : *queues) {
-                string name(queue.c_str());
-                cout << '\t' << name << " : ";
-                atomic<int>* queueSize = segment.find<atomic<int>>(name.c_str()).first;
-                if (queueSize) {
-                    cout << *queueSize << " messages" << endl;
-                } else {
-                    cout << "\tqueue does not have a queue size entry." << endl;
-                }
-            }
-        } else {
-            cout << "\tno queues found" << endl;
+        Uint16MsgDebugMapHashMap* debug = managementSegment.find<Uint16MsgDebugMapHashMap>(bipc::unique_instance).first;
+
+        size_t numMessages = 0;
+
+        for (const auto& e : *debug) {
+            numMessages += e.second.size();
         }
+        cout << endl << "found " << numMessages << " messages." << endl;
+
+        for (const auto& s : *debug) {
+            for (const auto& e : s.second) {
+                using time_point = chrono::system_clock::time_point;
+                time_point tmpt{chrono::duration_cast<time_point::duration>(chrono::nanoseconds(e.second.fCreationTime))};
+                time_t t = chrono::system_clock::to_time_t(tmpt);
+                uint64_t ms = e.second.fCreationTime % 1000000;
+                auto tm = localtime(&t);
+                cout << "segment: " << setw(3) << setfill(' ') << s.first
+                     << ", offset: " << setw(12) << setfill(' ') << e.first
+                     << ", size: " << setw(10) << setfill(' ') << e.second.fSize
+                     << ", creator PID: " << e.second.fPid << setfill('0')
+                     << ", at: " << setw(2) << tm->tm_hour << ":" << setw(2) << tm->tm_min << ":" << setw(2) << tm->tm_sec << "." << setw(6) << ms << endl;
+            }
+        }
+        cout << setfill(' ');
     } catch (bie&) {
-        cout << "\tno queues found" << endl;
-    } catch (out_of_range&) {
-        cout << "\tno queues found" << endl;
+        cout << "no segments found" << endl;
     }
-
-    cout << "\n    --> last heartbeats: " << endl << endl;
-    auto now = chrono::high_resolution_clock::now();
-    for (const auto& h : fDeviceHeartbeats)  {
-        cout << "\t" << h.first << " : " << chrono::duration<double, milli>(now - h.second).count() << "ms ago." << endl;
-    }
-
-    cout << endl;
+#else
+    cout << "FairMQ was not compiled in debug mode (FAIRMQ_DEBUG_MODE)" << endl;
+#endif
 }
 
-void Monitor::PrintHeader()
+void Monitor::PrintDebugInfo(const SessionId& sessionId)
 {
-    cout << "| "
-         << setw(18) << "name"    << " | "
-         << setw(10) << "size"    << " | "
-         << setw(10) << "free"    << " | "
-         << setw(8)  << "devices" << " | "
-         << setw(10) << "last hb" << " |"
-         << endl;
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    PrintDebugInfo(shmId);
+}
+
+unordered_map<uint16_t, std::vector<BufferDebugInfo>> Monitor::GetDebugInfo(const ShmId& shmId __attribute__((unused)))
+{
+    unordered_map<uint16_t, std::vector<BufferDebugInfo>> result;
+
+#ifdef FAIRMQ_DEBUG_MODE
+    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
+    try {
+        bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
+        boost::interprocess::named_mutex mtx(boost::interprocess::open_only, string("fmq_" + shmId.shmId + "_mtx").c_str());
+        boost::interprocess::scoped_lock<bipc::named_mutex> lock(mtx);
+
+        Uint16MsgDebugMapHashMap* debug = managementSegment.find<Uint16MsgDebugMapHashMap>(bipc::unique_instance).first;
+
+        result.reserve(debug->size());
+
+        for (const auto& s : *debug) {
+            result[s.first].reserve(s.second.size());
+            for (const auto& e : s.second) {
+                result[s.first][e.first] = BufferDebugInfo(e.first, e.second.fPid, e.second.fSize, e.second.fCreationTime);
+            }
+        }
+    } catch (bie&) {
+        cout << "no segments found" << endl;
+    }
+#else
+    cout << "FairMQ was not compiled in debug mode (FAIRMQ_DEBUG_MODE)" << endl;
+#endif
+
+    return result;
+}
+unordered_map<uint16_t, std::vector<BufferDebugInfo>> Monitor::GetDebugInfo(const SessionId& sessionId)
+{
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    return GetDebugInfo(shmId);
 }
 
 void Monitor::PrintHelp()
 {
-    cout << "controls: [x] close memory, [p] print queues, [h] help, [q] quit." << endl;
+    cout << "controls: [x] close memory, "
+         << "[b] print a list of allocated messages (only available when compiled with FAIMQ_DEBUG_MODE=ON), "
+         << "[h] help, "
+         << "[q] quit." << endl;
 }
 
-void Monitor::RemoveObject(const string& name)
+
+std::pair<std::string, bool> RunRemoval(std::function<bool(const std::string&)> f, std::string name, bool verbose)
 {
-    if (bipc::shared_memory_object::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
+    if (f(name)) {
+        if (verbose) {
+            cout << "Successfully removed '" << name << "'." << endl;
+        }
+        return {name, true};
     } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+        if (verbose) {
+            cout << "Did not remove '" << name << "'. Already removed?" << endl;
+        }
+        return {name, false};
     }
 }
 
-void Monitor::RemoveFileMapping(const string& name)
-{
-    if (bipc::file_mapping::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
-    } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
-    }
-}
+bool Monitor::RemoveObject(const string& name)      { return bipc::shared_memory_object::remove(name.c_str()); }
+bool Monitor::RemoveFileMapping(const string& name) { return bipc::file_mapping::remove(name.c_str()); }
+bool Monitor::RemoveQueue(const string& name)       { return bipc::message_queue::remove(name.c_str()); }
+bool Monitor::RemoveMutex(const string& name)       { return bipc::named_mutex::remove(name.c_str()); }
+bool Monitor::RemoveCondition(const string& name)   { return bipc::named_condition::remove(name.c_str()); }
 
-void Monitor::RemoveQueue(const string& name)
+std::vector<std::pair<std::string, bool>> Monitor::Cleanup(const ShmId& shmId, bool verbose /* = true */)
 {
-    if (bipc::message_queue::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
-    } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
-    }
-}
+    std::vector<std::pair<std::string, bool>> result;
 
-void Monitor::RemoveMutex(const string& name)
-{
-    if (bipc::named_mutex::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
-    } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
+    if (verbose) {
+        cout << "Cleaning up for shared memory id '" << shmId.shmId << "'..." << endl;
     }
-}
 
-void Monitor::RemoveCondition(const string& name)
-{
-    if (bipc::named_condition::remove(name.c_str())) {
-        cout << "Successfully removed \"" << name << "\"." << endl;
-    } else {
-        cout << "Did not remove \"" << name << "\". Already removed?" << endl;
-    }
-}
-
-void Monitor::Cleanup(const string& shmId)
-{
-    string managementSegmentName("fmq_" + shmId + "_mng");
+    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
     try {
         bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
-        RegionCounter* rc = managementSegment.find<RegionCounter>(bipc::unique_instance).first;
-        if (rc) {
-            cout << "Region counter found: " << rc->fCount << endl;
-            uint64_t regionCount = rc->fCount;
 
-            Uint64RegionInfoMap* m = managementSegment.find<Uint64RegionInfoMap>(bipc::unique_instance).first;
-
-            for (uint64_t i = 1; i <= regionCount; ++i) {
-                if (m != nullptr) {
-                    RegionInfo ri = m->at(i);
-                    string path = ri.fPath.c_str();
-                    int flags = ri.fFlags;
-                    cout << "Found RegionInfo with path: '" << path << "', flags: " << flags << "'." << endl;
-                    if (path != "") {
-                        RemoveFileMapping(tools::ToString(path, "fmq_" + shmId + "_rg_" + to_string(i)));
-                    } else {
-                        RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
-                    }
-                } else {
-                    RemoveObject("fmq_" + shmId + "_rg_" + to_string(i));
+        try {
+            RegionCounter* rc = managementSegment.find<RegionCounter>(bipc::unique_instance).first;
+            if (rc) {
+                if (verbose) {
+                    cout << "Region counter found: " << rc->fCount << endl;
                 }
+                uint16_t regionCount = rc->fCount;
 
-                RemoveQueue(string("fmq_" + shmId + "_rgq_" + to_string(i)));
+                Uint16RegionInfoMap* m = managementSegment.find<Uint16RegionInfoMap>(bipc::unique_instance).first;
+
+                for (uint16_t i = 1; i <= regionCount; ++i) {
+                    if (m != nullptr) {
+                        RegionInfo ri = m->at(i);
+                        string path = ri.fPath.c_str();
+                        int flags = ri.fFlags;
+                        if (verbose) {
+                            cout << "Found RegionInfo with path: '" << path << "', flags: " << flags << ", fDestroyed: " << ri.fDestroyed << "." << endl;
+                        }
+                        if (path != "") {
+                            result.emplace_back(RunRemoval(Monitor::RemoveFileMapping, path + "fmq_" + shmId.shmId + "_rg_" + to_string(i), verbose));
+                        } else {
+                            result.emplace_back(RunRemoval(Monitor::RemoveObject, "fmq_" + shmId.shmId + "_rg_" + to_string(i), verbose));
+                        }
+                    } else {
+                        result.emplace_back(RunRemoval(Monitor::RemoveObject, "fmq_" + shmId.shmId + "_rg_" + to_string(i), verbose));
+                    }
+
+                    result.emplace_back(RunRemoval(Monitor::RemoveQueue, string("fmq_" + shmId.shmId + "_rgq_" + to_string(i)), verbose));
+                }
+            } else {
+                if (verbose) {
+                    cout << "No region counter found. No regions to cleanup." << endl;
+                }
+            }
+        } catch(out_of_range& oor) {
+            if (verbose) {
+                cout << "Could not locate element in the region map, out of range: " << oor.what() << endl;
             }
-        } else {
-            cout << "No region counter found. no regions to cleanup." << endl;
         }
 
-        RemoveObject(managementSegmentName.c_str());
+        Uint16SegmentInfoHashMap* segmentInfos = managementSegment.find<Uint16SegmentInfoHashMap>(bipc::unique_instance).first;
+
+        for (const auto& s : *segmentInfos) {
+            result.emplace_back(RunRemoval(Monitor::RemoveObject, "fmq_" + shmId.shmId + "_m_" + to_string(s.first), verbose));
+        }
+
+        result.emplace_back(RunRemoval(Monitor::RemoveObject, managementSegmentName.c_str(), verbose));
     } catch (bie&) {
-        cout << "Did not find '" << managementSegmentName << "' shared memory segment. No regions to cleanup." << endl;
-    } catch(std::out_of_range& oor) {
-        cout << "Could not locate element in the region map, out of range: " << oor.what() << endl;
+        if (verbose) {
+            cout << "Did not find '" << managementSegmentName << "' shared memory segment. No regions to cleanup." << endl;
+        }
     }
 
-    RemoveObject("fmq_" + shmId + "_main");
-    RemoveMutex("fmq_" + shmId + "_mtx");
-    RemoveCondition("fmq_" + shmId + "_cv");
+    result.emplace_back(RunRemoval(Monitor::RemoveMutex, "fmq_" + shmId.shmId + "_mtx", verbose));
+    result.emplace_back(RunRemoval(Monitor::RemoveCondition, "fmq_" + shmId.shmId + "_cv", verbose));
 
-    cout << endl;
+    return result;
+}
+
+std::vector<std::pair<std::string, bool>> Monitor::Cleanup(const SessionId& sessionId, bool verbose /* = true */)
+{
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    if (verbose) {
+        cout << "Cleanup called with session id '" << sessionId.sessionId << "', translating to shared memory id '" << shmId.shmId << "'" << endl;
+    }
+    return Cleanup(shmId, verbose);
+}
+
+std::vector<std::pair<std::string, bool>> Monitor::CleanupFull(const ShmId& shmId, bool verbose /* = true */)
+{
+    auto result = Cleanup(shmId, verbose);
+    result.emplace_back(RunRemoval(Monitor::RemoveMutex, "fmq_" + shmId.shmId + "_ms", verbose));
+    result.emplace_back(RunRemoval(Monitor::RemoveQueue, "fmq_" + shmId.shmId + "_cq", verbose));
+    return result;
+}
+
+std::vector<std::pair<std::string, bool>> Monitor::CleanupFull(const SessionId& sessionId, bool verbose /* = true */)
+{
+    ShmId shmId{buildShmIdFromSessionIdAndUserId(sessionId.sessionId)};
+    if (verbose) {
+        cout << "Cleanup called with session id '" << sessionId.sessionId << "', translating to shared memory id '" << shmId.shmId << "'" << endl;
+    }
+    return CleanupFull(shmId, verbose);
 }
 
 Monitor::~Monitor()
@@ -490,7 +568,7 @@ Monitor::~Monitor()
         fSignalThread.join();
     }
     if (fCleanOnExit) {
-        Cleanup(fShmId);
+        Cleanup(ShmId{fShmId});
     }
     if (!fViewOnly) {
         RemoveMutex("fmq_" + fShmId + "_ms");

fairmq/shmem/Monitor.h

@@ -8,14 +8,14 @@
 #ifndef FAIR_MQ_SHMEM_MONITOR_H_
 #define FAIR_MQ_SHMEM_MONITOR_H_
 
-#include <boost/interprocess/managed_shared_memory.hpp>
-
 #include <thread>
 #include <chrono>
 #include <atomic>
 #include <string>
 #include <stdexcept>
 #include <unordered_map>
+#include <utility> // pair
+#include <vector>
 
 namespace fair
 {
@@ -24,10 +24,37 @@ namespace mq
 namespace shmem
 {
 
+struct SessionId
+{
+    std::string sessionId;
+    explicit operator std::string() const { return sessionId; }
+};
+
+struct ShmId
+{
+    std::string shmId;
+    explicit operator std::string() const { return shmId; }
+};
+
+struct BufferDebugInfo
+{
+    BufferDebugInfo(size_t offset, pid_t pid, size_t size, uint64_t creationTime)
+        : fOffset(offset)
+        , fPid(pid)
+        , fSize(size)
+        , fCreationTime(creationTime)
+    {}
+
+    size_t fOffset;
+    pid_t fPid;
+    size_t fSize;
+    uint64_t fCreationTime;
+};
+
 class Monitor
 {
   public:
-    Monitor(const std::string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, bool runAsDaemon, bool cleanOnExit);
+    Monitor(const std::string& shmId, bool selfDestruct, bool interactive, bool viewOnly, unsigned int timeoutInMS, unsigned int intervalInMS, bool runAsDaemon, bool cleanOnExit);
 
     Monitor(const Monitor&) = delete;
     Monitor operator=(const Monitor&) = delete;
@@ -37,19 +64,38 @@ class Monitor
     void CatchSignals();
     void Run();
 
-    static void Cleanup(const std::string& shmId);
-    static void RemoveObject(const std::string&);
-    static void RemoveFileMapping(const std::string&);
-    static void RemoveQueue(const std::string&);
-    static void RemoveMutex(const std::string&);
-    static void RemoveCondition(const std::string&);
+    /// @brief Cleanup all shared memory artifacts created by devices
+    /// @param shmId shared memory id
+    /// @param verbose output cleanup results to stdout
+    static std::vector<std::pair<std::string, bool>> Cleanup(const ShmId& shmId, bool verbose = true);
+    /// @brief Cleanup all shared memory artifacts created by devices
+    /// @param sessionId session id
+    /// @param verbose output cleanup results to stdout
+    static std::vector<std::pair<std::string, bool>> Cleanup(const SessionId& sessionId, bool verbose = true);
+    /// @brief Cleanup all shared memory artifacts created by devices and monitors
+    /// @param shmId shared memory id
+    /// @param verbose output cleanup results to stdout
+    static std::vector<std::pair<std::string, bool>> CleanupFull(const ShmId& shmId, bool verbose = true);
+    /// @brief Cleanup all shared memory artifacts created by devices and monitors
+    /// @param sessionId session id
+    /// @param verbose output cleanup results to stdout
+    static std::vector<std::pair<std::string, bool>> CleanupFull(const SessionId& sessionId, bool verbose = true);
+
+    static void PrintDebugInfo(const ShmId& shmId);
+    static void PrintDebugInfo(const SessionId& shmId);
+    static std::unordered_map<uint16_t, std::vector<BufferDebugInfo>> GetDebugInfo(const ShmId& shmId);
+    static std::unordered_map<uint16_t, std::vector<BufferDebugInfo>> GetDebugInfo(const SessionId& shmId);
+
+    static bool RemoveObject(const std::string& name);
+    static bool RemoveFileMapping(const std::string& name);
+    static bool RemoveQueue(const std::string& name);
+    static bool RemoveMutex(const std::string& name);
+    static bool RemoveCondition(const std::string& name);
 
     struct DaemonPresent : std::runtime_error { using std::runtime_error::runtime_error; };
 
   private:
-    void PrintHeader();
     void PrintHelp();
-    void PrintQueues();
     void MonitorHeartbeats();
     void CheckSegment();
     void Interactive();
@@ -62,6 +108,7 @@ class Monitor
     bool fSeenOnce; // true is segment has been opened successfully at least once
     bool fCleanOnExit;
     unsigned int fTimeoutInMS;
+    unsigned int fIntervalInMS;
     std::string fShmId;
     std::string fSegmentName;
     std::string fManagementSegmentName;
@@ -70,7 +117,6 @@ class Monitor
     std::atomic<bool> fHeartbeatTriggered;
     std::chrono::high_resolution_clock::time_point fLastHeartbeat;
     std::thread fSignalThread;
-    boost::interprocess::managed_shared_memory fManagementSegment;
     std::unordered_map<std::string, std::chrono::high_resolution_clock::time_point> fDeviceHeartbeats;
 };
 

fairmq/shmem/Poller.h

@@ -127,13 +127,20 @@ class Poller final : public fair::mq::Poller
 
     void Poll(const int timeout) override
     {
-        if (zmq_poll(fItems, fNumItems, timeout) < 0) {
-            if (errno == ETERM) {
-                LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
-            } else {
-                LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
-                throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+        while (true) {
+            if (zmq_poll(fItems, fNumItems, timeout) < 0) {
+                if (errno == ETERM) {
+                    LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
+                    return;
+                } else if (errno == EINTR) {
+                    LOG(debug) << "polling interrupted by system call";
+                    continue;
+                } else {
+                    LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
+                    throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+                }
             }
+            break;
         }
     }
 

fairmq/shmem/README.md

@@ -6,6 +6,23 @@ The transport manages shared memory via boost::interprocess library. The transfe
 
 Devices track and cleanup shared memory on shutdown. For more information on the current shared memory segment and additional cleanup options, see following section.
 
+# Shared Memory objects / files
+
+FairMQ Shared Memory currently uses the following names to register shared memory on the system:
+
+| name                        | info                                           | created by         | used by                        |
+| --------------------------- | ---------------------------------------------- | ------------------ | ------------------------------ |
+| `fmq_<shmId>_m_<segmentId>` | managed segment(s) (user data)                 | one of the devices | devices                        |
+| `fmq_<shmId>_mng`           | management segment (management data)           | one of the devices | devices                        |
+| `fmq_<shmId>_mtx`           | mutex                                          | one of the devices | devices                        |
+| `fmq_<shmId>_cv`            | condition variable                             | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_rg_<index>`    | unmanaged region(s)                            | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_rgq_<index>`   | unmanaged region queue(s)                      | one of the devices | devices with unmanaged regions |
+| `fmq_<shmId>_ms`            | shmmonitor status                              | shmmonitor         | devices, shmmonitor            |
+| `fmq_<shmId>_cq`            | message queue between transport and shmmonitor | shmmonitor         | devices, shmmonitor            |
+
+The shmId is generated out of session id and user id.
+
 ## Shared memory monitor
 
 The shared memory monitor tool, supplied with the shared memory transport can be used to monitor shared memory use and automatically cleanup shared memory in case of device crashes.
@@ -25,13 +42,3 @@ Without the `--self-destruct` option, the monitor will run continuously, moitori
 Possible further implementation would be to run the monitor with `--self-destruct` with each topology.
 
 The Monitor class can also be used independently from the supplied executable (built from `runMonitor.cxx`), allowing integration on any level. For example invoking the monitor could be a functionality that a device offers.
-
-FairMQ Shared Memory currently uses following names to register shared memory on the system:
-
-`fmq_<shmId>_main` - main segment name, used for user data (the shmId is generated out of session id and user id).
-`fmq_<shmId>_mng` - management segment name, used for storing management data.
-`fmq_<shmId>_cq` - message queue for communicating between shm transport and shm monitor (exists independent of above segments).
-`fmq_<shmId>_mtx` - boost::interprocess::named_mutex for management purposes (exists independent of above segments).
-`fmq_<shmId>_ms` - shmmonitor status used to signal if it is active or not (exists independent of above segments).
-`fmq_<shmId>_rg_<index>` - names of unmanaged regions.
-`fmq_<shmId>_rgq_<index>` - names of queues for the unmanaged regions.

fairmq/shmem/Region.h

@@ -30,6 +30,7 @@
 #include <boost/interprocess/ipc/message_queue.hpp>
 
 #include <algorithm> // min
+#include <atomic>
 #include <thread>
 #include <mutex>
 #include <condition_variable>
@@ -47,8 +48,9 @@ namespace shmem
 
 struct Region
 {
-    Region(const std::string& shmId, uint64_t id, uint64_t size, bool remote, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string& path, int flags)
+    Region(const std::string& shmId, uint16_t id, uint64_t size, bool remote, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string& path, int flags)
         : fRemote(remote)
+        , fLinger(100)
         , fStop(false)
         , fName("fmq_" + shmId + "_rg_" + std::to_string(id))
         , fQueueName("fmq_" + shmId + "_rgq_" + std::to_string(id))
@@ -56,8 +58,8 @@ struct Region
         , fFile(nullptr)
         , fFileMapping()
         , fQueue(nullptr)
-        , fReceiveAcksWorker()
-        , fSendAcksWorker()
+        , fAcksReceiver()
+        , fAcksSender()
         , fCallback(callback)
         , fBulkCallback(bulkCallback)
     {
@@ -118,38 +120,35 @@ struct Region
         LOG(debug) << "shmem: initialized region queue: " << fQueueName;
     }
 
-    void StartSendingAcks()
-    {
-        fSendAcksWorker = std::thread(&Region::SendAcks, this);
-    }
-
+    void StartSendingAcks() { fAcksSender = std::thread(&Region::SendAcks, this); }
     void SendAcks()
     {
         std::unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
+        size_t blocksToSend = 0;
 
-        while (true) { // we'll try to send all acks before stopping
-            size_t blocksToSend = 0;
-
-            {   // mutex locking block
+        while (true) {
+            blocksToSend = 0;
+            {
                 std::unique_lock<std::mutex> lock(fBlockMtx);
 
-                // try to get more blocks without waiting (we can miss a notify from CloseMessage())
-                if (!fStop && (fBlocksToFree.size() < fAckBunchSize)) {
-                    // cv.wait() timeout: send whatever blocks we have
+                // try to get <fAckBunchSize> blocks
+                if (fBlocksToFree.size() < fAckBunchSize) {
                     fBlockSendCV.wait_for(lock, std::chrono::milliseconds(500));
                 }
 
+                // send whatever blocks we have
                 blocksToSend = std::min(fBlocksToFree.size(), fAckBunchSize);
 
                 copy_n(fBlocksToFree.end() - blocksToSend, blocksToSend, blocks.get());
                 fBlocksToFree.resize(fBlocksToFree.size() - blocksToSend);
-            } // unlock the block mutex here while sending over IPC
+            }
 
             if (blocksToSend > 0) {
                 while (!fQueue->try_send(blocks.get(), blocksToSend * sizeof(RegionBlock), 0) && !fStop) {
                     // receiver slow? yield and try again...
                     std::this_thread::yield();
                 }
+                // LOG(debug) << "Sent " << blocksToSend << " blocks.";
             } else { // blocksToSend == 0
                 if (fStop) {
                     break;
@@ -157,14 +156,11 @@ struct Region
             }
         }
 
-        LOG(debug) << "send ack worker for " << fName << " leaving.";
-    }
-
-    void StartReceivingAcks()
-    {
-        fReceiveAcksWorker = std::thread(&Region::ReceiveAcks, this);
+        LOG(debug) << "AcksSender for " << fName << " leaving " << "(blocks left to free: " << fBlocksToFree.size() << ", "
+                                                                << " blocks left to send: " << blocksToSend << ").";
     }
 
+    void StartReceivingAcks() { fAcksReceiver = std::thread(&Region::ReceiveAcks, this); }
     void ReceiveAcks()
     {
         unsigned int priority;
@@ -173,12 +169,18 @@ struct Region
         std::vector<fair::mq::RegionBlock> result;
         result.reserve(fAckBunchSize);
 
-        while (!fStop) { // end thread condition (should exist until region is destroyed)
-            auto rcvTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(500);
+        while (true) {
+            uint32_t timeout = 100;
+            bool leave = false;
+            if (fStop) {
+                timeout = fLinger;
+                leave = true;
+            }
+            auto rcvTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(timeout);
 
             while (fQueue->timed_receive(blocks.get(), fAckBunchSize * sizeof(RegionBlock), recvdSize, priority, rcvTill)) {
-                // LOG(debug) << "received: " << block.fHandle << " " << block.fSize << " " << block.fMessageId;
                 const auto numBlocks = recvdSize / sizeof(RegionBlock);
+                // LOG(debug) << "Received " << numBlocks << " blocks (recvdSize: " << recvdSize << "). (remaining queue size: " << fQueue->get_num_msg() << ").";
                 if (fBulkCallback) {
                     result.clear();
                     for (size_t i = 0; i < numBlocks; i++) {
@@ -191,9 +193,13 @@ struct Region
                     }
                 }
             }
-        } // while !fStop
 
-        LOG(debug) << "ReceiveAcks() worker for " << fName << " leaving.";
+            if (leave) {
+                break;
+            }
+        }
+
+        LOG(debug) << "AcksReceiver for " << fName << " leaving (remaining queue size: " << fQueue->get_num_msg() << ").";
     }
 
     void ReleaseBlock(const RegionBlock& block)
@@ -203,31 +209,34 @@ struct Region
         fBlocksToFree.emplace_back(block);
 
         if (fBlocksToFree.size() >= fAckBunchSize) {
-            lock.unlock(); // reduces contention on fBlockMtx
+            lock.unlock();
             fBlockSendCV.notify_one();
         }
     }
 
+    void SetLinger(uint32_t linger) { fLinger = linger; }
+    uint32_t GetLinger() const { return fLinger; }
+
     ~Region()
     {
         fStop = true;
 
-        if (fSendAcksWorker.joinable()) {
+        if (fAcksSender.joinable()) {
             fBlockSendCV.notify_one();
-            fSendAcksWorker.join();
+            fAcksSender.join();
         }
 
         if (!fRemote) {
-            if (fReceiveAcksWorker.joinable()) {
-                fReceiveAcksWorker.join();
+            if (fAcksReceiver.joinable()) {
+                fAcksReceiver.join();
             }
 
             if (boost::interprocess::shared_memory_object::remove(fName.c_str())) {
-                LOG(debug) << "shmem: destroyed region " << fName;
+                LOG(debug) << "Region '" << fName << "' destroyed.";
             }
 
             if (boost::interprocess::file_mapping::remove(fName.c_str())) {
-                LOG(debug) << "shmem: destroyed file mapping " << fName;
+                LOG(debug) << "File mapping '" << fName << "' destroyed.";
             }
 
             if (fFile) {
@@ -235,16 +244,19 @@ struct Region
             }
 
             if (boost::interprocess::message_queue::remove(fQueueName.c_str())) {
-                LOG(debug) << "shmem: removed region queue " << fQueueName;
+                LOG(debug) << "Region queue '" << fQueueName << "' destroyed.";
             }
         } else {
             // LOG(debug) << "shmem: region '" << fName << "' is remote, no cleanup necessary.";
-            LOG(debug) << "shmem: region queue '" << fQueueName << "' is remote, no cleanup necessary";
+            LOG(debug) << "Region queue '" << fQueueName << "' is remote, no cleanup necessary";
         }
+
+        LOG(debug) << "Region '" << fName << "' (" << (fRemote ? "remote" : "local") << ") destructed.";
     }
 
     bool fRemote;
-    bool fStop;
+    uint32_t fLinger;
+    std::atomic<bool> fStop;
     std::string fName;
     std::string fQueueName;
     boost::interprocess::shared_memory_object fShmemObject;
@@ -258,8 +270,8 @@ struct Region
     const std::size_t fAckBunchSize = 256;
     std::unique_ptr<boost::interprocess::message_queue> fQueue;
 
-    std::thread fReceiveAcksWorker;
-    std::thread fSendAcksWorker;
+    std::thread fAcksReceiver;
+    std::thread fAcksSender;
     RegionCallback fCallback;
     RegionBulkCallback fBulkCallback;
 };

fairmq/shmem/Socket.h

@@ -55,8 +55,7 @@ class Socket final : public fair::mq::Socket
         , fBytesRx(0)
         , fMessagesTx(0)
         , fMessagesRx(0)
-        , fSndTimeout(100)
-        , fRcvTimeout(100)
+        , fTimeout(100)
     {
         assert(context);
         fSocket = zmq_socket(context, GetConstant(type));
@@ -77,11 +76,11 @@ class Socket final : public fair::mq::Socket
             LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
         }
 
-        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0) {
+        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fTimeout, sizeof(fTimeout)) != 0) {
             LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
         }
 
-        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0) {
+        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fTimeout, sizeof(fTimeout)) != 0) {
             LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
         }
 
@@ -129,6 +128,32 @@ class Socket final : public fair::mq::Socket
         return true;
     }
 
+    bool ShouldRetry(int flags, int timeout, int& elapsed) const
+    {
+        if ((flags & ZMQ_DONTWAIT) == 0) {
+            if (timeout > 0) {
+                elapsed += fTimeout;
+                if (elapsed >= timeout) {
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    int HandleErrors() const
+    {
+        if (zmq_errno() == ETERM) {
+            LOG(debug) << "Terminating socket " << fId;
+            return static_cast<int>(TransferResult::error);
+        } else {
+            LOG(error) << "Failed transfer on socket " << fId << ", reason: " << zmq_strerror(errno);
+            return static_cast<int>(TransferResult::error);
+        }
+    }
+
     int Send(MessagePtr& msg, const int timeout = -1) override
     {
         int flags = 0;
@@ -141,7 +166,7 @@ class Socket final : public fair::mq::Socket
         ZMsg zmqMsg(sizeof(MetaHeader));
         std::memcpy(zmqMsg.Data(), &(shmMsg->fMeta), sizeof(MetaHeader));
 
-        while (true && !fManager.Interrupted()) {
+        while (true) {
             int nbytes = zmq_msg_send(zmqMsg.Msg(), fSocket, flags);
             if (nbytes > 0) {
                 shmMsg->fQueued = true;
@@ -149,31 +174,20 @@ class Socket final : public fair::mq::Socket
                 size_t size = msg->GetSize();
                 fBytesTx += size;
                 return size;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fSndTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fManager.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Send interrupted by system call";
-                return nbytes;
-            }else {
-                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-                return nbytes;
+            } else {
+                return HandleErrors();
             }
         }
 
-        return -1;
+        return static_cast<int>(TransferResult::error);
     }
 
     int Receive(MessagePtr& msg, const int timeout = -1) override
@@ -205,27 +219,16 @@ class Socket final : public fair::mq::Socket
                 fBytesRx += size;
                 ++fMessagesRx;
                 return size;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fRcvTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fManager.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Receive interrupted by system call";
-                return nbytes;
-            }else {
-                LOG(error) << "Failed receiving on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-                return nbytes;
+            } else {
+                return HandleErrors();
             }
         }
     }
@@ -250,7 +253,7 @@ class Socket final : public fair::mq::Socket
             std::memcpy(metas++, &(shmMsg->fMeta), sizeof(MetaHeader));
         }
 
-        while (!fManager.Interrupted()) {
+        while (true) {
             int64_t totalSize = 0;
             int nbytes = zmq_msg_send(zmqMsg.Msg(), fSocket, flags);
             if (nbytes > 0) {
@@ -267,31 +270,20 @@ class Socket final : public fair::mq::Socket
                 fBytesTx += totalSize;
 
                 return totalSize;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fSndTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fManager.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Send interrupted by system call";
-                return nbytes;
-            }else {
-                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-                return nbytes;
+            } else {
+                return HandleErrors();
             }
         }
 
-        return -1;
+        return static_cast<int>(TransferResult::error);
     }
 
     int64_t Receive(std::vector<MessagePtr>& msgVec, const int timeout = -1) override
@@ -304,7 +296,7 @@ class Socket final : public fair::mq::Socket
 
         ZMsg zmqMsg;
 
-        while (!fManager.Interrupted()) {
+        while (true) {
             int64_t totalSize = 0;
             int nbytes = zmq_msg_recv(zmqMsg.Msg(), fSocket, flags);
             if (nbytes > 0) {
@@ -334,28 +326,20 @@ class Socket final : public fair::mq::Socket
                 fBytesRx += totalSize;
 
                 return totalSize;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fRcvTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fManager.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Receive interrupted by system call";
-                return nbytes;
             } else {
-                LOG(error) << "Failed receiving on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-                return nbytes;
+                return HandleErrors();
             }
         }
 
-        return -1;
+        return static_cast<int>(TransferResult::error);
     }
 
     void* GetSocket() const { return fSocket; }
@@ -396,6 +380,15 @@ class Socket final : public fair::mq::Socket
         }
     }
 
+    void Events(uint32_t* events) override
+    {
+        size_t eventsSize = sizeof(uint32_t);
+        if (zmq_getsockopt(fSocket, ZMQ_EVENTS, events, &eventsSize) < 0) {
+            throw SocketError(
+                tools::ToString("failed setting ZMQ_EVENTS, reason: ", zmq_strerror(errno)));
+        }
+    }
+
     int GetLinger() const override
     {
         int value = 0;
@@ -506,8 +499,14 @@ class Socket final : public fair::mq::Socket
         if (constant == "snd-more no-block") return ZMQ_DONTWAIT|ZMQ_SNDMORE;
 
         if (constant == "fd") return ZMQ_FD;
+        if (constant == "events")
+            return ZMQ_EVENTS;
+        if (constant == "pollin")
+            return ZMQ_POLLIN;
+        if (constant == "pollout")
+            return ZMQ_POLLOUT;
 
-        return -1;
+        throw SocketError(tools::ToString("GetConstant called with an invalid argument: ", constant));
     }
 
     ~Socket() override { Close(); }
@@ -521,8 +520,7 @@ class Socket final : public fair::mq::Socket
     std::atomic<unsigned long> fMessagesTx;
     std::atomic<unsigned long> fMessagesRx;
 
-    int fSndTimeout;
-    int fRcvTimeout;
+    int fTimeout;
 };
 
 }

fairmq/shmem/TransportFactory.h

@@ -43,7 +43,7 @@ class TransportFactory final : public fair::mq::TransportFactory
         : fair::mq::TransportFactory(id)
         , fDeviceId(id)
         , fShmId()
-        , fZMQContext(zmq_ctx_new())
+        , fZmqCtx(zmq_ctx_new())
         , fManager(nullptr)
     {
         int major, minor, patch;
@@ -51,42 +51,42 @@ class TransportFactory final : public fair::mq::TransportFactory
         LOG(debug) << "Transport: Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
                 << "boost::interprocess (" << (BOOST_VERSION / 100000) << "." << (BOOST_VERSION / 100 % 1000) << "." << (BOOST_VERSION % 100) << ")";
 
-        if (!fZMQContext) {
+        if (!fZmqCtx) {
             throw std::runtime_error(tools::ToString("failed creating context, reason: ", zmq_strerror(errno)));
         }
 
         int numIoThreads = 1;
         std::string sessionName = "default";
-        size_t segmentSize = 2000000000;
-        bool autolaunchMonitor = false;
-        bool throwOnBadAlloc = true;
+        size_t segmentSize = 2ULL << 30;
+        std::string allocationAlgorithm("rbtree_best_fit");
         if (config) {
             numIoThreads = config->GetProperty<int>("io-threads", numIoThreads);
             sessionName = config->GetProperty<std::string>("session", sessionName);
             segmentSize = config->GetProperty<size_t>("shm-segment-size", segmentSize);
-            autolaunchMonitor = config->GetProperty<bool>("shm-monitor", autolaunchMonitor);
-            throwOnBadAlloc = config->GetProperty<bool>("shm-throw-bad-alloc", throwOnBadAlloc);
+            allocationAlgorithm = config->GetProperty<std::string>("shm-allocation", allocationAlgorithm);
         } else {
             LOG(debug) << "ProgOptions not available! Using defaults.";
         }
 
+        if (allocationAlgorithm != "rbtree_best_fit" && allocationAlgorithm != "simple_seq_fit") {
+            LOG(error) << "Provided shared memory allocation algorithm '" << allocationAlgorithm << "' is not supported. Supported are 'rbtree_best_fit'/'simple_seq_fit'";
+            throw SharedMemoryError(tools::ToString("Provided shared memory allocation algorithm '", allocationAlgorithm, "' is not supported. Supported are 'rbtree_best_fit'/'simple_seq_fit'"));
+        }
+
         fShmId = buildShmIdFromSessionIdAndUserId(sessionName);
+        LOG(debug) << "Generated shmid '" << fShmId << "' out of session id '" << sessionName << "'.";
 
         try {
-            if (zmq_ctx_set(fZMQContext, ZMQ_IO_THREADS, numIoThreads) != 0) {
+            if (zmq_ctx_set(fZmqCtx, ZMQ_IO_THREADS, numIoThreads) != 0) {
                 LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
             }
 
             // Set the maximum number of allowed sockets on the context.
-            if (zmq_ctx_set(fZMQContext, ZMQ_MAX_SOCKETS, 10000) != 0) {
+            if (zmq_ctx_set(fZmqCtx, ZMQ_MAX_SOCKETS, 10000) != 0) {
                 LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
             }
 
-            if (autolaunchMonitor) {
-                Manager::StartMonitor(fShmId);
-            }
-
-            fManager = tools::make_unique<Manager>(fShmId, fDeviceId, segmentSize, throwOnBadAlloc);
+            fManager = tools::make_unique<Manager>(fShmId, fDeviceId, segmentSize, config);
         } catch (boost::interprocess::interprocess_exception& e) {
             LOG(error) << "Could not initialize shared memory transport: " << e.what();
             throw std::runtime_error(tools::ToString("Could not initialize shared memory transport: ", e.what()));
@@ -128,7 +128,7 @@ class TransportFactory final : public fair::mq::TransportFactory
 
     SocketPtr CreateSocket(const std::string& type, const std::string& name) override
     {
-        return tools::make_unique<Socket>(*fManager, type, name, GetId(), fZMQContext, this);
+        return tools::make_unique<Socket>(*fManager, type, name, GetId(), fZmqCtx, this);
     }
 
     PollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override
@@ -186,14 +186,17 @@ class TransportFactory final : public fair::mq::TransportFactory
     {
         LOG(debug) << "Destroying Shared Memory transport...";
 
-        if (fZMQContext) {
-            if (zmq_ctx_term(fZMQContext) != 0) {
-                if (errno == EINTR) {
-                    LOG(error) << "failed closing context, reason: " << zmq_strerror(errno);
-                } else {
-                    fZMQContext = nullptr;
-                    return;
+        if (fZmqCtx) {
+            while (true) {
+                if (zmq_ctx_term(fZmqCtx) != 0) {
+                    if (errno == EINTR) {
+                        LOG(debug) << "zmq_ctx_term interrupted by system call, retrying";
+                        continue;
+                    } else {
+                        fZmqCtx = nullptr;
+                    }
                 }
+                break;
             }
         } else {
             LOG(error) << "context not available for shutdown";
@@ -203,7 +206,7 @@ class TransportFactory final : public fair::mq::TransportFactory
   private:
     std::string fDeviceId;
     std::string fShmId;
-    void* fZMQContext;
+    void* fZmqCtx;
     std::unique_ptr<Manager> fManager;
 };
 

fairmq/shmem/UnmanagedRegion.h

@@ -56,14 +56,16 @@ class UnmanagedRegion final : public fair::mq::UnmanagedRegion
 
     void* GetData() const override { return fRegion->get_address(); }
     size_t GetSize() const override { return fRegion->get_size(); }
-    uint64_t GetId() const override { return fRegionId; }
+    uint16_t GetId() const override { return fRegionId; }
+    void SetLinger(uint32_t linger) override { fManager.GetRegion(fRegionId)->SetLinger(linger); }
+    uint32_t GetLinger() const override { return fManager.GetRegion(fRegionId)->GetLinger(); }
 
     ~UnmanagedRegion() override { fManager.RemoveRegion(fRegionId); }
 
   private:
     Manager& fManager;
     boost::interprocess::mapped_region* fRegion;
-    uint64_t fRegionId;
+    uint16_t fRegionId;
 };
 
 }

fairmq/shmem/runMonitor.cxx

@@ -76,7 +76,9 @@ int main(int argc, char** argv)
         bool interactive = false;
         bool viewOnly = false;
         unsigned int timeoutInMS = 5000;
+        unsigned int intervalInMS = 100;
         bool runAsDaemon = false;
+        bool debug = false;
         bool cleanOnExit = false;
 
         options_description desc("Options");
@@ -89,7 +91,9 @@ int main(int argc, char** argv)
             ("view,v"         , value<bool>(&viewOnly)->implicit_value(true),           "Run in view only mode")
             ("timeout,t"      , value<unsigned int>(&timeoutInMS)->default_value(5000), "Heartbeat timeout in milliseconds")
             ("daemonize,d"    , value<bool>(&runAsDaemon)->implicit_value(true),        "Daemonize the monitor")
+            ("debug,b"        , value<bool>(&debug)->implicit_value(true),             "Debug - Print a list of messages)")
             ("clean-on-exit,e", value<bool>(&cleanOnExit)->implicit_value(true),        "Perform cleanup on exit")
+            ("interval"       , value<unsigned int>(&intervalInMS)->default_value(100), "Output interval for interactive/view-only mode")
             ("help,h", "Print help");
 
         variables_map vm;
@@ -111,15 +115,21 @@ int main(int argc, char** argv)
         }
 
         if (cleanup) {
-            cout << "Cleaning up \"" << shmId << "\"..." << endl;
-            Monitor::Cleanup(shmId);
-            Monitor::RemoveQueue("fmq_" + shmId + "_cq");
+            Monitor::CleanupFull(ShmId{shmId});
+            return 0;
+        }
+
+        if (debug) {
+            Monitor::PrintDebugInfo(ShmId{shmId});
             return 0;
         }
 
         cout << "Starting shared memory monitor for session: \"" << sessionName << "\" (shmId: " << shmId << ")..." << endl;
+        if (viewOnly && !interactive) {
+            cout << "running in non-interactive view-only mode, outputting with interval of " << intervalInMS << "ms. (change with --interval), press ctrl+C to exit." << endl;
+        }
 
-        Monitor monitor(shmId, selfDestruct, interactive, viewOnly, timeoutInMS, runAsDaemon, cleanOnExit);
+        Monitor monitor(shmId, selfDestruct, interactive, viewOnly, timeoutInMS, intervalInMS, runAsDaemon, cleanOnExit);
 
         monitor.CatchSignals();
         monitor.Run();

fairmq/zeromq/Context.h

@@ -56,7 +56,7 @@ class Context
             throw ContextError(tools::ToString("failed configuring context, reason: ", zmq_strerror(errno)));
         }
 
-        fRegionEvents.emplace(0, nullptr, 0, 0, RegionEvent::local_only);
+        fRegionEvents.emplace(true, 0, nullptr, 0, 0, RegionEvent::local_only);
     }
 
     Context(const Context&) = delete;
@@ -114,24 +114,24 @@ class Context
         return fRegionInfos;
     }
 
-    uint64_t RegionCount() const
+    uint16_t RegionCount() const
     {
         std::lock_guard<std::mutex> lock(fMtx);
         return fRegionCounter;
     }
 
-    void AddRegion(uint64_t id, void* ptr, size_t size, int64_t userFlags, RegionEvent event)
+    void AddRegion(bool managed, uint16_t id, void* ptr, size_t size, int64_t userFlags, RegionEvent event)
     {
         {
             std::lock_guard<std::mutex> lock(fMtx);
             ++fRegionCounter;
-            fRegionInfos.emplace_back(id, ptr, size, userFlags, event);
-            fRegionEvents.emplace(id, ptr, size, userFlags, event);
+            fRegionInfos.emplace_back(managed, id, ptr, size, userFlags, event);
+            fRegionEvents.emplace(managed, id, ptr, size, userFlags, event);
         }
         fRegionEventsCV.notify_one();
     }
 
-    void RemoveRegion(uint64_t id)
+    void RemoveRegion(uint16_t id)
     {
         {
             std::lock_guard<std::mutex> lock(fMtx);
@@ -161,13 +161,16 @@ class Context
         UnsubscribeFromRegionEvents();
 
         if (fZmqCtx) {
-            if (zmq_ctx_term(fZmqCtx) != 0) {
-                if (errno == EINTR) {
-                    LOG(error) << " failed closing context, reason: " << zmq_strerror(errno);
-                } else {
-                    fZmqCtx = nullptr;
-                    return;
+            while (true) {
+                if (zmq_ctx_term(fZmqCtx) != 0) {
+                    if (errno == EINTR) {
+                        LOG(debug) << "zmq_ctx_term interrupted by system call, retrying";
+                        continue;
+                    } else {
+                        fZmqCtx = nullptr;
+                    }
                 }
+                break;
             }
         } else {
             LOG(error) << "context not available for shutdown";
@@ -179,7 +182,7 @@ class Context
     mutable std::mutex fMtx;
     std::atomic<bool> fInterrupted;
 
-    uint64_t fRegionCounter;
+    uint16_t fRegionCounter;
     std::condition_variable fRegionEventsCV;
     std::vector<RegionInfo> fRegionInfos;
     std::queue<RegionInfo> fRegionEvents;

fairmq/zeromq/Poller.h

@@ -130,13 +130,20 @@ class Poller final : public fair::mq::Poller
 
     void Poll(const int timeout) override
     {
-        if (zmq_poll(fItems, fNumItems, timeout) < 0) {
-            if (errno == ETERM) {
-                LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
-            } else {
-                LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
-                throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+        while (true) {
+            if (zmq_poll(fItems, fNumItems, timeout) < 0) {
+                if (errno == ETERM) {
+                    LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
+                    return;
+                } else if (errno == EINTR) {
+                    LOG(debug) << "polling interrupted by system call";
+                    continue;
+                } else {
+                    LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
+                    throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+                }
             }
+            break;
         }
     }
 

fairmq/zeromq/Socket.h

@@ -12,13 +12,15 @@
 #include <FairMQLogger.h>
 #include <FairMQMessage.h>
 #include <FairMQSocket.h>
-#include <atomic>
 #include <fairmq/Tools.h>
 #include <fairmq/zeromq/Context.h>
 #include <fairmq/zeromq/Message.h>
-#include <memory>   // unique_ptr
+
 #include <zmq.h>
 
+#include <atomic>
+#include <memory> // unique_ptr
+
 namespace fair {
 namespace mq {
 namespace zmq {
@@ -35,8 +37,7 @@ class Socket final : public fair::mq::Socket
         , fBytesRx(0)
         , fMessagesTx(0)
         , fMessagesRx(0)
-        , fSndTimeout(100)
-        , fRcvTimeout(100)
+        , fTimeout(100)
     {
         if (fSocket == nullptr) {
             LOG(error) << "Failed creating socket " << fId << ", reason: " << zmq_strerror(errno);
@@ -54,11 +55,11 @@ class Socket final : public fair::mq::Socket
             LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
         }
 
-        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0) {
+        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fTimeout, sizeof(fTimeout)) != 0) {
             LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
         }
 
-        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0) {
+        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fTimeout, sizeof(fTimeout)) != 0) {
             LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
         }
 
@@ -78,13 +79,12 @@ class Socket final : public fair::mq::Socket
 
     bool Bind(const std::string& address) override
     {
-        // LOG(info) << "bind socket " << fId << " on " << address;
+        // LOG(debug) << "Binding socket " << fId << " on " << address;
 
         if (zmq_bind(fSocket, address.c_str()) != 0) {
             if (errno == EADDRINUSE) {
                 // do not print error in this case, this is handled by FairMQDevice in case no
-                // connection could be established after trying a number of random ports from a
-                // range.
+                // connection could be established after trying a number of random ports from a range.
                 return false;
             }
             LOG(error) << "Failed binding socket " << fId << ", address: " << address << ", reason: " << zmq_strerror(errno);
@@ -96,7 +96,7 @@ class Socket final : public fair::mq::Socket
 
     bool Connect(const std::string& address) override
     {
-        // LOG(info) << "connect socket " << fId << " on " << address;
+        // LOG(debug) << "Connecting socket " << fId << " on " << address;
 
         if (zmq_connect(fSocket, address.c_str()) != 0) {
             LOG(error) << "Failed connecting socket " << fId << ", address: " << address << ", reason: " << zmq_strerror(errno);
@@ -106,6 +106,32 @@ class Socket final : public fair::mq::Socket
         return true;
     }
 
+    bool ShouldRetry(int flags, int timeout, int& elapsed) const
+    {
+        if ((flags & ZMQ_DONTWAIT) == 0) {
+            if (timeout > 0) {
+                elapsed += fTimeout;
+                if (elapsed >= timeout) {
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    int HandleErrors() const
+    {
+        if (zmq_errno() == ETERM) {
+            LOG(debug) << "Terminating socket " << fId;
+            return static_cast<int>(TransferResult::error);
+        } else {
+            LOG(error) << "Failed transfer on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno);
+            return static_cast<int>(TransferResult::error);
+        }
+    }
+
     int Send(MessagePtr& msg, const int timeout = -1) override
     {
         int flags = 0;
@@ -121,29 +147,17 @@ class Socket final : public fair::mq::Socket
             if (nbytes >= 0) {
                 fBytesTx += nbytes;
                 ++fMessagesTx;
-
                 return nbytes;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fSndTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fCtx.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Send interrupted by system call";
-                return nbytes;
             } else {
-                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-                return nbytes;
+                return HandleErrors();
             }
         }
     }
@@ -162,27 +176,16 @@ class Socket final : public fair::mq::Socket
                 fBytesRx += nbytes;
                 ++fMessagesRx;
                 return nbytes;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fRcvTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
+            } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                if (fCtx.Interrupted()) {
+                    return static_cast<int>(TransferResult::interrupted);
+                } else if (ShouldRetry(flags, timeout, elapsed)) {
                     continue;
                 } else {
-                    return -2;
+                    return static_cast<int>(TransferResult::timeout);
                 }
-            } else if (zmq_errno() == ETERM) {
-                LOG(info) << "terminating socket " << fId;
-                return -1;
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Receive interrupted by system call";
-                return nbytes;
             } else {
-                LOG(error) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
-                return nbytes;
+                return HandleErrors();
             }
         }
     }
@@ -210,32 +213,17 @@ class Socket final : public fair::mq::Socket
                     int nbytes = zmq_msg_send(static_cast<Message*>(msgVec[i].get())->GetMessage(), fSocket, (i < vecSize - 1) ? ZMQ_SNDMORE | flags : flags);
                     if (nbytes >= 0) {
                         totalSize += nbytes;
-                    } else {
-                        // according to ZMQ docs, this can only occur for the first part
-                        if (zmq_errno() == EAGAIN) {
-                            if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                                if (timeout > 0) {
-                                    elapsed += fSndTimeout;
-                                    if (elapsed >= timeout) {
-                                        return -2;
-                                    }
-                                }
-                                repeat = true;
-                                break;
-                            } else {
-                                return -2;
-                            }
-                        }
-                        if (zmq_errno() == ETERM) {
-                            LOG(info) << "terminating socket " << fId;
-                            return -1;
-                        } else if (zmq_errno() == EINTR) {
-                            LOG(debug) << "Receive interrupted by system call";
-                            return nbytes;
+                    } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                        if (fCtx.Interrupted()) {
+                            return static_cast<int>(TransferResult::interrupted);
+                        } else if (ShouldRetry(flags, timeout, elapsed)) {
+                            repeat = true;
+                            break;
                         } else {
-                            LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-                            return nbytes;
+                            return static_cast<int>(TransferResult::timeout);
                         }
+                    } else {
+                        return HandleErrors();
                     }
                 }
 
@@ -243,18 +231,16 @@ class Socket final : public fair::mq::Socket
                     continue;
                 }
 
-                // store statistics on how many messages have been sent (handle all parts as a
-                // single message)
+                // store statistics on how many messages have been sent (handle all parts as a single message)
                 ++fMessagesTx;
                 fBytesTx += totalSize;
                 return totalSize;
             }
-        }   // If there's only one part, send it as a regular message
-        else if (vecSize == 1) {
+        } else if (vecSize == 1) { // If there's only one part, send it as a regular message
             return Send(msgVec.back(), timeout);
-        } else {   // if the vector is empty, something might be wrong
+        } else { // if the vector is empty, something might be wrong
             LOG(warn) << "Will not send empty vector";
-            return -1;
+            return static_cast<int>(TransferResult::error);
         }
     }
 
@@ -278,24 +264,17 @@ class Socket final : public fair::mq::Socket
                 if (nbytes >= 0) {
                     msgVec.push_back(move(part));
                     totalSize += nbytes;
-                } else if (zmq_errno() == EAGAIN) {
-                    if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                        if (timeout > 0) {
-                            elapsed += fRcvTimeout;
-                            if (elapsed >= timeout) {
-                                return -2;
-                            }
-                        }
+                } else if (zmq_errno() == EAGAIN || zmq_errno() == EINTR) {
+                    if (fCtx.Interrupted()) {
+                        return static_cast<int>(TransferResult::interrupted);
+                    } else if (ShouldRetry(flags, timeout, elapsed)) {
                         repeat = true;
                         break;
                     } else {
-                        return -2;
+                        return static_cast<int>(TransferResult::timeout);
                     }
-                } else if (zmq_errno() == EINTR) {
-                    LOG(debug) << "Receive interrupted by system call";
-                    return nbytes;
                 } else {
-                    return nbytes;
+                    return HandleErrors();
                 }
 
                 size_t moreSize = sizeof(more);
@@ -306,8 +285,7 @@ class Socket final : public fair::mq::Socket
                 continue;
             }
 
-            // store statistics on how many messages have been received (handle all parts as a
-            // single message)
+            // store statistics on how many messages have been received (handle all parts as a single message)
             ++fMessagesRx;
             fBytesRx += totalSize;
             return totalSize;
@@ -345,6 +323,15 @@ class Socket final : public fair::mq::Socket
         }
     }
 
+    void Events(uint32_t* events) override
+    {
+        size_t eventsSize = sizeof(uint32_t);
+        if (zmq_getsockopt(fSocket, ZMQ_EVENTS, events, &eventsSize) < 0) {
+            throw SocketError(
+                tools::ToString("failed setting ZMQ_EVENTS, reason: ", zmq_strerror(errno)));
+        }
+    }
+
     void SetLinger(const int value) override
     {
         if (zmq_setsockopt(fSocket, ZMQ_LINGER, &value, sizeof(value)) < 0) {
@@ -460,8 +447,14 @@ class Socket final : public fair::mq::Socket
         if (constant == "linger") return ZMQ_LINGER;
 
         if (constant == "fd") return ZMQ_FD;
+        if (constant == "events")
+            return ZMQ_EVENTS;
+        if (constant == "pollin")
+            return ZMQ_POLLIN;
+        if (constant == "pollout")
+            return ZMQ_POLLOUT;
 
-        return -1;
+        throw SocketError(tools::ToString("GetConstant called with an invalid argument: ", constant));
     }
 
     ~Socket() override { Close(); }
@@ -475,8 +468,7 @@ class Socket final : public fair::mq::Socket
     std::atomic<unsigned long> fMessagesTx;
     std::atomic<unsigned long> fMessagesRx;
 
-    int fSndTimeout;
-    int fRcvTimeout;
+    int fTimeout;
 };
 
 }   // namespace zmq

fairmq/zeromq/TransportFactory.h

@@ -55,7 +55,7 @@ class TransportFactory final : public FairMQTransportFactory
     {
         return tools::make_unique<Message>(this);
     }
-    
+
     MessagePtr CreateMessage(Alignment alignment) override
     {
         return tools::make_unique<Message>(alignment, this);
@@ -125,7 +125,7 @@ class TransportFactory final : public FairMQTransportFactory
     {
         UnmanagedRegionPtr ptr = tools::make_unique<UnmanagedRegion>(*fCtx, size, userFlags, callback, bulkCallback, this);
         auto zPtr = static_cast<UnmanagedRegion*>(ptr.get());
-        fCtx->AddRegion(zPtr->GetId(), zPtr->GetData(), zPtr->GetSize(), zPtr->GetUserFlags(), RegionEvent::created);
+        fCtx->AddRegion(false, zPtr->GetId(), zPtr->GetData(), zPtr->GetSize(), zPtr->GetUserFlags(), RegionEvent::created);
         return ptr;
     }
 

fairmq/zeromq/UnmanagedRegion.h

@@ -50,8 +50,10 @@ class UnmanagedRegion final : public fair::mq::UnmanagedRegion
 
     virtual void* GetData() const override { return fBuffer; }
     virtual size_t GetSize() const override { return fSize; }
-    uint64_t GetId() const override { return fId; }
+    uint16_t GetId() const override { return fId; }
     int64_t GetUserFlags() const { return fUserFlags; }
+    void SetLinger(uint32_t /* linger */) override { LOG(debug) << "ZeroMQ UnmanagedRegion linger option not implemented. Acknowledgements are local."; }
+    uint32_t GetLinger() const override { LOG(debug) << "ZeroMQ UnmanagedRegion linger option not implemented. Acknowledgements are local."; return 0; }
 
     virtual ~UnmanagedRegion()
     {
@@ -62,7 +64,7 @@ class UnmanagedRegion final : public fair::mq::UnmanagedRegion
 
   private:
     Context& fCtx;
-    uint64_t fId;
+    uint16_t fId;
     void* fBuffer;
     size_t fSize;
     int64_t fUserFlags;

test/CMakeLists.txt

@@ -12,6 +12,18 @@ include(GTestHelper)
 # FairMQ Testsuites/helpers #
 #############################
 
+if(FairLogger_VERSION VERSION_LESS 1.9.0 AND FairLogger_VERSION VERSION_GREATER_EQUAL 1.7.0)
+  LIST(APPEND definitions FAIR_MIN_SEVERITY=trace)
+endif()
+
+if(BUILD_OFI_TRANSPORT)
+  LIST(APPEND definitions BUILD_OFI_TRANSPORT)
+endif()
+
+if(definitions)
+  set(definitions DEFINITIONS ${definitions})
+endif()
+
 add_testhelper(runTestDevice
     SOURCES
     helper/runTestDevice.cxx
@@ -30,16 +42,9 @@ add_testhelper(runTestDevice
     helper/devices/TestExceptions.h
 
     LINKS FairMQ
+    ${definitions}
 )
 
-if(BUILD_OFI_TRANSPORT)
-  LIST(APPEND definitions BUILD_OFI_TRANSPORT)
-endif()
-
-if(definitions)
-  set(definitions DEFINITIONS ${definitions})
-endif()
-
 set(MQ_CONFIG "${CMAKE_BINARY_DIR}/test/testsuite_FairMQ.IOPatterns_config.json")
 set(RUN_TEST_DEVICE "${CMAKE_BINARY_DIR}/test/testhelper_runTestDevice")
 set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)

test/commands/_commands.cxx

@@ -31,7 +31,7 @@ TEST(Format, Construction)
     Cmds setPropertiesCmds(make<SetProperties>(42, props));
     Cmds subscriptionHeartbeatCmds(make<SubscriptionHeartbeat>(60000));
     Cmds currentStateCmds(make<CurrentState>("somedeviceid", State::Running));
-    Cmds transitionStatusCmds(make<TransitionStatus>("somedeviceid", 123456, Result::Ok, Transition::Stop));
+    Cmds transitionStatusCmds(make<TransitionStatus>("somedeviceid", 123456, Result::Ok, Transition::Stop, State::Running));
     Cmds configCmds(make<Config>("somedeviceid", "someconfig"));
     Cmds stateChangeSubscriptionCmds(make<StateChangeSubscription>("somedeviceid", 123456, Result::Ok));
     Cmds stateChangeUnsubscriptionCmds(make<StateChangeUnsubscription>("somedeviceid", 123456, Result::Ok));
@@ -63,6 +63,7 @@ TEST(Format, Construction)
     ASSERT_EQ(static_cast<TransitionStatus&>(transitionStatusCmds.At(0)).GetTaskId(), 123456);
     ASSERT_EQ(static_cast<TransitionStatus&>(transitionStatusCmds.At(0)).GetResult(), Result::Ok);
     ASSERT_EQ(static_cast<TransitionStatus&>(transitionStatusCmds.At(0)).GetTransition(), Transition::Stop);
+    ASSERT_EQ(static_cast<TransitionStatus&>(transitionStatusCmds.At(0)).GetCurrentState(), State::Running);
     ASSERT_EQ(configCmds.At(0).GetType(), Type::config);
     ASSERT_EQ(static_cast<Config&>(configCmds.At(0)).GetDeviceId(), "somedeviceid");
     ASSERT_EQ(static_cast<Config&>(configCmds.At(0)).GetConfig(), "someconfig");
@@ -104,7 +105,7 @@ void fillCommands(Cmds& cmds)
     cmds.Add<SetProperties>(42, props);
     cmds.Add<SubscriptionHeartbeat>(60000);
     cmds.Add<CurrentState>("somedeviceid", State::Running);
-    cmds.Add<TransitionStatus>("somedeviceid", 123456, Result::Ok, Transition::Stop);
+    cmds.Add<TransitionStatus>("somedeviceid", 123456, Result::Ok, Transition::Stop, State::Running);
     cmds.Add<Config>("somedeviceid", "someconfig");
     cmds.Add<StateChangeSubscription>("somedeviceid", 123456, Result::Ok);
     cmds.Add<StateChangeUnsubscription>("somedeviceid", 123456, Result::Ok);
@@ -167,6 +168,7 @@ void checkCommands(Cmds& cmds)
                 ASSERT_EQ(static_cast<TransitionStatus&>(*cmd).GetTaskId(), 123456);
                 ASSERT_EQ(static_cast<TransitionStatus&>(*cmd).GetResult(), Result::Ok);
                 ASSERT_EQ(static_cast<TransitionStatus&>(*cmd).GetTransition(), Transition::Stop);
+                ASSERT_EQ(static_cast<TransitionStatus&>(*cmd).GetCurrentState(), State::Running);
             break;
             case Type::config:
                 ++count;

test/helper/devices/TestTransferTimeout.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2015-2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2015static_cast<int>(TransferResult::timeout017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH ) *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -24,20 +24,20 @@ class TransferTimeout : public FairMQDevice
   protected:
     auto Run() -> void override
     {
-        bool sendMsgCancelingAfter100ms = false;
-        bool receiveMsgCancelingAfter100ms = false;
+        bool sendMsgCancelingAfter200ms = false;
+        bool receiveMsgCancelingAfter200ms = false;
 
         bool sendMsgCancelingAfter0ms = false;
         bool receiveMsgCancelingAfter0ms = false;
 
-        bool send1PartCancelingAfter100ms = false;
-        bool receive1PartCancelingAfter100ms = false;
+        bool send1PartCancelingAfter200ms = false;
+        bool receive1PartCancelingAfter200ms = false;
 
         bool send1PartCancelingAfter0ms = false;
         bool receive1PartCancelingAfter0ms = false;
 
-        bool send2PartsCancelingAfter100ms = false;
-        bool receive2PartsCancelingAfter100ms = false;
+        bool send2PartsCancelingAfter200ms = false;
+        bool receive2PartsCancelingAfter200ms = false;
 
         bool send2PartsCancelingAfter0ms = false;
         bool receive2PartsCancelingAfter0ms = false;
@@ -45,28 +45,28 @@ class TransferTimeout : public FairMQDevice
         FairMQMessagePtr msg1(NewMessage());
         FairMQMessagePtr msg2(NewMessage());
 
-        if (Send(msg1, "data-out", 0, 100) == -2) {
-            LOG(info) << "send msg canceled (100ms)";
-            sendMsgCancelingAfter100ms = true;
+        if (Send(msg1, "data-out", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "send msg canceled (200ms)";
+            sendMsgCancelingAfter200ms = true;
         } else {
-            LOG(error) << "send msg did not cancel (100ms)";
+            LOG(error) << "send msg did not cancel (200ms)";
         }
 
-        if (Receive(msg2, "data-in", 0, 100) == -2) {
-            LOG(info) << "receive msg canceled (100ms)";
-            receiveMsgCancelingAfter100ms = true;
+        if (Receive(msg2, "data-in", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "receive msg canceled (200ms)";
+            receiveMsgCancelingAfter200ms = true;
         } else {
-            LOG(error) << "receive msg did not cancel (100ms)";
+            LOG(error) << "receive msg did not cancel (200ms)";
         }
 
-        if (Send(msg1, "data-out", 0, 0) == -2) {
+        if (Send(msg1, "data-out", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "send msg canceled (0ms)";
             sendMsgCancelingAfter0ms = true;
         } else {
             LOG(error) << "send msg did not cancel (0ms)";
         }
 
-        if (Receive(msg2, "data-in", 0, 0) == -2) {
+        if (Receive(msg2, "data-in", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "receive msg canceled (0ms)";
             receiveMsgCancelingAfter0ms = true;
         } else {
@@ -77,28 +77,28 @@ class TransferTimeout : public FairMQDevice
         parts1.AddPart(NewMessage(10));
         FairMQParts parts2;
 
-        if (Send(parts1, "data-out", 0, 100) == -2) {
-            LOG(info) << "send 1 part canceled (100ms)";
-            send1PartCancelingAfter100ms = true;
+        if (Send(parts1, "data-out", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "send 1 part canceled (200ms)";
+            send1PartCancelingAfter200ms = true;
         } else {
-            LOG(error) << "send 1 part did not cancel (100ms)";
+            LOG(error) << "send 1 part did not cancel (200ms)";
         }
 
-        if (Receive(parts2, "data-in", 0, 100) == -2) {
-            LOG(info) << "receive 1 part canceled (100ms)";
-            receive1PartCancelingAfter100ms = true;
+        if (Receive(parts2, "data-in", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "receive 1 part canceled (200ms)";
+            receive1PartCancelingAfter200ms = true;
         } else {
-            LOG(error) << "receive 1 part did not cancel (100ms)";
+            LOG(error) << "receive 1 part did not cancel (200ms)";
         }
 
-        if (Send(parts1, "data-out", 0, 0) == -2) {
+        if (Send(parts1, "data-out", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "send 1 part canceled (0ms)";
             send1PartCancelingAfter0ms = true;
         } else {
             LOG(error) << "send 1 part did not cancel (0ms)";
         }
 
-        if (Receive(parts2, "data-in", 0, 0) == -2) {
+        if (Receive(parts2, "data-in", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "receive 1 part canceled (0ms)";
             receive1PartCancelingAfter0ms = true;
         } else {
@@ -110,44 +110,44 @@ class TransferTimeout : public FairMQDevice
         parts3.AddPart(NewMessage(10));
         FairMQParts parts4;
 
-        if (Send(parts3, "data-out", 0, 100) == -2) {
-            LOG(info) << "send 2 parts canceled (100ms)";
-            send2PartsCancelingAfter100ms = true;
+        if (Send(parts3, "data-out", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "send 2 parts canceled (200ms)";
+            send2PartsCancelingAfter200ms = true;
         } else {
-            LOG(error) << "send 2 parts did not cancel (100ms)";
+            LOG(error) << "send 2 parts did not cancel (200ms)";
         }
 
-        if (Receive(parts4, "data-in", 0, 100) == -2) {
-            LOG(info) << "receive 2 parts canceled (100ms)";
-            receive2PartsCancelingAfter100ms = true;
+        if (Receive(parts4, "data-in", 0, 200) == static_cast<int>(TransferResult::timeout)) {
+            LOG(info) << "receive 2 parts canceled (200ms)";
+            receive2PartsCancelingAfter200ms = true;
         } else {
-            LOG(error) << "receive 2 parts did not cancel (100ms)";
+            LOG(error) << "receive 2 parts did not cancel (200ms)";
         }
 
-        if (Send(parts3, "data-out", 0, 0) == -2) {
+        if (Send(parts3, "data-out", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "send 2 parts canceled (0ms)";
             send2PartsCancelingAfter0ms = true;
         } else {
             LOG(error) << "send 2 parts did not cancel (0ms)";
         }
 
-        if (Receive(parts4, "data-in", 0, 0) == -2) {
+        if (Receive(parts4, "data-in", 0, 0) == static_cast<int>(TransferResult::timeout)) {
             LOG(info) << "receive 2 parts canceled (0ms)";
             receive2PartsCancelingAfter0ms = true;
         } else {
             LOG(error) << "receive 2 parts did not cancel (0ms)";
         }
 
-        if (sendMsgCancelingAfter100ms &&
-            receiveMsgCancelingAfter100ms &&
+        if (sendMsgCancelingAfter200ms &&
+            receiveMsgCancelingAfter200ms &&
             sendMsgCancelingAfter0ms &&
             receiveMsgCancelingAfter0ms &&
-            send1PartCancelingAfter100ms &&
-            receive1PartCancelingAfter100ms &&
+            send1PartCancelingAfter200ms &&
+            receive1PartCancelingAfter200ms &&
             send1PartCancelingAfter0ms &&
             receive1PartCancelingAfter0ms &&
-            send2PartsCancelingAfter100ms &&
-            receive2PartsCancelingAfter100ms &&
+            send2PartsCancelingAfter200ms &&
+            receive2PartsCancelingAfter200ms &&
             send2PartsCancelingAfter0ms &&
             receive2PartsCancelingAfter0ms)
         {

test/memory_resources/_memory_resources.cxx

@@ -6,66 +6,83 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <cstring>
 #include <fairmq/FairMQTransportFactory.h>
 #include <fairmq/MemoryResourceTools.h>
+#include <fairmq/ProgOptions.h>
+#include <fairmq/Tools.h>
+
+#include <boost/container/pmr/polymorphic_allocator.hpp>
+
 #include <gtest/gtest.h>
+
+#include <cstring>
 #include <vector>
 
-namespace {
+namespace
+{
 
 using namespace std;
 using namespace fair::mq;
-using factoryType = std::shared_ptr<FairMQTransportFactory>;
 
-factoryType factoryZMQ = FairMQTransportFactory::CreateTransportFactory("zeromq");
-factoryType factorySHM = FairMQTransportFactory::CreateTransportFactory("shmem");
+using FactoryType = shared_ptr<FairMQTransportFactory>;
 
-struct testData
+struct TestData
 {
     int i{1};
     static int nallocated;
     static int nallocations;
     static int ndeallocations;
-    testData()
+
+    TestData()
     {
         ++nallocated;
         ++nallocations;
     }
-    testData(const testData& in)
+
+    TestData(const TestData& in)
         : i{in.i}
     {
         ++nallocated;
         ++nallocations;
     }
-    testData(const testData&& in)
+
+    TestData(const TestData&& in)
         : i{in.i}
     {
         ++nallocated;
         ++nallocations;
     }
-    testData(int in)
+
+    TestData(int in)
         : i{in}
     {
         ++nallocated;
         ++nallocations;
     }
-    ~testData()
+
+    ~TestData()
     {
         --nallocated;
         ++ndeallocations;
     }
 };
 
-int testData::nallocated = 0;
-int testData::nallocations = 0;
-int testData::ndeallocations = 0;
+int TestData::nallocated = 0;
+int TestData::nallocations = 0;
+int TestData::ndeallocations = 0;
 
-auto allocZMQ = factoryZMQ -> GetMemoryResource();
-auto allocSHM = factorySHM -> GetMemoryResource();
-
-TEST(MemoryResources, transportallocatormap)
+TEST(MemoryResources, transportAllocatorMap)
 {
+    size_t session{tools::UuidHash()};
+    ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    FactoryType factoryZMQ = FairMQTransportFactory::CreateTransportFactory("zeromq", fair::mq::tools::Uuid(), &config);
+    FactoryType factorySHM = FairMQTransportFactory::CreateTransportFactory("shmem", fair::mq::tools::Uuid(), &config);
+
+    auto allocZMQ = factoryZMQ->GetMemoryResource();
+    auto allocSHM = factorySHM->GetMemoryResource();
+
     EXPECT_TRUE(allocZMQ != nullptr && allocSHM != allocZMQ);
     auto _tmp = factoryZMQ->GetMemoryResource();
     EXPECT_TRUE(_tmp == allocZMQ);
@@ -75,29 +92,45 @@ using namespace fair::mq::pmr;
 
 TEST(MemoryResources, allocator)
 {
-    testData::nallocations = 0;
-    testData::ndeallocations = 0;
+    TestData::nallocations = 0;
+    TestData::ndeallocations = 0;
+
+    size_t session{tools::UuidHash()};
+    ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    FactoryType factoryZMQ = FairMQTransportFactory::CreateTransportFactory("zeromq", fair::mq::tools::Uuid(), &config);
+
+    auto allocZMQ = factoryZMQ->GetMemoryResource();
 
     {
-        std::vector<testData, polymorphic_allocator<testData>> v(
-            polymorphic_allocator<testData>{allocZMQ});
+        std::vector<TestData, polymorphic_allocator<TestData>> v(polymorphic_allocator<TestData>{allocZMQ});
         v.reserve(3);
         EXPECT_TRUE(v.capacity() == 3);
         EXPECT_TRUE(allocZMQ->getNumberOfMessages() == 1);
         v.emplace_back(1);
         v.emplace_back(2);
         v.emplace_back(3);
-        EXPECT_TRUE((fair::mq::byte*)&(*v.end()) - (fair::mq::byte*)&(*v.begin()) == 3 * sizeof(testData));
-        EXPECT_TRUE(testData::nallocated == 3);
+        EXPECT_TRUE((fair::mq::byte*)&(*v.end()) - (fair::mq::byte*)&(*v.begin()) == 3 * sizeof(TestData));
+        EXPECT_TRUE(TestData::nallocated == 3);
     }
-    EXPECT_TRUE(testData::nallocated == 0);
-    EXPECT_TRUE(testData::nallocations == testData::ndeallocations);
+    EXPECT_TRUE(TestData::nallocated == 0);
+    EXPECT_TRUE(TestData::nallocations == TestData::ndeallocations);
 }
 
 TEST(MemoryResources, getMessage)
 {
-    testData::nallocations = 0;
-    testData::ndeallocations = 0;
+    TestData::nallocations = 0;
+    TestData::ndeallocations = 0;
+
+    size_t session{tools::UuidHash()};
+    ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    FactoryType factoryZMQ = FairMQTransportFactory::CreateTransportFactory("zeromq", fair::mq::tools::Uuid(), &config);
+    FactoryType factorySHM = FairMQTransportFactory::CreateTransportFactory("shmem", fair::mq::tools::Uuid(), &config);
+
+    auto allocZMQ = factoryZMQ->GetMemoryResource();
 
     FairMQMessagePtr message{nullptr};
 
@@ -105,8 +138,7 @@ TEST(MemoryResources, getMessage)
 
     // test message creation on the same channel it was allocated with
     {
-        std::vector<testData, polymorphic_allocator<testData>> v(
-            polymorphic_allocator<testData>{allocZMQ});
+        std::vector<TestData, polymorphic_allocator<TestData>> v(polymorphic_allocator<TestData>{allocZMQ});
         v.emplace_back(1);
         v.emplace_back(2);
         v.emplace_back(3);
@@ -115,14 +147,13 @@ TEST(MemoryResources, getMessage)
         EXPECT_TRUE(message != nullptr);
         EXPECT_TRUE(message->GetData() == vectorBeginPtr);
     }
-    EXPECT_TRUE(message->GetSize() == 3 * sizeof(testData));
+    EXPECT_TRUE(message->GetSize() == 3 * sizeof(TestData));
     messageArray = static_cast<int*>(message->GetData());
     EXPECT_TRUE(messageArray[0] == 1 && messageArray[1] == 2 && messageArray[2] == 3);
 
     // test message creation on a different channel than it was allocated with
     {
-        std::vector<testData, polymorphic_allocator<testData>> v(
-            polymorphic_allocator<testData>{allocZMQ});
+        std::vector<TestData, polymorphic_allocator<TestData>> v(polymorphic_allocator<TestData>{allocZMQ});
         v.emplace_back(4);
         v.emplace_back(5);
         v.emplace_back(6);
@@ -132,7 +163,7 @@ TEST(MemoryResources, getMessage)
         EXPECT_TRUE(message->GetData() != vectorBeginPtr);
     }
 
-    EXPECT_TRUE(message->GetSize() == 3 * sizeof(testData));
+    EXPECT_TRUE(message->GetSize() == 3 * sizeof(TestData));
     messageArray = static_cast<int*>(message->GetData());
     EXPECT_TRUE(messageArray[0] == 4 && messageArray[1] == 5 && messageArray[2] == 6);
 }

test/protocols/_push_pull_multipart.cxx

@@ -163,9 +163,9 @@ TEST(PushPull, Multipart_ST_ipc_zeromq)
     RunSingleThreadedMultipart("zeromq", "ipc://test_Multipart_ST_ipc_zeromq_1", "ipc://test_Multipart_ST_ipc_zeromq_2");
 }
 
-TEST(PushPull, Multipart_ST_ipc_shmen)
+TEST(PushPull, Multipart_ST_ipc_shmem)
 {
-    RunSingleThreadedMultipart("shmem", "ipc://test_Multipart_ST_ipc_shmen_1", "ipc://test_Multipart_ST_ipc_shmen_2");
+    RunSingleThreadedMultipart("shmem", "ipc://test_Multipart_ST_ipc_shmem_1", "ipc://test_Multipart_ST_ipc_shmem_2");
 }
 
 TEST(PushPull, Multipart_MT_inproc_zeromq)

test/region/_region.cxx

@@ -48,6 +48,7 @@ void RegionEventSubscriptions(const string& transport)
         ASSERT_EQ(factory->SubscribedToRegionEvents(), false);
         factory->SubscribeToRegionEvents([&](FairMQRegionInfo info) {
             LOG(warn) << ">>>" << info.event;
+            LOG(warn) << "managed: " << info.managed;
             LOG(warn) << "id: " << info.id;
             LOG(warn) << "ptr: " << info.ptr;
             LOG(warn) << "size: " << info.size;

test/sdk/_topology.cxx

@@ -139,6 +139,32 @@ TEST_F(Topology, ChangeState)
     EXPECT_EQ(sdk::StateEqualsTo(currentState, sdk::DeviceState::InitializingDevice), true);
 }
 
+TEST_F(Topology, MixedState)
+{
+    using namespace fair::mq;
+
+    sdk::Topology topo(mDDSTopo, mDDSSession);
+    auto result1(topo.ChangeState(sdk::TopologyTransition::InitDevice, "main/Sampler.*"));
+    LOG(info) << result1.first;
+
+    EXPECT_EQ(result1.first, std::error_code());
+    EXPECT_EQ(sdk::AggregateState(result1.second), sdk::AggregatedTopologyState::Mixed);
+    EXPECT_EQ(sdk::StateEqualsTo(result1.second, sdk::DeviceState::InitializingDevice), false);
+    auto const currentState1 = topo.GetCurrentState();
+    EXPECT_EQ(sdk::AggregateState(currentState1), sdk::AggregatedTopologyState::Mixed);
+    EXPECT_EQ(sdk::StateEqualsTo(currentState1, sdk::DeviceState::InitializingDevice), false);
+
+    auto result2(topo.ChangeState(sdk::TopologyTransition::InitDevice, "main/SinkGroup/.*"));
+    LOG(info) << result2.first;
+
+    EXPECT_EQ(result2.first, std::error_code());
+    EXPECT_EQ(sdk::AggregateState(result2.second), sdk::AggregatedTopologyState::InitializingDevice);
+    EXPECT_EQ(sdk::StateEqualsTo(result2.second, sdk::DeviceState::InitializingDevice), true);
+    auto const currentState2 = topo.GetCurrentState();
+    EXPECT_EQ(sdk::AggregateState(currentState2), sdk::AggregatedTopologyState::InitializingDevice);
+    EXPECT_EQ(sdk::StateEqualsTo(currentState2, sdk::DeviceState::InitializingDevice), true);
+}
+
 TEST_F(Topology, AsyncChangeStateConcurrent)
 {
     using namespace fair::mq;
@@ -191,9 +217,9 @@ TEST_F(Topology, AsyncChangeStateCollectionView)
             ASSERT_EQ(cstate.size(), 5);
             for (const auto& c : cstate) {
                 LOG(debug) << "\t" << c.first;
-                State s;
+                sdk::AggregatedTopologyState s;
                 ASSERT_NO_THROW(s = sdk::AggregateState(c.second));
-                ASSERT_EQ(s, State::InitializingDevice);
+                ASSERT_EQ(s, static_cast<sdk::AggregatedTopologyState>(State::InitializingDevice));
                 LOG(debug) << "\tAggregated state: " << s;
                 for (const auto& ds : c.second) {
                     LOG(debug) << "\t\t" << ds.state;
@@ -335,7 +361,7 @@ TEST_F(Topology, AsyncSetPropertiesTimeout)
 
     topo.AsyncSetProperties({{"key1", "val1"}},
                             "",
-                            std::chrono::milliseconds(1),
+                            std::chrono::microseconds(1),
                             [=](std::error_code ec, sdk::FailedDevices) mutable {
                                 LOG(info) << ec;
                                 EXPECT_EQ(ec, MakeErrorCode(ErrorCode::OperationTimeout));
@@ -426,4 +452,62 @@ TEST_F(Topology, SetAndGetProperties)
     ASSERT_EQ(topo.ChangeState(TopologyTransition::ResetDevice).first, std::error_code());
 }
 
+TEST(Topology2, AggregatedTopologyStateComparison)
+{
+    using namespace fair::mq::sdk;
+    ASSERT_TRUE(DeviceState::Undefined == AggregatedTopologyState::Undefined);
+    ASSERT_TRUE(AggregatedTopologyState::Undefined == DeviceState::Undefined);
+    ASSERT_TRUE(DeviceState::Ok == AggregatedTopologyState::Ok);
+    ASSERT_TRUE(DeviceState::Error == AggregatedTopologyState::Error);
+    ASSERT_TRUE(DeviceState::Idle == AggregatedTopologyState::Idle);
+    ASSERT_TRUE(DeviceState::InitializingDevice == AggregatedTopologyState::InitializingDevice);
+    ASSERT_TRUE(DeviceState::Initialized == AggregatedTopologyState::Initialized);
+    ASSERT_TRUE(DeviceState::Binding == AggregatedTopologyState::Binding);
+    ASSERT_TRUE(DeviceState::Bound == AggregatedTopologyState::Bound);
+    ASSERT_TRUE(DeviceState::Connecting == AggregatedTopologyState::Connecting);
+    ASSERT_TRUE(DeviceState::DeviceReady == AggregatedTopologyState::DeviceReady);
+    ASSERT_TRUE(DeviceState::InitializingTask == AggregatedTopologyState::InitializingTask);
+    ASSERT_TRUE(DeviceState::Ready == AggregatedTopologyState::Ready);
+    ASSERT_TRUE(DeviceState::Running == AggregatedTopologyState::Running);
+    ASSERT_TRUE(DeviceState::ResettingTask == AggregatedTopologyState::ResettingTask);
+    ASSERT_TRUE(DeviceState::ResettingDevice == AggregatedTopologyState::ResettingDevice);
+    ASSERT_TRUE(DeviceState::Exiting == AggregatedTopologyState::Exiting);
+
+    ASSERT_TRUE(GetAggregatedTopologyState("UNDEFINED") == AggregatedTopologyState::Undefined);
+    ASSERT_TRUE(GetAggregatedTopologyState("OK") == AggregatedTopologyState::Ok);
+    ASSERT_TRUE(GetAggregatedTopologyState("ERROR") == AggregatedTopologyState::Error);
+    ASSERT_TRUE(GetAggregatedTopologyState("IDLE") == AggregatedTopologyState::Idle);
+    ASSERT_TRUE(GetAggregatedTopologyState("INITIALIZING DEVICE") == AggregatedTopologyState::InitializingDevice);
+    ASSERT_TRUE(GetAggregatedTopologyState("INITIALIZED") == AggregatedTopologyState::Initialized);
+    ASSERT_TRUE(GetAggregatedTopologyState("BINDING") == AggregatedTopologyState::Binding);
+    ASSERT_TRUE(GetAggregatedTopologyState("BOUND") == AggregatedTopologyState::Bound);
+    ASSERT_TRUE(GetAggregatedTopologyState("CONNECTING") == AggregatedTopologyState::Connecting);
+    ASSERT_TRUE(GetAggregatedTopologyState("DEVICE READY") == AggregatedTopologyState::DeviceReady);
+    ASSERT_TRUE(GetAggregatedTopologyState("INITIALIZING TASK") == AggregatedTopologyState::InitializingTask);
+    ASSERT_TRUE(GetAggregatedTopologyState("READY") == AggregatedTopologyState::Ready);
+    ASSERT_TRUE(GetAggregatedTopologyState("RUNNING") == AggregatedTopologyState::Running);
+    ASSERT_TRUE(GetAggregatedTopologyState("RESETTING TASK") == AggregatedTopologyState::ResettingTask);
+    ASSERT_TRUE(GetAggregatedTopologyState("RESETTING DEVICE") == AggregatedTopologyState::ResettingDevice);
+    ASSERT_TRUE(GetAggregatedTopologyState("EXITING") == AggregatedTopologyState::Exiting);
+    ASSERT_TRUE(GetAggregatedTopologyState("MIXED") == AggregatedTopologyState::Mixed);
+
+    ASSERT_TRUE("UNDEFINED" == GetAggregatedTopologyStateName(AggregatedTopologyState::Undefined));
+    ASSERT_TRUE("OK" == GetAggregatedTopologyStateName(AggregatedTopologyState::Ok));
+    ASSERT_TRUE("ERROR" == GetAggregatedTopologyStateName(AggregatedTopologyState::Error));
+    ASSERT_TRUE("IDLE" == GetAggregatedTopologyStateName(AggregatedTopologyState::Idle));
+    ASSERT_TRUE("INITIALIZING DEVICE" == GetAggregatedTopologyStateName(AggregatedTopologyState::InitializingDevice));
+    ASSERT_TRUE("INITIALIZED" == GetAggregatedTopologyStateName(AggregatedTopologyState::Initialized));
+    ASSERT_TRUE("BINDING" == GetAggregatedTopologyStateName(AggregatedTopologyState::Binding));
+    ASSERT_TRUE("BOUND" == GetAggregatedTopologyStateName(AggregatedTopologyState::Bound));
+    ASSERT_TRUE("CONNECTING" == GetAggregatedTopologyStateName(AggregatedTopologyState::Connecting));
+    ASSERT_TRUE("DEVICE READY" == GetAggregatedTopologyStateName(AggregatedTopologyState::DeviceReady));
+    ASSERT_TRUE("INITIALIZING TASK" == GetAggregatedTopologyStateName(AggregatedTopologyState::InitializingTask));
+    ASSERT_TRUE("READY" == GetAggregatedTopologyStateName(AggregatedTopologyState::Ready));
+    ASSERT_TRUE("RUNNING" == GetAggregatedTopologyStateName(AggregatedTopologyState::Running));
+    ASSERT_TRUE("RESETTING TASK" == GetAggregatedTopologyStateName(AggregatedTopologyState::ResettingTask));
+    ASSERT_TRUE("RESETTING DEVICE" == GetAggregatedTopologyStateName(AggregatedTopologyState::ResettingDevice));
+    ASSERT_TRUE("EXITING" == GetAggregatedTopologyStateName(AggregatedTopologyState::Exiting));
+    ASSERT_TRUE("MIXED" == GetAggregatedTopologyStateName(AggregatedTopologyState::Mixed));
+}
+
 }   // namespace

test/transport/_options.cxx

@@ -66,6 +66,24 @@ void RunOptionsTest(const string& transport)
     ASSERT_EQ(channel.GetSocket().GetRcvKernelSize(), 8000);
 }
 
+void ZeroingAndMlock(const string& transport)
+{
+    size_t session{fair::mq::tools::UuidHash()};
+
+    fair::mq::ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+    config.SetProperty<size_t>("shm-segment-size", 16384);
+    config.SetProperty<bool>("shm-zero-segment", true);
+    config.SetProperty<bool>("shm-mlock-segment", true);
+
+    auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config);
+
+    FairMQMessagePtr outMsg(factory->CreateMessage(10000));
+    char test[10000];
+    memset(test, 0, sizeof(test));
+    ASSERT_EQ(memcmp(test, outMsg->GetData(), outMsg->GetSize()), 0);
+}
+
 TEST(Options, zeromq)
 {
     RunOptionsTest("zeromq");
@@ -76,4 +94,9 @@ TEST(Options, shmem)
     RunOptionsTest("shmem");
 }
 
+TEST(ZeroingAndMlock, shmem)
+{
+    ZeroingAndMlock("shmem");
+}
+
 } // namespace

test/transport/_transfer_timeout.cxx

@@ -7,9 +7,16 @@
  ********************************************************************************/
 
 #include "runner.h"
+#include <FairMQChannel.h>
+#include <FairMQLogger.h>
+#include <FairMQTransportFactory.h>
+#include <fairmq/ProgOptions.h>
 #include <fairmq/Tools.h>
 #include <gtest/gtest.h>
+
+#include <chrono>
 #include <sstream> // std::stringstream
+#include <thread>
 
 namespace
 {
@@ -18,6 +25,12 @@ using namespace std;
 using namespace fair::mq::test;
 using namespace fair::mq::tools;
 
+void delayedInterruptor(FairMQTransportFactory& transport)
+{
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    transport.Interrupt();
+}
+
 auto RunTransferTimeout(string transport) -> void
 {
     size_t session{fair::mq::tools::UuidHash()};
@@ -31,6 +44,28 @@ auto RunTransferTimeout(string transport) -> void
     exit(res.exit_code);
 }
 
+void InterruptTransfer(const string& transport, const string& _address)
+{
+    size_t session{fair::mq::tools::UuidHash()};
+    std::string address(fair::mq::tools::ToString(_address, "_", transport));
+
+    fair::mq::ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config);
+
+    FairMQChannel pull{"Pull", "pull", factory};
+    pull.Bind(address);
+
+    FairMQMessagePtr msg(pull.NewMessage());
+
+    auto t = thread(delayedInterruptor, ref(*factory));
+
+    auto result = pull.Receive(msg);
+    t.join();
+    ASSERT_EQ(result, static_cast<int>(fair::mq::TransferResult::interrupted));
+}
+
 TEST(TransferTimeout, zeromq)
 {
     EXPECT_EXIT(RunTransferTimeout("zeromq"), ::testing::ExitedWithCode(0), "Transfer timeout test successfull");
@@ -41,4 +76,14 @@ TEST(TransferTimeout, shmem)
     EXPECT_EXIT(RunTransferTimeout("shmem"), ::testing::ExitedWithCode(0), "Transfer timeout test successfull");
 }
 
+TEST(InterruptTransfer, zeromq)
+{
+    InterruptTransfer("zeromq", "ipc://test_interrupt_transfer");
+}
+
+TEST(InterruptTransfer, shmem)
+{
+    InterruptTransfer("shmem", "ipc://test_interrupt_transfer");
+}
+
 } // namespace