Adjust program options style to disallow guessing

A double SIGTERM, or even worse a SIGINT + SIGTERM combination should
not result in an abort, given such a signal really means "die whenever
you want" and can only be generated programmatically. If one wants the
child to die programmatically, they should use SIGKILL.

On the other hand that bashing ctrl-c (i.e. SIGINT) multiple times
on the keyboard really means abort.

CMakeLists.txt

@@ -35,8 +35,6 @@ fairmq_build_option(BUILD_FAIRMQ            "Build FairMQ library and devices."
                                              DEFAULT ON)
 fairmq_build_option(BUILD_TESTING           "Build tests."
                                              DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_NANOMSG_TRANSPORT "Build nanomsg transport."
-                                             DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_OFI_TRANSPORT     "Build experimental OFI transport."
                                              DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_SDK_COMMANDS      "Build the FairMQ SDK commands."
@@ -53,6 +51,8 @@ fairmq_build_option(BUILD_DOCS              "Build FairMQ documentation."
                                              DEFAULT OFF)
 fairmq_build_option(FAST_BUILD              "Fast production build. Not recommended for development."
                                              DEFAULT OFF)
+fairmq_build_option(USE_EXTERNAL_GTEST      "Do not use bundled GTest. Not recommended."
+                                             DEFAULT OFF)
 ################################################################################
 
 
@@ -65,11 +65,6 @@ set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 
-if(BUILD_NANOMSG_TRANSPORT)
-  find_package2(PRIVATE nanomsg REQUIRED)
-  set(PROJECT_nanomsg_VERSION 1.1.3) # Once upstream releases 1.1.5, we should bump again and use version check
-endif()
-
 if(BUILD_OFI_TRANSPORT)
   find_package2(PRIVATE asiofi REQUIRED
     VERSION 0.3.1
@@ -79,12 +74,6 @@ if(BUILD_OFI_TRANSPORT)
   )
 endif()
 
-if(BUILD_NANOMSG_TRANSPORT)
-  find_package2(PRIVATE msgpack REQUIRED
-    VERSION 3.1.0
-  )
-endif()
-
 if(BUILD_SDK_COMMANDS)
   find_package2(PRIVATE Flatbuffers REQUIRED)
 endif()
@@ -160,7 +149,9 @@ if(BUILD_FAIRMQ)
 endif()
 
 if(BUILD_TESTING)
+  if(USE_EXTERNAL_GTEST)
     find_package2(PRIVATE GTest VERSION 1.7.0)
+  endif()
   if(NOT GTest_FOUND)
     build_bundled(GTest extern/googletest)
     find_package2(PRIVATE GTest REQUIRED)
@@ -215,9 +206,6 @@ endif()
 if(BUILD_PMIX_PLUGIN)
   list(APPEND PROJECT_PACKAGE_COMPONENTS pmix_plugin)
 endif()
-if(BUILD_NANOMSG_TRANSPORT)
-  list(APPEND PROJECT_PACKAGE_COMPONENTS nanomsg_transport)
-endif()
 if(BUILD_OFI_TRANSPORT)
   list(APPEND PROJECT_PACKAGE_COMPONENTS ofi_transport)
 endif()
@@ -320,16 +308,10 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
       endif()
     elseif(${dep} STREQUAL GTest)
       get_filename_component(prefix ${GTEST_INCLUDE_DIRS}/.. ABSOLUTE)
-    elseif(${dep} STREQUAL msgpack)
-      get_target_property(msgpack_include msgpackc-cxx INTERFACE_INCLUDE_DIRECTORIES)
-      get_filename_component(prefix ${msgpack_include}/.. ABSOLUTE)
     elseif(${dep} STREQUAL asiofi)
       set(prefix ${asiofi_ROOT})
     elseif(${dep} STREQUAL OFI)
       get_filename_component(prefix ${${dep}_INCLUDE_DIRS}/.. ABSOLUTE)
-    elseif(${dep} STREQUAL nanomsg)
-      get_target_property(nn_include nanomsg INTERFACE_INCLUDE_DIRECTORIES)
-      get_filename_component(prefix ${nn_include}/.. ABSOLUTE)
     elseif(${dep} STREQUAL DDS)
       set(prefix "${DDS_INSTALL_PREFIX}")
     elseif(${dep} STREQUAL Boost)
@@ -378,12 +360,6 @@ else()
   set(tests_summary "${BRed} NO${CR}    (enable with ${BMagenta}-DBUILD_TESTING=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}tests${CR}              ${tests_summary}")
-if(BUILD_NANOMSG_TRANSPORT)
-  set(nn_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_NANOMSG_TRANSPORT=OFF${CR})")
-else()
-  set(nn_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_NANOMSG_TRANSPORT=ON${CR})")
-endif()
-message(STATUS "  ${BWhite}nanomsg_transport${CR}  ${nn_summary}")
 if(BUILD_OFI_TRANSPORT)
   set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (requires C++14) (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
 else()

FairMQTest.cmake

@@ -26,7 +26,6 @@ Set(configure_options "${configure_options};-DCTEST_USE_LAUNCHERS=${CTEST_USE_LA
 
 Set(configure_options "${configure_options};-DDISABLE_COLOR=ON")
 Set(configure_options "${configure_options};-DCMAKE_PREFIX_PATH=$ENV{SIMPATH}")
-Set(configure_options "${configure_options};-DBUILD_NANOMSG_TRANSPORT=ON")
 # Set(configure_options "${configure_options};-DBUILD_OFI_TRANSPORT=ON")
 Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
 Set(configure_options "${configure_options};-DBUILD_SDK=ON")

README.md

@@ -21,7 +21,7 @@ FairMQ is designed to help implementing large-scale data processing workflows ne
 The core of FairMQ provides an abstract asynchronous message passing API with scalability protocols
 inspired by [ZeroMQ](https://github.com/zeromq/libzmq) (e.g. PUSH/PULL, PUB/SUB).
 FairMQ provides multiple implementations for its API (so-called "transports",
-e.g. `zeromq`, `shmem`, `nanomsg`, and `ofi` (in development)) to cover a variety of use cases
+e.g. `zeromq`, `shmem` and `ofi` (in development)) to cover a variety of use cases
 (e.g. inter-thread, inter-process, inter-node communication) and machines (e.g. Ethernet, Infiniband).
 In addition to this core functionality FairMQ provides a framework for creating "devices" - actors which
 are communicating through message passing. FairMQ does not only allow the user to use different transport but also to mix them; i.e: A Device can communicate using different transport on different channels at the same time. Device execution is modelled as a simple state machine that
@@ -47,7 +47,7 @@ cmake --build fairmq_build --target install
 
 Please consult the [manpages of your CMake version](https://cmake.org/cmake/help/latest/manual/cmake.1.html) for more options.
 
-If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PMIX`, `ASIO`, `ASIOFI` or `DDS` (`*_ROOT` variables can also be environment variables).
+If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `OFI`, `PMIX`, `ASIO`, `ASIOFI` or `DDS` (`*_ROOT` variables can also be environment variables).
 
 ## Usage
 
@@ -86,7 +86,7 @@ If your project shares a dependency with FairMQ or if you want to omit a certain
 Optionally, you can require certain FairMQ package components and a minimum version:
 
 ```cmake
-find_package(FairMQ 1.1.0 COMPONENTS nanomsg_transport dds_plugin)
+find_package(FairMQ 1.1.0 COMPONENTS dds_plugin)
 ```
 
 When building FairMQ, CMake will print a summary table of all available package components.
@@ -101,8 +101,6 @@ When building FairMQ, CMake will print a summary table of all available package
   * [Doxygen](http://www.doxygen.org/)
   * [FairLogger](https://github.com/FairRootGroup/FairLogger)
   * [GTest](https://github.com/google/googletest) (optionally bundled)
-  * [Msgpack](https://msgpack.org/index.html)
-  * [nanomsg](http://nanomsg.org/)
   * [PMIx](https://pmix.org/)
   * [ZeroMQ](http://zeromq.org/)
 
@@ -117,7 +115,6 @@ On command line:
   * `-DDISABLE_COLOR=ON` disables coloured console output.
   * `-DBUILD_TESTING=OFF` disables building of tests.
   * `-DBUILD_EXAMPLES=OFF` disables building of examples.
-  * `-DBUILD_NANOMSG_TRANSPORT=ON` enables building of nanomsg transport.
   * `-DBUILD_OFI_TRANSPORT=ON` enables building of the experimental OFI transport.
   * `-DBUILD_DDS_PLUGIN=ON` enables building of the DDS plugin.
   * `-DBUILD_PMIX_PLUGIN=ON` enables building of the PMIx plugin.
@@ -182,4 +179,3 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
        1. [DDS](docs/Plugins.md#731-dds)
        2. [PMIx](docs/Plugins.md#732-pmix)
 8. [Controller SDK](docs/SDK.md)
-

docs/Device.md

@@ -18,7 +18,7 @@ Topology configuration is currently happening via setup scripts. This is very ru
 
 ## 1.2 Communication Patterns
 
-FairMQ devices communicate via the communication patterns offered by ZeroMQ (or nanomsg): PUSH-PULL, PUB-SUB, REQ-REP, PAIR, [more info here](http://api.zeromq.org/4-0:zmq-socket). Each transport may provide further patterns.
+FairMQ devices communicate via the communication patterns offered by ZeroMQ: PUSH-PULL, PUB-SUB, REQ-REP, PAIR, [more info here](http://api.zeromq.org/4-0:zmq-socket). Each transport may provide further patterns.
 
 ## 1.3 State Machine
 

docs/Transport.md

@@ -2,7 +2,7 @@
 
 # 2. Transport Interface
 
-The communication layer is available through the transport interface. Three interface implementations are currently available. Main implementation uses the [ZeroMQ](http://zeromq.org) library. Alternative implementation relies on the [nanomsg](http://nanomsg.org) library. Third transport implementation is using shared memory via boost::interprocess & ZeroMQ combination.
+The communication layer is available through the transport interface. Three interface implementations are currently available. Main implementation uses the [ZeroMQ](http://zeromq.org) library. Second transport implementation is using shared memory via boost::interprocess & ZeroMQ combination.
 
 Here is an overview to give an idea how the interface is implemented:
 
@@ -10,20 +10,20 @@ Here is an overview to give an idea how the interface is implemented:
 
 Currently, the transports have been tested to work with these communication patterns:
 
-|               | zeromq | nanomsg | shmem |
-| ------------- |--------| ------- | ----- |
-| PAIR          | yes    | yes     | yes   |
-| PUSH/PULL     | yes    | yes     | yes   |
-| PUB/SUB       | yes    | yes     | no    |
-| REQ/REP       | yes    | yes     | yes   |
+|               | zeromq | shmem |
+| ------------- |--------| ----- |
+| PAIR          | yes    | yes   |
+| PUSH/PULL     | yes    | yes   |
+| PUB/SUB       | yes    | no    |
+| REQ/REP       | yes    | yes   |
 
 The next table shows the supported address types for each transport implementation:
 
-|             | zeromq | nanomsg | shmem | comment                                       |
-| ----------- | ------ | ------- | ----- | --------------------------------------------- |
-| `inproc://` | yes    | yes     | yes   | in process: useful for unit testing           |
-| `ipc://`    | yes    | yes     | yes   | inter process comm: useful on single machine  |
-| `tcp://`    | yes    | yes     | yes   | useful for any communication, local or remote |
+|             | zeromq | shmem | comment                                       |
+| ----------- | ------ | ----- | --------------------------------------------- |
+| `inproc://` | yes    | yes   | in process: useful for unit testing           |
+| `ipc://`    | yes    | yes   | inter process comm: useful on single machine  |
+| `tcp://`    | yes    | yes   | useful for any communication, local or remote |
 
 ## 2.1 Message
 

examples/1-1/CMakeLists.txt

@@ -34,11 +34,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-1.sh.in ${CMAKE_CURRENT_BIN
 add_test(NAME Example.1-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
 set_tests_properties(Example.1-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.1-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh nanomsg)
-  set_tests_properties(Example.1-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
-endif()
-
 add_test(NAME Example.1-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
 set_tests_properties(Example.1-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 

examples/1-n-1/CMakeLists.txt

@@ -42,11 +42,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-n-1.sh.in ${CMAKE_CURRENT_B
 add_test(NAME Example.1-n-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
 set_tests_properties(Example.1-n-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.1-n-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh nanomsg)
-  set_tests_properties(Example.1-n-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
-endif()
-
 add_test(NAME Example.1-n-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
 set_tests_properties(Example.1-n-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 

examples/CMakeLists.txt

@@ -13,9 +13,7 @@ add_subdirectory(copypush)
 add_subdirectory(dds)
 add_subdirectory(multipart)
 add_subdirectory(multiple-channels)
-if(BUILD_NANOMSG_TRANSPORT)
-  add_subdirectory(multiple-transports)
-endif()
+add_subdirectory(multiple-transports)
 add_subdirectory(n-m)
 add_subdirectory(qc)
 add_subdirectory(readout)

examples/README.md

@@ -32,7 +32,7 @@ This example demonstrates how to work with multiple channels and multiplex betwe
 
 ## Multiple Transports
 
-This examples shows how to combine different channel transports (zeromq/nanomsg/shmem) inside of one device and/or topology.
+This examples shows how to combine different channel transports (zeromq/shmem) inside of one device and/or topology.
 
 ## n-m
 

examples/builtin-devices/CMakeLists.txt

@@ -17,22 +17,12 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-builtin-devices.sh.in ${CMAKE
 add_test(NAME Example.BuiltinDevices.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
 set_tests_properties(Example.BuiltinDevices.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.BuiltinDevices.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg)
-  set_tests_properties(Example.BuiltinDevices.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
-endif()
-
 add_test(NAME Example.BuiltinDevices.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
 set_tests_properties(Example.BuiltinDevices.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 add_test(NAME Example.BuiltinDevices.multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
 set_tests_properties(Example.BuiltinDevices.multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.BuiltinDevices.multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg true 2)
-  set_tests_properties(Example.BuiltinDevices.multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
-endif()
-
 add_test(NAME Example.BuiltinDevices.multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
 set_tests_properties(Example.BuiltinDevices.multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 

examples/copypush/CMakeLists.txt

@@ -35,11 +35,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-copypush.sh.in ${CMAKE_CURREN
 add_test(NAME Example.CopyPush.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
 set_tests_properties(Example.CopyPush.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.CopyPush.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh nanomsg)
-  set_tests_properties(Example.CopyPush.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
-endif()
-
 add_test(NAME Example.CopyPush.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
 set_tests_properties(Example.CopyPush.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 

examples/multipart/CMakeLists.txt

@@ -34,11 +34,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multipart.sh.in ${CMAKE_CURRE
 add_test(NAME Example.Multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
 set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.Multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
-  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
-endif()
-
 add_test(NAME Example.Multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
 set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 7 parts")
 

examples/multiple-channels/CMakeLists.txt

@@ -39,11 +39,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-channels.sh.in ${CMA
 add_test(NAME Example.MultipleChannels.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
 set_tests_properties(Example.MultipleChannels.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.MultipleChannels.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh nanomsg)
-  set_tests_properties(Example.MultipleChannels.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
-endif()
-
 # install
 
 install(

examples/multiple-transports/README.md

@@ -1,12 +1,12 @@
 Multiple Transports example
 ===========================
 
-This example demonstrates use of multiple transports (zeromq/nanomsg/shmem) within the same topology and/or device. It is a simple topology consisting of two samplers and a sink. The devices are connected via 3 channels:
+This example demonstrates use of multiple transports (zeromq/shmem) within the same topology and/or device. It is a simple topology consisting of two samplers and a sink. The devices are connected via 3 channels:
 
 ![Multiple Transports example](../../docs/images/example_multiple_transports.png?raw=true "Multiple Transports example")
 
 Each device has main transport that it uses. By default it is ZeroMQ, and can be overriden via the `--transport` cmd option. The device will initialize additional transports if any of the channels have them configured (e.g. as an option to `--channel-config`).
 
-In this example sampler1 and sink are started with `--transport shmem`, making shared memory their main transport, sampler2 with `--transport nanomsg`. Additionally, the ack channel is configured to use zeromq as its transport.
+In this example sampler1 and sink are started with `--transport shmem`, making shared memory their main transport, sampler2 with `--transport zeromq`. Additionally, the ack channel is configured to use zeromq as its transport.
 
 The main two things that a transport does is transfer of data and allocation of memory for the messages. By default, new messages are created via the main device transport. If a message has been created with one transport and is to be transferred with another, it has to be copied into a new message of the target transport. This happens automatically behind the scenes. To avoid this copy the device can create messages via `NewMessageFor(const string& channelName, int subChannelIndex, ...)` method, that creates the messages via the transport of the given channel (check sampler1 and sink for an example) or as the channel directly to create a message.

examples/multiple-transports/fairmq-start-ex-multiple-transports.sh.in

@@ -13,7 +13,7 @@ xterm -geometry 80x30+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER1 &
 SAMPLER2="fairmq-ex-multiple-transports-sampler2"
 SAMPLER2+=" --id sampler2"
 SAMPLER2+=" --severity debug"
-SAMPLER2+=" --transport nanomsg"
+SAMPLER2+=" --transport shmem"
 SAMPLER2+=" --channel-config name=data2,type=push,method=bind,address=tcp://127.0.0.1:5556"
 xterm -geometry 80x30+0+450 -hold -e @EX_BIN_DIR@/$SAMPLER2 &
 
@@ -22,6 +22,6 @@ SINK+=" --id sink1"
 SINK+=" --severity debug"
 SINK+=" --transport shmem"
 SINK+=" --channel-config name=data1,type=pull,method=connect,address=tcp://127.0.0.1:5555"
-SINK+="                  name=data2,type=pull,method=connect,address=tcp://127.0.0.1:5556,transport=nanomsg"
+SINK+="                  name=data2,type=pull,method=connect,address=tcp://127.0.0.1:5556,transport=shmem"
 SINK+="                  name=ack,type=pub,method=connect,address=tcp://127.0.0.1:5557,transport=zeromq"
 xterm -geometry 80x30+500+225 -hold -e @EX_BIN_DIR@/$SINK &

examples/multiple-transports/test-ex-multiple-transports.sh.in

@@ -14,7 +14,7 @@ SINK+=" --max-iterations 1"
 SINK+=" --control static --color false"
 SINK+=" --transport shmem"
 SINK+=" --channel-config name=data1,type=pull,method=connect,address=tcp://127.0.0.1:5555"
-SINK+="                  name=data2,type=pull,method=connect,address=tcp://127.0.0.1:5556,transport=nanomsg"
+SINK+="                  name=data2,type=pull,method=connect,address=tcp://127.0.0.1:5556,transport=zeromq"
 SINK+="                  name=ack,type=pub,method=connect,address=tcp://127.0.0.1:5557,transport=zeromq"
 @CMAKE_CURRENT_BINARY_DIR@/$SINK &
 SINK_PID=$!
@@ -37,7 +37,7 @@ SAMPLER2+=" --session $SESSION"
 SAMPLER2+=" --verbosity veryhigh"
 SAMPLER2+=" --max-iterations 1"
 SAMPLER2+=" --control static --color false"
-SAMPLER2+=" --transport nanomsg"
+SAMPLER2+=" --transport zeromq"
 SAMPLER2+=" --channel-config name=data2,type=push,method=bind,address=tcp://127.0.0.1:5556"
 @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER2 &
 SAMPLER2_PID=$!

examples/readout/Readout.h

@@ -37,10 +37,10 @@ class Readout : public FairMQDevice
         fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("rb",
                                                                 0,
                                                                 10000000,
-                                                                [this](void* /*data*/, size_t /*size*/, void* /*hint*/) { // callback to be called when message buffers no longer needed by transport
-                                                                    --fNumUnackedMsgs;
+                                                                [this](const std::vector<fair::mq::RegionBlock>& blocks) { // callback to be called when message buffers no longer needed by transport
+                                                                    fNumUnackedMsgs -= blocks.size();
                                                                     if (fMaxIterations > 0) {
-                                                                        LOG(debug) << "Received ack";
+                                                                        LOG(debug) << "Received " << blocks.size() << " acks";
                                                                     }
                                                                 }
                                                                 ));

examples/region/CMakeLists.txt

@@ -32,15 +32,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-region.sh.in ${CMAKE_
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-region.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh)
 
 add_test(NAME Example.Region.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
-set_tests_properties(Example.Region.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
-
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.Region.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh nanomsg)
-  set_tests_properties(Example.Region.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
-endif()
+set_tests_properties(Example.Region.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received [0-9*] acks")
 
 add_test(NAME Example.Region.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
-set_tests_properties(Example.Region.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+set_tests_properties(Example.Region.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received [0-9*] acks")
 
 # install
 

examples/region/Sampler.cxx

@@ -46,10 +46,10 @@ void Sampler::InitTask()
     fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegionFor("data",
                                                              0,
                                                              10000000,
-                                                             [this](void* /*data*/, size_t /*size*/, void* /*hint*/) { // callback to be called when message buffers no longer needed by transport
-                                                                 --fNumUnackedMsgs;
+                                                             [this](const std::vector<fair::mq::RegionBlock>& blocks) { // callback to be called when message buffers no longer needed by transport
+                                                                 fNumUnackedMsgs -= blocks.size();
                                                                  if (fMaxIterations > 0) {
-                                                                     LOG(debug) << "Received ack";
+                                                                    LOG(debug) << "Received " << blocks.size() << " acks";
                                                                  }
                                                              }
                                                              ));

examples/req-rep/CMakeLists.txt

@@ -35,11 +35,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-req-rep.sh.in ${CMAKE_CURRENT
 add_test(NAME Example.ReqRep.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
 set_tests_properties(Example.ReqRep.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 
-if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example.ReqRep.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh nanomsg)
-  set_tests_properties(Example.ReqRep.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
-endif()
-
 add_test(NAME Example.ReqRep.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
 set_tests_properties(Example.ReqRep.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 

fairmq/CMakeLists.txt

@@ -190,23 +190,14 @@ if(BUILD_FAIRMQ)
     shmem/Common.h
     shmem/Manager.h
     shmem/Region.h
-    zeromq/FairMQMessageZMQ.h
-    zeromq/FairMQPollerZMQ.h
-    zeromq/FairMQUnmanagedRegionZMQ.h
-    zeromq/FairMQSocketZMQ.h
-    zeromq/FairMQTransportFactoryZMQ.h
+    zeromq/Context.h
+    zeromq/Message.h
+    zeromq/Poller.h
+    zeromq/UnmanagedRegion.h
+    zeromq/Socket.h
+    zeromq/TransportFactory.h
   )
 
-  if(BUILD_NANOMSG_TRANSPORT)
-    set(FAIRMQ_PRIVATE_HEADER_FILES ${FAIRMQ_PRIVATE_HEADER_FILES}
-      nanomsg/FairMQMessageNN.h
-      nanomsg/FairMQPollerNN.h
-      nanomsg/FairMQUnmanagedRegionNN.h
-      nanomsg/FairMQSocketNN.h
-      nanomsg/FairMQTransportFactoryNN.h
-    )
-  endif()
-
   if(BUILD_OFI_TRANSPORT)
     set(FAIRMQ_PRIVATE_HEADER_FILES ${FAIRMQ_PRIVATE_HEADER_FILES}
       ofi/Context.h
@@ -229,7 +220,6 @@ if(BUILD_FAIRMQ)
     FairMQPoller.cxx
     FairMQSocket.cxx
     FairMQTransportFactory.cxx
-    devices/FairMQBenchmarkSampler.cxx
     devices/FairMQMerger.cxx
     devices/FairMQMultiplier.cxx
     devices/FairMQProxy.cxx
@@ -243,30 +233,9 @@ if(BUILD_FAIRMQ)
     SuboptParser.cxx
     plugins/config/Config.cxx
     plugins/Control.cxx
-    shmem/Message.cxx
-    shmem/Poller.cxx
-    shmem/Socket.cxx
-    shmem/TransportFactory.cxx
-    shmem/Manager.cxx
-    shmem/Region.cxx
-    zeromq/FairMQMessageZMQ.cxx
-    zeromq/FairMQPollerZMQ.cxx
-    zeromq/FairMQUnmanagedRegionZMQ.cxx
-    zeromq/FairMQSocketZMQ.cxx
-    zeromq/FairMQTransportFactoryZMQ.cxx
     MemoryResources.cxx
   )
 
-  if(BUILD_NANOMSG_TRANSPORT)
-    set(FAIRMQ_SOURCE_FILES ${FAIRMQ_SOURCE_FILES}
-      nanomsg/FairMQMessageNN.cxx
-      nanomsg/FairMQPollerNN.cxx
-      nanomsg/FairMQUnmanagedRegionNN.cxx
-      nanomsg/FairMQSocketNN.cxx
-      nanomsg/FairMQTransportFactoryNN.cxx
-    )
-  endif()
-
   if(BUILD_OFI_TRANSPORT)
     set(FAIRMQ_SOURCE_FILES ${FAIRMQ_SOURCE_FILES}
       ofi/Context.cxx
@@ -307,9 +276,6 @@ if(BUILD_FAIRMQ)
   # preprocessor definitions #
   ############################
   target_compile_definitions(${_target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
-  if(BUILD_NANOMSG_TRANSPORT)
-    target_compile_definitions(${_target} PRIVATE BUILD_NANOMSG_TRANSPORT)
-  endif()
   if(BUILD_OFI_TRANSPORT)
     target_compile_definitions(${_target} PRIVATE BUILD_OFI_TRANSPORT)
   endif()
@@ -330,16 +296,13 @@ if(BUILD_FAIRMQ)
   ##################
   # link libraries #
   ##################
-  if(BUILD_NANOMSG_TRANSPORT)
-    set(NANOMSG_DEPS nanomsg msgpackc-cxx)
-  endif()
   if(BUILD_OFI_TRANSPORT)
     set(OFI_DEPS
       asiofi::asiofi
       Boost::container
     )
   endif()
-  set(optional_deps ${NANOMSG_DEPS} ${OFI_DEPS})
+  set(optional_deps ${OFI_DEPS})
   if(optional_deps)
     list(REMOVE_DUPLICATES optional_deps)
   endif()
@@ -362,7 +325,6 @@ if(BUILD_FAIRMQ)
 
     PRIVATE # only libFairMQ links against private dependencies
     libzmq
-    ${NANOMSG_DEPS}
     ${OFI_DEPS}
   )
   set_target_properties(${_target} PROPERTIES

fairmq/FairMQChannel.h

@@ -127,8 +127,8 @@ class FairMQChannel
     /// @return Returns socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
     std::string GetAddress() const;
 
-    /// Get channel transport name ("default", "zeromq", "nanomsg" or "shmem")
-    /// @return Returns channel transport name (e.g. "default", "zeromq", "nanomsg" or "shmem")
+    /// Get channel transport name ("default", "zeromq" or "shmem")
+    /// @return Returns channel transport name (e.g. "default", "zeromq" or "shmem")
     std::string GetTransportName() const;
 
     /// Get channel transport type
@@ -184,7 +184,7 @@ class FairMQChannel
     void UpdateAddress(const std::string& address);
 
     /// Set channel transport
-    /// @param transport transport string ("default", "zeromq", "nanomsg" or "shmem")
+    /// @param transport transport string ("default", "zeromq" or "shmem")
     void UpdateTransport(const std::string& transport);
 
     /// Set socket send buffer size
@@ -335,14 +335,10 @@ class FairMQChannel
         return Transport()->NewStaticMessage(data);
     }
 
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0)
+    template<typename... Args>
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(Args&&... args)
     {
-        return Transport()->CreateUnmanagedRegion(size, callback, path, flags);
-    }
-
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0)
-    {
-        return Transport()->CreateUnmanagedRegion(size, userFlags, callback, path, flags);
+        return Transport()->CreateUnmanagedRegion(std::forward<Args>(args)...);
     }
 
     static constexpr fair::mq::Transport DefaultTransportType = fair::mq::Transport::DEFAULT;

fairmq/FairMQDevice.h

@@ -217,45 +217,17 @@ class FairMQDevice
     }
 
     // creates unamanaged region with the default device transport
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size,
-                                                FairMQRegionCallback callback = nullptr,
-                                                const std::string& path = "",
-                                                int flags = 0)
+    template<typename... Args>
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(Args&&... args)
     {
-        return Transport()->CreateUnmanagedRegion(size, callback, path, flags);
-    }
-
-    // creates unamanaged region with the default device transport
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size,
-                                                const int64_t userFlags,
-                                                FairMQRegionCallback callback = nullptr,
-                                                const std::string& path = "",
-                                                int flags = 0)
-    {
-        return Transport()->CreateUnmanagedRegion(size, userFlags, callback, path, flags);
+        return Transport()->CreateUnmanagedRegion(std::forward<Args>(args)...);
     }
 
     // creates unmanaged region with the transport of the specified channel
-    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel,
-                                                   int index,
-                                                   const size_t size,
-                                                   FairMQRegionCallback callback = nullptr,
-                                                   const std::string& path = "",
-                                                   int flags = 0)
+    template<typename... Args>
+    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel, int index, Args&&... args)
     {
-        return GetChannel(channel, index).NewUnmanagedRegion(size, callback, path, flags);
-    }
-
-    // creates unmanaged region with the transport of the specified channel
-    FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel,
-                                                   int index,
-                                                   const size_t size,
-                                                   const int64_t userFlags,
-                                                   FairMQRegionCallback callback = nullptr,
-                                                   const std::string& path = "",
-                                                   int flags = 0)
-    {
-        return GetChannel(channel, index).NewUnmanagedRegion(size, userFlags, callback, path, flags);
+        return GetChannel(channel, index).NewUnmanagedRegion(std::forward<Args>(args)...);
     }
 
     template<typename ...Ts>
@@ -302,7 +274,7 @@ class FairMQDevice
     }
 
     /// Adds a transport to the device if it doesn't exist
-    /// @param transport  Transport string ("zeromq"/"nanomsg"/"shmem")
+    /// @param transport  Transport string ("zeromq"/"shmem")
     std::shared_ptr<FairMQTransportFactory> AddTransport(const fair::mq::Transport transport);
 
     /// Assigns config to the device
@@ -417,7 +389,7 @@ class FairMQDevice
     int GetInitTimeoutInS() const { return fConfig->GetProperty<int>("init-timeout", DefaultInitTimeout); }
 
     /// Sets the default transport for the device
-    /// @param transport  Transport string ("zeromq"/"nanomsg"/"shmem")
+    /// @param transport  Transport string ("zeromq"/"shmem")
     void SetTransport(const std::string& transport) { fConfig->SetProperty("transport", transport); }
     /// Gets the default transport name
     std::string GetTransportName() const { return fConfig->GetProperty<std::string>("transport", DefaultTransportName); }

fairmq/FairMQMessage.h

@@ -17,11 +17,26 @@
 using fairmq_free_fn = void(void* data, void* hint);
 class FairMQTransportFactory;
 
+namespace fair
+{
+namespace mq
+{
+
+struct Alignment
+{
+    size_t alignment;
+    explicit operator size_t() const { return alignment; }
+};
+
+} /* namespace mq */
+} /* namespace fair */
+
 class FairMQMessage
 {
   public:
     FairMQMessage() = default;
-    FairMQMessage(FairMQTransportFactory* factory):fTransport{factory} {}
+    FairMQMessage(FairMQTransportFactory* factory) : fTransport(factory) {}
+
     virtual void Rebuild() = 0;
     virtual void Rebuild(const size_t size) = 0;
     virtual void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) = 0;
@@ -33,7 +48,7 @@ class FairMQMessage
 
     virtual fair::mq::Transport GetType() const = 0;
     FairMQTransportFactory* GetTransport() { return fTransport; }
-    //void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }
+    void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }
 
     virtual void Copy(const FairMQMessage& msg) = 0;
 
@@ -53,6 +68,7 @@ namespace mq
 using Message = FairMQMessage;
 using MessagePtr = FairMQMessagePtr;
 struct MessageError : std::runtime_error { using std::runtime_error::runtime_error; };
+struct MessageBadAlloc : std::runtime_error { using std::runtime_error::runtime_error; };
 
 } /* namespace mq */
 } /* namespace fair */

fairmq/FairMQSocket.h

@@ -20,7 +20,7 @@ class FairMQSocket
 {
   public:
     FairMQSocket() {}
-    FairMQSocket(FairMQTransportFactory* fac): fTransport(fac) {}
+    FairMQSocket(FairMQTransportFactory* fac) : fTransport(fac) {}
 
     virtual std::string GetId() const = 0;
 
@@ -54,7 +54,7 @@ class FairMQSocket
     virtual unsigned long GetMessagesRx() const = 0;
 
     FairMQTransportFactory* GetTransport() { return fTransport; }
-    void SetTransport(FairMQTransportFactory* transport) { fTransport=transport; }
+    void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }
 
     virtual ~FairMQSocket() {};
 

fairmq/FairMQTransportFactory.cxx

@@ -8,15 +8,13 @@
 
 #include <FairMQTransportFactory.h>
 #include <fairmq/shmem/TransportFactory.h>
-#include <zeromq/FairMQTransportFactoryZMQ.h>
-#ifdef BUILD_NANOMSG_TRANSPORT
-#include <nanomsg/FairMQTransportFactoryNN.h>
-#endif /* BUILD_NANOMSG_TRANSPORT */
+#include <fairmq/zeromq/TransportFactory.h>
 #ifdef BUILD_OFI_TRANSPORT
 #include <fairmq/ofi/TransportFactory.h>
 #endif
 #include <FairMQLogger.h>
-#include <fairmq/tools/Unique.h>
+#include <fairmq/Tools.h>
+
 #include <memory>
 #include <string>
 
@@ -39,15 +37,10 @@ auto FairMQTransportFactory::CreateTransportFactory(const string& type,
     }
 
     if (type == "zeromq") {
-        return make_shared<FairMQTransportFactoryZMQ>(finalId, config);
+        return make_shared<fair::mq::zmq::TransportFactory>(finalId, config);
     } else if (type == "shmem") {
         return make_shared<fair::mq::shmem::TransportFactory>(finalId, config);
     }
-#ifdef BUILD_NANOMSG_TRANSPORT
-    else if (type == "nanomsg") {
-        return make_shared<FairMQTransportFactoryNN>(finalId, config);
-    }
-#endif /* BUILD_NANOMSG_TRANSPORT */
 #ifdef BUILD_OFI_TRANSPORT
     else if (type == "ofi") {
         return make_shared<fair::mq::ofi::TransportFactory>(finalId, config);
@@ -57,15 +50,12 @@ auto FairMQTransportFactory::CreateTransportFactory(const string& type,
         LOG(error) << "Unavailable transport requested: "
                    << "\"" << type << "\""
                    << ". Available are: "
-                   << "\"zeromq\""
+                   << "\"zeromq\","
                    << "\"shmem\""
-#ifdef BUILD_NANOMSG_TRANSPORT
-                   << ", \"nanomsg\""
-#endif /* BUILD_NANOMSG_TRANSPORT */
 #ifdef BUILD_OFI_TRANSPORT
                    << ", and \"ofi\""
 #endif /* BUILD_OFI_TRANSPORT */
                    << ". Exiting.";
-        exit(EXIT_FAILURE);
+        throw fair::mq::TransportFactoryError(fair::mq::tools::ToString("Unavailable transport requested: ", type));
     }
 }

fairmq/FairMQTransportFactory.h

@@ -18,7 +18,7 @@
 #include <fairmq/Transports.h>
 
 #include <string>
-#include <memory>
+#include <memory> // shared_ptr
 #include <vector>
 #include <unordered_map>
 #include <stdexcept>
@@ -47,13 +47,22 @@ class FairMQTransportFactory
     fair::mq::ChannelResource* GetMemoryResource() { return &fMemoryResource; }
     operator fair::mq::ChannelResource*() { return &fMemoryResource; }
 
-    /// @brief Create empty FairMQMessage
+    /// @brief Create empty FairMQMessage (for receiving)
     /// @return pointer to FairMQMessage
     virtual FairMQMessagePtr CreateMessage() = 0;
+    /// @brief Create empty FairMQMessage (for receiving), align received buffer to specified alignment
+    /// @param alignment alignment to align received buffer to
+    /// @return pointer to FairMQMessage
+    virtual FairMQMessagePtr CreateMessage(fair::mq::Alignment alignment) = 0;
     /// @brief Create new FairMQMessage of specified size
     /// @param size message size
     /// @return pointer to FairMQMessage
     virtual FairMQMessagePtr CreateMessage(const size_t size) = 0;
+    /// @brief Create new FairMQMessage of specified size and alignment
+    /// @param size message size
+    /// @param alignment message alignment
+    /// @return pointer to FairMQMessage
+    virtual FairMQMessagePtr CreateMessage(const size_t size, fair::mq::Alignment alignment) = 0;
     /// @brief Create new FairMQMessage with user provided buffer and size
     /// @param data pointer to user provided buffer
     /// @param size size of the user provided buffer
@@ -84,7 +93,8 @@ class FairMQTransportFactory
     /// @param path optional parameter to pass to the underlying transport
     /// @param flags optional parameter to pass to the underlying transport
     /// @return pointer to UnmanagedRegion
-    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const = 0;
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) = 0;
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) = 0;
     /// @brief Create new UnmanagedRegion
     /// @param size size of the region
     /// @param userFlags flags to be stored with the region, have no effect on the transport, but can be retrieved from the region by the user
@@ -92,11 +102,15 @@ class FairMQTransportFactory
     /// @param path optional parameter to pass to the underlying transport
     /// @param flags optional parameter to pass to the underlying transport
     /// @return pointer to UnmanagedRegion
-    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const = 0;
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) = 0;
+    virtual FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) = 0;
 
     /// @brief Subscribe to region events (creation, destruction, ...)
     /// @param callback the callback that is called when a region event occurs
     virtual void SubscribeToRegionEvents(FairMQRegionEventCallback callback) = 0;
+    /// @brief Check if there is an active subscription to region events
+    /// @return true/false
+    virtual bool SubscribedToRegionEvents() = 0;
     /// @brief Unsubscribe from region events
     virtual void UnsubscribeFromRegionEvents() = 0;
 

fairmq/FairMQUnmanagedRegion.h

@@ -13,14 +13,35 @@
 #include <memory> // std::unique_ptr
 #include <functional> // std::function
 #include <ostream> // std::ostream
+#include <vector>
+
+class FairMQTransportFactory;
 
 enum class FairMQRegionEvent : int
 {
     created,
-    destroyed
+    destroyed,
+    local_only
 };
 
-struct FairMQRegionInfo {
+struct FairMQRegionInfo
+{
+    FairMQRegionInfo()
+        : id(0)
+        , ptr(nullptr)
+        , size(0)
+        , flags(0)
+        , event(FairMQRegionEvent::created)
+    {}
+
+    FairMQRegionInfo(uint64_t _id, void* _ptr, size_t _size, int64_t _flags, FairMQRegionEvent _event)
+        : id(_id)
+        , ptr(_ptr)
+        , size(_size)
+        , flags(_flags)
+        , event (_event)
+    {}
+
     uint64_t id;   // id of the region
     void* ptr;     // pointer to the start of the region
     size_t size;   // region size
@@ -28,26 +49,52 @@ struct FairMQRegionInfo {
     FairMQRegionEvent event;
 };
 
+struct FairMQRegionBlock {
+    void* ptr;
+    size_t size;
+    void* hint;
+
+    FairMQRegionBlock(void* p, size_t s, void* h)
+        : ptr(p), size(s), hint(h)
+    {}
+};
+
 using FairMQRegionCallback = std::function<void(void*, size_t, void*)>;
+using FairMQRegionBulkCallback = std::function<void(const std::vector<FairMQRegionBlock>&)>;
 using FairMQRegionEventCallback = std::function<void(FairMQRegionInfo)>;
 
 class FairMQUnmanagedRegion
 {
   public:
+    FairMQUnmanagedRegion() {}
+    FairMQUnmanagedRegion(FairMQTransportFactory* factory) : fTransport(factory) {}
+
     virtual void* GetData() const = 0;
     virtual size_t GetSize() const = 0;
+    virtual uint64_t GetId() const = 0;
+
+    FairMQTransportFactory* GetTransport() { return fTransport; }
+    void SetTransport(FairMQTransportFactory* transport) { fTransport = transport; }
 
     virtual ~FairMQUnmanagedRegion() {};
+
+  private:
+    FairMQTransportFactory* fTransport{nullptr};
 };
 
 using FairMQUnmanagedRegionPtr = std::unique_ptr<FairMQUnmanagedRegion>;
 
 inline std::ostream& operator<<(std::ostream& os, const FairMQRegionEvent& event)
 {
-    if (event == FairMQRegionEvent::created) {
+    switch (event) {
+        case FairMQRegionEvent::created:
             return os << "created";
-    } else {
+        case FairMQRegionEvent::destroyed:
             return os << "destroyed";
+        case FairMQRegionEvent::local_only:
+            return os << "local_only";
+        default:
+            return os << "unrecognized event";
     }
 }
 
@@ -57,9 +104,11 @@ namespace mq
 {
 
 using RegionCallback = FairMQRegionCallback;
+using RegionBulkCallback = FairMQRegionBulkCallback;
 using RegionEventCallback = FairMQRegionEventCallback;
 using RegionEvent = FairMQRegionEvent;
 using RegionInfo = FairMQRegionInfo;
+using RegionBlock = FairMQRegionBlock;
 using UnmanagedRegion = FairMQUnmanagedRegion;
 using UnmanagedRegionPtr = FairMQUnmanagedRegionPtr;
 

fairmq/ProgOptions.cxx

@@ -117,16 +117,22 @@ void ProgOptions::ParseAll(const int argc, char const* const* argv, bool allowUn
     // clear the container because it was filled with default values and subsequent calls to store() do not overwrite the existing values
     fVarMap.clear();
 
-    if (allowUnregistered) {
+
     po::command_line_parser parser(argc, argv);
+
+    if (allowUnregistered) {
         parser.options(fAllOptions).allow_unregistered();
+    }
+
+    using namespace po::command_line_style;
+    style_t style = style_t(allow_short | short_allow_adjacent | short_allow_next | allow_long | long_allow_adjacent | long_allow_next | allow_sticky | allow_dash_for_short);
+    parser.style(style);
+
     po::parsed_options parsed = parser.run();
+
     fUnregisteredOptions = po::collect_unrecognized(parsed.options, po::include_positional);
 
     po::store(parsed, fVarMap);
-    } else {
-        po::store(po::parse_command_line(argc, argv, fAllOptions), fVarMap);
-    }
 }
 
 void ProgOptions::Notify()

fairmq/Transports.h

@@ -24,7 +24,6 @@ enum class Transport
 {
     DEFAULT,
     ZMQ,
-    NN,
     SHM,
     OFI
 };
@@ -48,7 +47,6 @@ namespace mq
 static std::unordered_map<std::string, Transport> TransportTypes {
     { "default", Transport::DEFAULT },
     { "zeromq", Transport::ZMQ },
-    { "nanomsg", Transport::NN },
     { "shmem", Transport::SHM },
     { "ofi", Transport::OFI }
 };
@@ -56,7 +54,6 @@ static std::unordered_map<std::string, Transport> TransportTypes {
 static std::unordered_map<Transport, std::string> TransportNames {
     { Transport::DEFAULT, "default" },
     { Transport::ZMQ, "zeromq" },
-    { Transport::NN, "nanomsg" },
     { Transport::SHM, "shmem" },
     { Transport::OFI, "ofi" }
 };

fairmq/devices/FairMQBenchmarkSampler.cxx

@@ -1,100 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQBenchmarkSampler.h"
-
-#include "tools/RateLimit.h"
-#include "../FairMQLogger.h"
-
-#include <chrono>
-
-using namespace std;
-
-FairMQBenchmarkSampler::FairMQBenchmarkSampler()
-    : fMultipart(false)
-    , fNumParts(1)
-    , fMsgSize(10000)
-    , fMsgRate(0)
-    , fNumIterations(0)
-    , fMaxIterations(0)
-    , fOutChannelName()
-{
-}
-
-void FairMQBenchmarkSampler::InitTask()
-{
-    fMultipart = fConfig->GetProperty<bool>("multipart");
-    fNumParts = fConfig->GetProperty<size_t>("num-parts");
-    fMsgSize = fConfig->GetProperty<size_t>("msg-size");
-    fMsgRate = fConfig->GetProperty<float>("msg-rate");
-    fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
-    fOutChannelName = fConfig->GetProperty<string>("out-channel");
-}
-
-void FairMQBenchmarkSampler::Run()
-{
-    // store the channel reference to avoid traversing the map on every loop iteration
-    FairMQChannel& dataOutChannel = fChannels.at(fOutChannelName).at(0);
-
-    FairMQMessagePtr baseMsg(dataOutChannel.NewMessage(fMsgSize));
-
-    LOG(info) << "Starting the benchmark with message size of " << fMsgSize << " and " << fMaxIterations << " iterations.";
-    auto tStart = chrono::high_resolution_clock::now();
-
-    fair::mq::tools::RateLimiter rateLimiter(fMsgRate);
-
-    while (!NewStatePending())
-    {
-        if (fMultipart)
-        {
-            FairMQParts parts;
-
-            for (size_t i = 0; i < fNumParts; ++i)
-            {
-                parts.AddPart(dataOutChannel.NewMessage(fMsgSize));
-            }
-
-            if (dataOutChannel.Send(parts) >= 0)
-            {
-                if (fMaxIterations > 0)
-                {
-                    if (fNumIterations >= fMaxIterations)
-                    {
-                        break;
-                    }
-                }
-                ++fNumIterations;
-            }
-        }
-        else
-        {
-            FairMQMessagePtr msg(dataOutChannel.NewMessage(fMsgSize));
-
-            if (dataOutChannel.Send(msg) >= 0)
-            {
-                if (fMaxIterations > 0)
-                {
-                    if (fNumIterations >= fMaxIterations)
-                    {
-                        break;
-                    }
-                }
-                ++fNumIterations;
-            }
-        }
-
-        if (fMsgRate > 0)
-        {
-            rateLimiter.maybe_sleep();
-        }
-    }
-
-    auto tEnd = chrono::high_resolution_clock::now();
-
-    LOG(info) << "Done " << fNumIterations << " iterations in " << chrono::duration<double, milli>(tEnd - tStart).count() << "ms.";
-}

fairmq/devices/FairMQBenchmarkSampler.h

@@ -9,13 +9,15 @@
 #ifndef FAIRMQBENCHMARKSAMPLER_H_
 #define FAIRMQBENCHMARKSAMPLER_H_
 
-#include <string>
+#include "../FairMQLogger.h"
+#include "FairMQDevice.h"
+#include "tools/RateLimit.h"
+
 #include <atomic>
+#include <chrono>
 #include <cstddef>   // size_t
 #include <cstdint>   // uint64_t
-
-
-#include "FairMQDevice.h"
+#include <string>
 
 /**
  * Sampler to generate traffic for benchmarking.
@@ -24,7 +26,77 @@
 class FairMQBenchmarkSampler : public FairMQDevice
 {
   public:
-    FairMQBenchmarkSampler();
+    FairMQBenchmarkSampler()
+        : fMultipart(false)
+        , fNumParts(1)
+        , fMsgSize(10000)
+        , fMsgRate(0)
+        , fNumIterations(0)
+        , fMaxIterations(0)
+        , fOutChannelName()
+    {}
+
+    void InitTask() override
+    {
+        fMultipart = fConfig->GetProperty<bool>("multipart");
+        fNumParts = fConfig->GetProperty<size_t>("num-parts");
+        fMsgSize = fConfig->GetProperty<size_t>("msg-size");
+        fMsgRate = fConfig->GetProperty<float>("msg-rate");
+        fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
+        fOutChannelName = fConfig->GetProperty<std::string>("out-channel");
+    }
+
+    void Run() override
+    {
+        // store the channel reference to avoid traversing the map on every loop iteration
+        FairMQChannel& dataOutChannel = fChannels.at(fOutChannelName).at(0);
+
+        FairMQMessagePtr baseMsg(dataOutChannel.NewMessage(fMsgSize));
+
+        LOG(info) << "Starting the benchmark with message size of " << fMsgSize << " and " << fMaxIterations << " iterations.";
+        auto tStart = std::chrono::high_resolution_clock::now();
+
+        fair::mq::tools::RateLimiter rateLimiter(fMsgRate);
+
+        while (!NewStatePending()) {
+            if (fMultipart) {
+                FairMQParts parts;
+
+                for (size_t i = 0; i < fNumParts; ++i) {
+                    parts.AddPart(dataOutChannel.NewMessage(fMsgSize));
+                }
+
+                if (dataOutChannel.Send(parts) >= 0) {
+                    if (fMaxIterations > 0) {
+                        if (fNumIterations >= fMaxIterations) {
+                            break;
+                        }
+                    }
+                    ++fNumIterations;
+                }
+            } else {
+                FairMQMessagePtr msg(dataOutChannel.NewMessage(fMsgSize));
+
+                if (dataOutChannel.Send(msg) >= 0) {
+                    if (fMaxIterations > 0) {
+                        if (fNumIterations >= fMaxIterations) {
+                            break;
+                        }
+                    }
+                    ++fNumIterations;
+                }
+            }
+
+            if (fMsgRate > 0) {
+                rateLimiter.maybe_sleep();
+            }
+        }
+
+        auto tEnd = std::chrono::high_resolution_clock::now();
+
+        LOG(info) << "Done " << fNumIterations << " iterations in " << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << "ms.";
+    }
+
     virtual ~FairMQBenchmarkSampler() {}
 
   protected:
@@ -36,9 +108,6 @@ class FairMQBenchmarkSampler : public FairMQDevice
     uint64_t fNumIterations;
     uint64_t fMaxIterations;
     std::string fOutChannelName;
-
-    virtual void InitTask() override;
-    virtual void Run() override;
 };
 
 #endif /* FAIRMQBENCHMARKSAMPLER_H_ */

fairmq/devices/FairMQSink.h

@@ -15,14 +15,14 @@
 #ifndef FAIRMQSINK_H_
 #define FAIRMQSINK_H_
 
-#include <string>
-#include <chrono>
-
 #include "../FairMQDevice.h"
 #include "../FairMQLogger.h"
 
+#include <chrono>
+#include <string>
+
 // template<typename OutputPolicy>
-class FairMQSink : public FairMQDevice//, public OutputPolicy
+class FairMQSink : public FairMQDevice   //, public OutputPolicy
 {
   public:
     FairMQSink()
@@ -32,8 +32,7 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
         , fInChannelName()
     {}
 
-    virtual ~FairMQSink()
-    {}
+    virtual ~FairMQSink() {}
 
   protected:
     bool fMultipart;
@@ -56,35 +55,25 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
         LOG(info) << "Starting the benchmark and expecting to receive " << fMaxIterations << " messages.";
         auto tStart = std::chrono::high_resolution_clock::now();
 
-        while (!NewStatePending())
-        {
-            if (fMultipart)
-            {
+        while (!NewStatePending()) {
+            if (fMultipart) {
                 FairMQParts parts;
 
-                if (dataInChannel.Receive(parts) >= 0)
-                {
-                    if (fMaxIterations > 0)
-                    {
-                        if (fNumIterations >= fMaxIterations)
-                        {
+                if (dataInChannel.Receive(parts) >= 0) {
+                    if (fMaxIterations > 0) {
+                        if (fNumIterations >= fMaxIterations) {
                             LOG(info) << "Configured maximum number of iterations reached.";
                             break;
                         }
                     }
                     fNumIterations++;
                 }
-            }
-            else
-            {
+            } else {
                 FairMQMessagePtr msg(dataInChannel.NewMessage());
 
-                if (dataInChannel.Receive(msg) >= 0)
-                {
-                    if (fMaxIterations > 0)
-                    {
-                        if (fNumIterations >= fMaxIterations)
-                        {
+                if (dataInChannel.Receive(msg) >= 0) {
+                    if (fMaxIterations > 0) {
+                        if (fNumIterations >= fMaxIterations) {
                             LOG(info) << "Configured maximum number of iterations reached.";
                             break;
                         }
@@ -96,7 +85,8 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
 
         auto tEnd = std::chrono::high_resolution_clock::now();
 
-        LOG(info) << "Leaving RUNNING state. Received " << fNumIterations << " messages in " << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << "ms.";
+        LOG(info) << "Leaving RUNNING state. Received " << fNumIterations << " messages in "
+                  << std::chrono::duration<double, std::milli>(tEnd - tStart).count() << "ms.";
     }
 };
 

fairmq/nanomsg/FairMQMessageNN.cxx

@@ -1,227 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQMessageNN.cxx
- *
- * @since 2013-12-05
- * @author A. Rybalchenko
- */
-
-#include <cstring>
-#include <stdlib.h>
-
-#include <nanomsg/nn.h>
-
-#include "FairMQMessageNN.h"
-#include "FairMQLogger.h"
-
-using namespace std;
-
-fair::mq::Transport FairMQMessageNN::fTransportType = fair::mq::Transport::NN;
-
-FairMQMessageNN::FairMQMessageNN(FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fMessage(nullptr)
-    , fSize(0)
-    , fHint(0)
-    , fReceiving(false)
-    , fRegionPtr(nullptr)
-{
-    fMessage = nn_allocmsg(0, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-}
-
-FairMQMessageNN::FairMQMessageNN(const size_t size, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fMessage(nullptr)
-    , fSize(0)
-    , fHint(0)
-    , fReceiving(false)
-    , fRegionPtr(nullptr)
-{
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    fSize = size;
-}
-
-
-/* nanomsg does not offer support for creating a message out of an existing buffer,
- * therefore the following method is using memcpy. For more efficient handling,
- * create FairMQMessage object only with size parameter and fill it with data.
- * possible TODO: make this zero copy (will should then be as efficient as ZeroMQ).
-*/
-FairMQMessageNN::FairMQMessageNN(void* data, const size_t size, fairmq_free_fn* ffn, void* hint, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fMessage(nullptr)
-    , fSize(0)
-    , fHint(0)
-    , fReceiving(false)
-    , fRegionPtr(nullptr)
-{
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    else
-    {
-        memcpy(fMessage, data, size);
-        fSize = size;
-        if (ffn)
-        {
-            ffn(data, hint);
-        }
-        else
-        {
-            free(data);
-        }
-    }
-}
-
-FairMQMessageNN::FairMQMessageNN(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fMessage(data)
-    , fSize(size)
-    , fHint(reinterpret_cast<size_t>(hint))
-    , fReceiving(false)
-    , fRegionPtr(region.get())
-{
-    // currently nanomsg will copy the buffer (data) inside nn_sendmsg()
-}
-
-void FairMQMessageNN::Rebuild()
-{
-    CloseMessage();
-    fReceiving = false;
-}
-
-void FairMQMessageNN::Rebuild(const size_t size)
-{
-    CloseMessage();
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    fSize = size;
-    fReceiving = false;
-}
-
-void FairMQMessageNN::Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    CloseMessage();
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    else
-    {
-        memcpy(fMessage, data, size);
-        fSize = size;
-        fReceiving = false;
-
-        if (ffn)
-        {
-            ffn(data, hint);
-        }
-    }
-}
-
-void* FairMQMessageNN::GetMessage() const
-{
-    return fMessage;
-}
-
-void* FairMQMessageNN::GetData() const
-{
-    return fMessage;
-}
-
-size_t FairMQMessageNN::GetSize() const
-{
-    return fSize;
-}
-
-bool FairMQMessageNN::SetUsedSize(const size_t size)
-{
-    if (size <= fSize)
-    {
-        // with size smaller than original nanomsg will simply "chop" the data, avoiding reallocation
-        fMessage = nn_reallocmsg(fMessage, size);
-        fSize = size;
-        return true;
-    }
-    else
-    {
-        LOG(error) << "cannot set used size higher than original.";
-        return false;
-    }
-}
-
-void FairMQMessageNN::SetMessage(void* data, const size_t size)
-{
-    fMessage = data;
-    fSize = size;
-}
-
-fair::mq::Transport FairMQMessageNN::GetType() const
-{
-    return fTransportType;
-}
-
-void FairMQMessageNN::Copy(const FairMQMessage& msg)
-{
-    if (fMessage)
-    {
-        if (nn_freemsg(fMessage) < 0)
-        {
-            LOG(error) << "failed freeing message, reason: " << nn_strerror(errno);
-        }
-    }
-
-    size_t size = msg.GetSize();
-
-    fMessage = nn_allocmsg(size, 0);
-    if (!fMessage)
-    {
-        LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
-    }
-    else
-    {
-        memcpy(fMessage, static_cast<const FairMQMessageNN&>(msg).GetMessage(), size);
-        fSize = size;
-    }
-}
-
-void FairMQMessageNN::CloseMessage()
-{
-    if (nn_freemsg(fMessage) < 0)
-    {
-        LOG(error) << "failed freeing message, reason: " << nn_strerror(errno);
-    }
-    else
-    {
-        fMessage = nullptr;
-        fSize = 0;
-    }
-}
-
-FairMQMessageNN::~FairMQMessageNN()
-{
-    if (fReceiving)
-    {
-        CloseMessage();
-    }
-}

fairmq/nanomsg/FairMQMessageNN.h

@@ -1,68 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQMessageNN.h
- *
- * @since 2013-12-05
- * @author A. Rybalchenko
- */
-
-#ifndef FAIRMQMESSAGENN_H_
-#define FAIRMQMESSAGENN_H_
-
-#include <cstddef>
-#include <string>
-#include <memory>
-
-#include "FairMQMessage.h"
-#include "FairMQUnmanagedRegion.h"
-
-class FairMQSocketNN;
-
-class FairMQMessageNN final : public FairMQMessage
-{
-    friend class FairMQSocketNN;
-
-  public:
-    FairMQMessageNN(FairMQTransportFactory* factory = nullptr);
-    FairMQMessageNN(const size_t size, FairMQTransportFactory* factory = nullptr);
-    FairMQMessageNN(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr);
-    FairMQMessageNN(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr);
-
-    FairMQMessageNN(const FairMQMessageNN&) = delete;
-    FairMQMessageNN operator=(const FairMQMessageNN&) = delete;
-
-    void Rebuild() override;
-    void Rebuild(const size_t size) override;
-    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-
-    void* GetData() const override;
-    size_t GetSize() const override;
-
-    bool SetUsedSize(const size_t size) override;
-
-    fair::mq::Transport GetType() const override;
-
-    void Copy(const FairMQMessage& msg) override;
-
-    ~FairMQMessageNN() override;
-
-  private:
-    void* fMessage;
-    size_t fSize;
-    size_t fHint;
-    bool fReceiving;
-    FairMQUnmanagedRegion* fRegionPtr;
-    static fair::mq::Transport fTransportType;
-
-    void* GetMessage() const;
-    void CloseMessage();
-    void SetMessage(void* data, const size_t size);
-};
-
-#endif /* FAIRMQMESSAGENN_H_ */

fairmq/nanomsg/FairMQPollerNN.cxx

@@ -1,207 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQPollerNN.cxx
- *
- * @since 2014-01-23
- * @author A. Rybalchenko
- */
-
-#include <nanomsg/nn.h>
-#include <nanomsg/pipeline.h>
-#include <nanomsg/pubsub.h>
-#include <nanomsg/reqrep.h>
-#include <nanomsg/pair.h>
-
-#include "FairMQPollerNN.h"
-#include "FairMQSocketNN.h"
-#include "FairMQLogger.h"
-
-using namespace std;
-
-FairMQPollerNN::FairMQPollerNN(const vector<FairMQChannel>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new nn_pollfd[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].fd = static_cast<const FairMQSocketNN*>(&(channels.at(i).GetSocket()))->GetSocket();
-
-        int type = 0;
-        size_t sz = sizeof(type);
-        nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channels.at(i).GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-FairMQPollerNN::FairMQPollerNN(const vector<FairMQChannel*>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new nn_pollfd[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].fd = static_cast<const FairMQSocketNN*>(&(channels.at(i)->GetSocket()))->GetSocket();
-
-        int type = 0;
-        size_t sz = sizeof(type);
-        nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channels.at(i)->GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-FairMQPollerNN::FairMQPollerNN(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    try
-    {
-        int offset = 0;
-        // calculate offsets and the total size of the poll item set
-        for (string channel : channelList)
-        {
-            fOffsetMap[channel] = offset;
-            offset += channelsMap.at(channel).size();
-            fNumItems += channelsMap.at(channel).size();
-        }
-
-        fItems = new nn_pollfd[fNumItems];
-
-        int index = 0;
-        for (string channel : channelList)
-        {
-            for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i)
-            {
-                index = fOffsetMap[channel] + i;
-                fItems[index].fd = static_cast<const FairMQSocketNN*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
-
-                int type = 0;
-                size_t sz = sizeof(type);
-                nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
-
-                SetItemEvents(fItems[index], type);
-            }
-        }
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "at least one of the provided channel keys for poller initialization is invalid";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-void FairMQPollerNN::SetItemEvents(nn_pollfd& item, const int type)
-{
-    if (type == NN_REQ || type == NN_REP || type == NN_PAIR)
-    {
-        item.events = NN_POLLIN|NN_POLLOUT;
-    }
-    else if (type == NN_PUSH || type == NN_PUB)
-    {
-        item.events = NN_POLLOUT;
-    }
-    else if (type == NN_PULL || type == NN_SUB)
-    {
-        item.events = NN_POLLIN;
-    }
-    else
-    {
-        LOG(error) << "invalid poller configuration, exiting.";
-        exit(EXIT_FAILURE);
-    }
-}
-
-void FairMQPollerNN::Poll(const int timeout)
-{
-    if (nn_poll(fItems, fNumItems, timeout) < 0)
-    {
-        if (errno == ETERM)
-        {
-            LOG(debug) << "polling exited, reason: " << nn_strerror(errno);
-        }
-        else
-        {
-            LOG(error) << "polling failed, reason: " << nn_strerror(errno);
-            throw std::runtime_error("polling failed");
-        }
-    }
-}
-
-bool FairMQPollerNN::CheckInput(const int index)
-{
-    if (fItems[index].revents & (NN_POLLIN | NN_POLLOUT))
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool FairMQPollerNN::CheckOutput(const int index)
-{
-    if (fItems[index].revents & NN_POLLOUT)
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool FairMQPollerNN::CheckInput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & (NN_POLLIN | NN_POLLOUT))
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-bool FairMQPollerNN::CheckOutput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & NN_POLLOUT)
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-FairMQPollerNN::~FairMQPollerNN()
-{
-    delete[] fItems;
-}

fairmq/nanomsg/FairMQPollerNN.h

@@ -1,58 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQPollerNN.h
- *
- * @since 2014-01-23
- * @author A. Rybalchenko
- */
-
-#ifndef FAIRMQPOLLERNN_H_
-#define FAIRMQPOLLERNN_H_
-
-#include <vector>
-#include <unordered_map>
-
-#include "FairMQPoller.h"
-#include "FairMQChannel.h"
-#include "FairMQTransportFactoryNN.h"
-
-class FairMQChannel;
-struct nn_pollfd;
-
-class FairMQPollerNN final : public FairMQPoller
-{
-    friend class FairMQChannel;
-    friend class FairMQTransportFactoryNN;
-
-  public:
-    FairMQPollerNN(const std::vector<FairMQChannel>& channels);
-    FairMQPollerNN(const std::vector<FairMQChannel*>& channels);
-    FairMQPollerNN(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList);
-
-    FairMQPollerNN(const FairMQPollerNN&) = delete;
-    FairMQPollerNN operator=(const FairMQPollerNN&) = delete;
-
-    void SetItemEvents(nn_pollfd& item, const int type);
-
-    void Poll(const int timeout) override;
-    bool CheckInput(const int index) override;
-    bool CheckOutput(const int index) override;
-    bool CheckInput(const std::string& channelKey, const int index) override;
-    bool CheckOutput(const std::string& channelKey, const int index) override;
-
-    ~FairMQPollerNN() override;
-
-  private:
-    nn_pollfd* fItems;
-    int fNumItems;
-
-    std::unordered_map<std::string, int> fOffsetMap;
-};
-
-#endif /* FAIRMQPOLLERNN_H_ */

fairmq/nanomsg/FairMQSocketNN.cxx

@@ -1,628 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQSocketNN.cxx
- *
- * @since 2012-12-05
- * @author A. Rybalchenko
- */
-
-#include "FairMQSocketNN.h"
-#include "FairMQMessageNN.h"
-#include "FairMQLogger.h"
-#include "FairMQUnmanagedRegionNN.h"
-#include <fairmq/Tools.h>
-
-#include <nanomsg/nn.h>
-#include <nanomsg/pipeline.h>
-#include <nanomsg/pubsub.h>
-#include <nanomsg/reqrep.h>
-#include <nanomsg/pair.h>
-
-#include <sstream>
-#include <msgpack.hpp>
-
-using namespace std;
-using namespace fair::mq;
-
-atomic<bool> FairMQSocketNN::fInterrupted(false);
-
-FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const string& id /*= ""*/, FairMQTransportFactory* fac /*=nullptr*/)
-    : FairMQSocket{fac}
-    , fSocket(-1)
-    , fId(id + "." + name + "." + type)
-    , fBytesTx(0)
-    , fBytesRx(0)
-    , fMessagesTx(0)
-    , fMessagesRx(0)
-    , fSndTimeout(100)
-    , fRcvTimeout(100)
-    , fLinger(500)
-{
-    if (type == "router" || type == "dealer")
-    {
-        // Additional info about using the sockets ROUTER and DEALER with nanomsg can be found in:
-        // http://250bpm.com/blog:14
-        // http://www.freelists.org/post/nanomsg/a-stupid-load-balancing-question,1
-        fSocket = nn_socket(AF_SP_RAW, GetConstant(type));
-        if (fSocket == -1)
-        {
-            LOG(error) << "failed creating socket " << fId << ", reason: " << nn_strerror(errno);
-            exit(EXIT_FAILURE);
-        }
-    }
-    else
-    {
-        fSocket = nn_socket(AF_SP, GetConstant(type));
-        if (fSocket == -1)
-        {
-            LOG(error) << "failed creating socket " << fId << ", reason: " << nn_strerror(errno);
-            exit(EXIT_FAILURE);
-        }
-        if (type == "sub")
-        {
-            nn_setsockopt(fSocket, NN_SUB, NN_SUB_SUBSCRIBE, nullptr, 0);
-        }
-    }
-
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0)
-    {
-        LOG(error) << "Failed setting NN_SNDTIMEO socket option, reason: " << nn_strerror(errno);
-    }
-
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0)
-    {
-        LOG(error) << "Failed setting NN_RCVTIMEO socket option, reason: " << nn_strerror(errno);
-    }
-
-#ifdef NN_RCVMAXSIZE
-    int rcvSize = -1;
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVMAXSIZE, &rcvSize, sizeof(rcvSize)) != 0)
-    {
-        LOG(error) << "Failed setting NN_RCVMAXSIZE socket option, reason: " << nn_strerror(errno);
-    }
-#endif
-
-    LOG(debug) << "Created socket " << GetId();
-}
-
-bool FairMQSocketNN::Bind(const string& address)
-{
-    // LOG(info) << "bind socket " << fId << " on " << address;
-
-    if (nn_bind(fSocket, address.c_str()) < 0)
-    {
-        LOG(error) << "failed binding socket " << fId << ", reason: " << nn_strerror(errno);
-        return false;
-    }
-
-    return true;
-}
-
-bool FairMQSocketNN::Connect(const string& address)
-{
-    // LOG(info) << "connect socket " << fId << " to " << address;
-
-    if (nn_connect(fSocket, address.c_str()) < 0)
-    {
-        LOG(error) << "failed connecting socket " << fId << ", reason: " << nn_strerror(errno);
-        return false;
-    }
-
-    return true;
-}
-
-int FairMQSocketNN::Send(FairMQMessagePtr& msg, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0)
-    {
-        flags = NN_DONTWAIT;
-    }
-    int nbytes = -1;
-    int elapsed = 0;
-
-    FairMQMessageNN* msgPtr = static_cast<FairMQMessageNN*>(msg.get());
-    void* bufPtr = msgPtr->GetMessage();
-
-    while (true)
-    {
-        if (msgPtr->fRegionPtr == nullptr)
-        {
-            nbytes = nn_send(fSocket, &bufPtr, NN_MSG, flags);
-        }
-        else
-        {
-            nbytes = nn_send(fSocket, bufPtr, msg->GetSize(), flags);
-            // nn_send copies the data, safe to call region callback here
-            static_cast<FairMQUnmanagedRegionNN*>(msgPtr->fRegionPtr)->fCallback(bufPtr, msg->GetSize(), reinterpret_cast<void*>(msgPtr->fHint));
-        }
-
-        if (nbytes >= 0)
-        {
-            fBytesTx += nbytes;
-            ++fMessagesTx;
-            static_cast<FairMQMessageNN*>(msg.get())->fReceiving = false;
-
-            return nbytes;
-        }
-        else if (nn_errno() == ETIMEDOUT)
-        {
-            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
-                    elapsed += fSndTimeout;
-                    if (elapsed >= timeout)
-                    {
-                        return -2;
-                    }
-                }
-                continue;
-            }
-            else
-            {
-                return -2;
-            }
-        }
-        else if (nn_errno() == EAGAIN)
-        {
-            return -2;
-        }
-        else if (nn_errno() == ETERM)
-        {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        }
-        else
-        {
-            LOG(error) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-int FairMQSocketNN::Receive(FairMQMessagePtr& msg, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0)
-    {
-        flags = NN_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    FairMQMessageNN* msgPtr = static_cast<FairMQMessageNN*>(msg.get());
-
-    while (true)
-    {
-        void* ptr = nullptr;
-        int nbytes = nn_recv(fSocket, &ptr, NN_MSG, flags);
-        if (nbytes >= 0)
-        {
-            fBytesRx += nbytes;
-            ++fMessagesRx;
-            msgPtr->SetMessage(ptr, nbytes);
-            msgPtr->fReceiving = true;
-            return nbytes;
-        }
-        else if (nn_errno() == ETIMEDOUT)
-        {
-            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
-                    elapsed += fRcvTimeout;
-                    if (elapsed >= timeout)
-                    {
-                        return -2;
-                    }
-                }
-                continue;
-            }
-            else
-            {
-                return -2;
-            }
-        }
-        else if (nn_errno() == EAGAIN)
-        {
-            return -2;
-        }
-        else if (nn_errno() == ETERM)
-        {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        }
-        else
-        {
-            LOG(error) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-int64_t FairMQSocketNN::Send(vector<FairMQMessagePtr>& msgVec, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0)
-    {
-        flags = NN_DONTWAIT;
-    }
-    const unsigned int vecSize = msgVec.size();
-    int elapsed = 0;
-
-    // create msgpack simple buffer
-    msgpack::sbuffer sbuf;
-    // create msgpack packer
-    msgpack::packer<msgpack::sbuffer> packer(&sbuf);
-
-    // pack all parts into a single msgpack simple buffer
-    for (unsigned int i = 0; i < vecSize; ++i)
-    {
-        FairMQMessageNN* partPtr = static_cast<FairMQMessageNN*>(msgVec[i].get());
-
-        partPtr->fReceiving = false;
-        packer.pack_bin(msgVec[i]->GetSize());
-        packer.pack_bin_body(static_cast<char*>(msgVec[i]->GetData()), msgVec[i]->GetSize());
-        // call region callback
-        if (partPtr->fRegionPtr)
-        {
-            static_cast<FairMQUnmanagedRegionNN*>(partPtr->fRegionPtr)->fCallback(partPtr->GetMessage(), partPtr->GetSize(), reinterpret_cast<void*>(partPtr->fHint));
-        }
-    }
-
-    while (true)
-    {
-        int64_t nbytes = nn_send(fSocket, sbuf.data(), sbuf.size(), flags);
-        if (nbytes >= 0)
-        {
-            fBytesTx += nbytes;
-            ++fMessagesTx;
-            return nbytes;
-        }
-        else if (nn_errno() == ETIMEDOUT)
-        {
-            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
-                    elapsed += fSndTimeout;
-                    if (elapsed >= timeout)
-                    {
-                        return -2;
-                    }
-                }
-                continue;
-            }
-            else
-            {
-                return -2;
-            }
-        }
-        else if (nn_errno() == EAGAIN)
-        {
-            return -2;
-        }
-        else if (nn_errno() == ETERM)
-        {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        }
-        else
-        {
-            LOG(error) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-int64_t FairMQSocketNN::Receive(vector<FairMQMessagePtr>& msgVec, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0)
-    {
-        flags = NN_DONTWAIT;
-    }
-    // Warn if the vector is filled before Receive() and empty it.
-    // if (msgVec.size() > 0)
-    // {
-    //     LOG(warn) << "Message vector contains elements before Receive(), they will be deleted!";
-    //     msgVec.clear();
-    // }
-
-    int elapsed = 0;
-
-    while (true)
-    {
-        // pointer to point to received message buffer
-        char* ptr = nullptr;
-        // receive the message into a buffer allocated by nanomsg and let ptr point to it
-        int nbytes = nn_recv(fSocket, &ptr, NN_MSG, flags);
-        if (nbytes >= 0) // if no errors or non-blocking timeouts
-        {
-            // store statistics on how many bytes received
-            fBytesRx += nbytes;
-            // store statistics on how many messages received (count messages instead of parts)
-            ++fMessagesRx;
-
-            // offset to be used by msgpack to handle separate chunks
-            size_t offset = 0;
-            while (offset != static_cast<size_t>(nbytes)) // continue until all parts have been read
-            {
-                // vector of chars to hold blob (unlike char*/void* this type can be converted to by msgpack)
-                vector<char> buf;
-
-                // unpack and convert chunk
-                msgpack::unpacked result;
-                unpack(result, ptr, nbytes, offset);
-                msgpack::object object(result.get());
-                object.convert(buf);
-                // get the single message size
-                size_t size = buf.size() * sizeof(char);
-                FairMQMessagePtr part(new FairMQMessageNN(size, GetTransport()));
-                static_cast<FairMQMessageNN*>(part.get())->fReceiving = true;
-                memcpy(part->GetData(), buf.data(), size);
-                msgVec.push_back(move(part));
-            }
-
-            nn_freemsg(ptr);
-            return nbytes;
-        }
-        else if (nn_errno() == ETIMEDOUT)
-        {
-            if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
-            {
-                if (timeout > 0)
-                {
-                    elapsed += fRcvTimeout;
-                    if (elapsed >= timeout)
-                    {
-                        return -2;
-                    }
-                }
-                continue;
-            }
-            else
-            {
-                return -2;
-            }
-        }
-        else if (nn_errno() == EAGAIN)
-        {
-            return -2;
-        }
-        else if (nn_errno() == ETERM)
-        {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        }
-        else
-        {
-            LOG(error) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-void FairMQSocketNN::Close()
-{
-    nn_close(fSocket);
-}
-
-void FairMQSocketNN::Interrupt()
-{
-    fInterrupted = true;
-}
-
-void FairMQSocketNN::Resume()
-{
-    fInterrupted = false;
-}
-
-int FairMQSocketNN::GetSocket() const
-{
-    return fSocket;
-}
-
-void FairMQSocketNN::SetOption(const string& option, const void* value, size_t valueSize)
-{
-    if (option == "snd-size" || option == "rcv-size")
-    {
-        int val = *(static_cast<int*>(const_cast<void*>(value)));
-        if (val <= 0)
-        {
-            LOG(warn) << "value for sndKernelSize/rcvKernelSize should be greater than 0, leaving unchanged.";
-            return;
-        }
-    }
-
-    if (option == "snd-hwm" || option == "rcv-hwm")
-    {
-        return;
-    }
-
-    if (option == "linger")
-    {
-        fLinger = *static_cast<int*>(const_cast<void*>(value));
-        return;
-    }
-
-    int rc = nn_setsockopt(fSocket, NN_SOL_SOCKET, GetConstant(option), value, valueSize);
-    if (rc < 0)
-    {
-        LOG(error) << "failed setting socket option, reason: " << nn_strerror(errno);
-    }
-}
-
-void FairMQSocketNN::GetOption(const string& option, void* value, size_t* valueSize)
-{
-    if (option == "linger")
-    {
-        *static_cast<int*>(value) = fLinger;
-        return;
-    }
-
-    if (option == "snd-hwm" || option == "rcv-hwm")
-    {
-        *static_cast<int*>(value) = -1;
-        return;
-    }
-
-
-    int rc = nn_getsockopt(fSocket, NN_SOL_SOCKET, GetConstant(option), value, valueSize);
-    if (rc < 0)
-    {
-        LOG(error) << "failed getting socket option, reason: " << nn_strerror(errno);
-    }
-}
-
-void FairMQSocketNN::SetLinger(const int value)
-{
-    fLinger = value;
-}
-
-int FairMQSocketNN::GetLinger() const
-{
-    return fLinger;
-}
-
-void FairMQSocketNN::SetSndBufSize(const int /* value */)
-{
-    // not used in nanomsg
-}
-
-int FairMQSocketNN::GetSndBufSize() const
-{
-    // not used in nanomsg
-    return -1;
-}
-
-void FairMQSocketNN::SetRcvBufSize(const int /* value */)
-{
-    // not used in nanomsg
-}
-
-int FairMQSocketNN::GetRcvBufSize() const
-{
-    // not used in nanomsg
-    return -1;
-}
-
-void FairMQSocketNN::SetSndKernelSize(const int value)
-{
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_SNDBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting NN_SNDBUF, reason: ", nn_strerror(errno)));
-    }
-}
-
-int FairMQSocketNN::GetSndKernelSize() const
-{
-    int value = 0;
-    size_t valueSize;
-    if (nn_getsockopt(fSocket, NN_SOL_SOCKET, NN_SNDBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting NN_SNDBUF, reason: ", nn_strerror(errno)));
-    }
-    return value;
-}
-
-void FairMQSocketNN::SetRcvKernelSize(const int value)
-{
-    if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting NN_RCVBUF, reason: ", nn_strerror(errno)));
-    }
-}
-
-int FairMQSocketNN::GetRcvKernelSize() const
-{
-    int value = 0;
-    size_t valueSize;
-    if (nn_getsockopt(fSocket, NN_SOL_SOCKET, NN_RCVBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting NN_RCVBUF, reason: ", nn_strerror(errno)));
-    }
-    return value;
-}
-
-
-unsigned long FairMQSocketNN::GetBytesTx() const
-{
-    return fBytesTx;
-}
-
-unsigned long FairMQSocketNN::GetBytesRx() const
-{
-    return fBytesRx;
-}
-
-unsigned long FairMQSocketNN::GetMessagesTx() const
-{
-    return fMessagesTx;
-}
-
-unsigned long FairMQSocketNN::GetMessagesRx() const
-{
-    return fMessagesRx;
-}
-
-int FairMQSocketNN::GetConstant(const string& constant)
-{
-    if (constant == "")
-        return 0;
-    if (constant == "sub")
-        return NN_SUB;
-    if (constant == "pub")
-        return NN_PUB;
-    if (constant == "xsub")
-        return NN_SUB;
-    if (constant == "xpub")
-        return NN_PUB;
-    if (constant == "push")
-        return NN_PUSH;
-    if (constant == "pull")
-        return NN_PULL;
-    if (constant == "req")
-        return NN_REQ;
-    if (constant == "rep")
-        return NN_REP;
-    if (constant == "dealer")
-        return NN_REQ;
-    if (constant == "router")
-        return NN_REP;
-    if (constant == "pair")
-        return NN_PAIR;
-
-    if (constant == "snd-hwm")
-        return NN_SNDBUF;
-    if (constant == "rcv-hwm")
-        return NN_RCVBUF;
-    if (constant == "snd-size")
-        return NN_SNDBUF;
-    if (constant == "rcv-size")
-        return NN_RCVBUF;
-    if (constant == "snd-more")
-    {
-        LOG(error) << "Multipart messages functionality currently not supported by nanomsg!";
-        return -1;
-    }
-    if (constant == "rcv-more")
-    {
-        LOG(error) << "Multipart messages functionality currently not supported by nanomsg!";
-        return -1;
-    }
-
-    if (constant == "linger")
-        return NN_LINGER;
-    if (constant == "no-block")
-        return NN_DONTWAIT;
-
-    return -1;
-}
-
-FairMQSocketNN::~FairMQSocketNN()
-{
-    Close();
-}

fairmq/nanomsg/FairMQSocketNN.h

@@ -1,81 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQSOCKETNN_H_
-#define FAIRMQSOCKETNN_H_
-
-#include <vector>
-#include <atomic>
-
-#include "FairMQSocket.h"
-#include "FairMQMessage.h"
-class FairMQTransportFactory;
-
-class FairMQSocketNN final : public FairMQSocket
-{
-  public:
-    FairMQSocketNN(const std::string& type, const std::string& name, const std::string& id = "", FairMQTransportFactory* fac = nullptr);
-    FairMQSocketNN(const FairMQSocketNN&) = delete;
-    FairMQSocketNN operator=(const FairMQSocketNN&) = delete;
-
-    std::string GetId() const override { return fId; }
-
-    bool Bind(const std::string& address) override;
-    bool Connect(const std::string& address) override;
-
-    int Send(FairMQMessagePtr& msg, const int timeout = -1) override;
-    int Receive(FairMQMessagePtr& msg, const int timeout = -1) override;
-    int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
-    int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
-
-    int GetSocket() const;
-
-    void Close() override;
-
-    static void Interrupt();
-    static void Resume();
-
-    void SetOption(const std::string& option, const void* value, size_t valueSize) override;
-    void GetOption(const std::string& option, void* value, size_t* valueSize) override;
-
-    void SetLinger(const int value) override;
-    int GetLinger() const override;
-    void SetSndBufSize(const int value) override;
-    int GetSndBufSize() const override;
-    void SetRcvBufSize(const int value) override;
-    int GetRcvBufSize() const override;
-    void SetSndKernelSize(const int value) override;
-    int GetSndKernelSize() const override;
-    void SetRcvKernelSize(const int value) override;
-    int GetRcvKernelSize() const override;
-
-    unsigned long GetBytesTx() const override;
-    unsigned long GetBytesRx() const override;
-    unsigned long GetMessagesTx() const override;
-    unsigned long GetMessagesRx() const override;
-
-    static int GetConstant(const std::string& constant);
-
-    ~FairMQSocketNN() override;
-
-  private:
-    int fSocket;
-    std::string fId;
-    std::atomic<unsigned long> fBytesTx;
-    std::atomic<unsigned long> fBytesRx;
-    std::atomic<unsigned long> fMessagesTx;
-    std::atomic<unsigned long> fMessagesRx;
-
-    static std::atomic<bool> fInterrupted;
-
-    int fSndTimeout;
-    int fRcvTimeout;
-    int fLinger;
-};
-
-#endif /* FAIRMQSOCKETNN_H_ */

fairmq/nanomsg/FairMQTransportFactoryNN.cxx

@@ -1,99 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2017 GSI Helmholtzzentrum fuer Schwerionenforschung Gmb   *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQTransportFactoryNN.h"
-
-#include <nanomsg/nn.h>
-#include <algorithm>
-#include <thread>
-#include <chrono>
-
-using namespace std;
-
-fair::mq::Transport FairMQTransportFactoryNN::fTransportType = fair::mq::Transport::NN;
-
-FairMQTransportFactoryNN::FairMQTransportFactoryNN(const string& id, const fair::mq::ProgOptions* /*config*/)
-    : FairMQTransportFactory(id)
-{
-    LOG(debug) << "Transport: Using nanomsg library";
-}
-
-FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage()
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageNN(this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(const size_t size)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageNN(size, this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageNN(data, size, ffn, hint, this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageNN(region, data, size, hint, this));
-}
-
-FairMQSocketPtr FairMQTransportFactoryNN::CreateSocket(const string& type, const string& name)
-{
-    unique_ptr<FairMQSocket> socket(new FairMQSocketNN(type, name, GetId(), this));
-    fSockets.push_back(socket.get());
-    return socket;
-}
-
-FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const vector<FairMQChannel>& channels) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerNN(channels));
-}
-
-FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const std::vector<FairMQChannel*>& channels) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerNN(channels));
-}
-
-FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerNN(channelsMap, channelList));
-}
-
-FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
-{
-    return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(size, callback, path, flags));
-}
-
-FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
-{
-    return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(size, userFlags, callback, path, flags));
-}
-
-fair::mq::Transport FairMQTransportFactoryNN::GetType() const
-{
-    return fTransportType;
-}
-
-void FairMQTransportFactoryNN::Reset()
-{
-    auto it = max_element(fSockets.begin(), fSockets.end(), [](FairMQSocket* s1, FairMQSocket* s2) {
-        return static_cast<FairMQSocketNN*>(s1)->GetLinger() < static_cast<FairMQSocketNN*>(s2)->GetLinger();
-    });
-    if (it != fSockets.end()) {
-        this_thread::sleep_for(chrono::milliseconds(static_cast<FairMQSocketNN*>(*it)->GetLinger()));
-    }
-    fSockets.clear();
-}
-
-FairMQTransportFactoryNN::~FairMQTransportFactoryNN()
-{
-    LOG(debug) << "Destroying Shared Memory transport...";
-    // nn_term();
-    // see https://www.freelists.org/post/nanomsg/Getting-rid-of-nn-init-and-nn-term,8
-}

fairmq/nanomsg/FairMQTransportFactoryNN.h

@@ -1,57 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQTRANSPORTFACTORYNN_H_
-#define FAIRMQTRANSPORTFACTORYNN_H_
-
-#include "FairMQTransportFactory.h"
-#include "FairMQMessageNN.h"
-#include "FairMQSocketNN.h"
-#include "FairMQPollerNN.h"
-#include "FairMQUnmanagedRegionNN.h"
-#include <fairmq/ProgOptions.h>
-
-#include <vector>
-#include <string>
-
-class FairMQTransportFactoryNN final : public FairMQTransportFactory
-{
-  public:
-    FairMQTransportFactoryNN(const std::string& id = "", const fair::mq::ProgOptions* config = nullptr);
-    ~FairMQTransportFactoryNN() override;
-
-    FairMQMessagePtr CreateMessage() override;
-    FairMQMessagePtr CreateMessage(const size_t size) override;
-    FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-    FairMQMessagePtr CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0) override;
-
-    FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name) override;
-
-    FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override;
-    FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const override;
-    FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
-
-    FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0) const override;
-    FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const override;
-
-    void SubscribeToRegionEvents(FairMQRegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for nanomsg"; }
-    void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for nanomsg"; }
-    std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for nanomsg, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
-
-    fair::mq::Transport GetType() const override;
-
-    void Interrupt() override { FairMQSocketNN::Interrupt(); }
-    void Resume() override { FairMQSocketNN::Resume(); }
-    void Reset() override;
-
-  private:
-    static fair::mq::Transport fTransportType;
-    mutable std::vector<FairMQSocket*> fSockets;
-};
-
-#endif /* FAIRMQTRANSPORTFACTORYNN_H_ */

fairmq/nanomsg/FairMQUnmanagedRegionNN.cxx

@@ -1,42 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQUnmanagedRegionNN.h"
-#include "FairMQLogger.h"
-
-using namespace std;
-
-FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(const size_t size, FairMQRegionCallback callback, const std::string& /*path = "" */, int /*flags = 0 */)
-    : fBuffer(malloc(size))
-    , fSize(size)
-    , fCallback(callback)
-{
-}
-
-FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(const size_t size, const int64_t /*userFlags*/, FairMQRegionCallback callback, const std::string& /*path = "" */, int /*flags = 0 */)
-    : fBuffer(malloc(size))
-    , fSize(size)
-    , fCallback(callback)
-{
-}
-
-void* FairMQUnmanagedRegionNN::GetData() const
-{
-    return fBuffer;
-}
-
-size_t FairMQUnmanagedRegionNN::GetSize() const
-{
-    return fSize;
-}
-
-FairMQUnmanagedRegionNN::~FairMQUnmanagedRegionNN()
-{
-    LOG(debug) << "destroying region";
-    free(fBuffer);
-}

fairmq/nanomsg/FairMQUnmanagedRegionNN.h

@@ -1,39 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQUNMANAGEDREGIONNN_H_
-#define FAIRMQUNMANAGEDREGIONNN_H_
-
-#include "FairMQUnmanagedRegion.h"
-
-#include <cstddef> // size_t
-#include <string>
-
-class FairMQUnmanagedRegionNN final : public FairMQUnmanagedRegion
-{
-    friend class FairMQSocketNN;
-
-  public:
-    FairMQUnmanagedRegionNN(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
-    FairMQUnmanagedRegionNN(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
-
-    FairMQUnmanagedRegionNN(const FairMQUnmanagedRegionNN&) = delete;
-    FairMQUnmanagedRegionNN operator=(const FairMQUnmanagedRegionNN&) = delete;
-
-    virtual void* GetData() const override;
-    virtual size_t GetSize() const override;
-
-    virtual ~FairMQUnmanagedRegionNN();
-
-  private:
-    void* fBuffer;
-    size_t fSize;
-    FairMQRegionCallback fCallback;
-};
-
-#endif /* FAIRMQUNMANAGEDREGIONNN_H_ */

fairmq/ofi/Message.cxx

@@ -34,6 +34,16 @@ Message::Message(boost::container::pmr::memory_resource* pmr)
 {
 }
 
+Message::Message(boost::container::pmr::memory_resource* pmr, Alignment /* alignment */)
+    : fInitialSize(0)
+    , fSize(0)
+    , fData(nullptr)
+    , fFreeFunction(nullptr)
+    , fHint(nullptr)
+    , fPmr(pmr)
+{
+}
+
 Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size)
     : fInitialSize(size)
     , fSize(size)
@@ -48,6 +58,20 @@ Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size)
     }
 }
 
+Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size, Alignment /* alignment */)
+    : fInitialSize(size)
+    , fSize(size)
+    , fData(nullptr)
+    , fFreeFunction(nullptr)
+    , fHint(nullptr)
+    , fPmr(pmr)
+{
+    if (size) {
+        fData = fPmr->allocate(size);
+        assert(fData);
+    }
+}
+
 Message::Message(boost::container::pmr::memory_resource* pmr,
                  void* data,
                  const size_t size,

fairmq/ofi/Message.h

@@ -34,7 +34,9 @@ class Message final : public fair::mq::Message
 {
   public:
     Message(boost::container::pmr::memory_resource* pmr);
+    Message(boost::container::pmr::memory_resource* pmr, Alignment alignment);
     Message(boost::container::pmr::memory_resource* pmr, const size_t size);
+    Message(boost::container::pmr::memory_resource* pmr, const size_t size, Alignment alignment);
     Message(boost::container::pmr::memory_resource* pmr,
             void* data,
             const size_t size,

fairmq/ofi/TransportFactory.cxx

@@ -41,11 +41,23 @@ auto TransportFactory::CreateMessage() -> MessagePtr
     return MessagePtr{new Message(&fMemoryResource)};
 }
 
+auto TransportFactory::CreateMessage(Alignment /* alignment */) -> MessagePtr
+{
+    // TODO Do not ignore alignment
+    return MessagePtr{new Message(&fMemoryResource)};
+}
+
 auto TransportFactory::CreateMessage(const size_t size) -> MessagePtr
 {
     return MessagePtr{new Message(&fMemoryResource, size)};
 }
 
+auto TransportFactory::CreateMessage(const size_t size, Alignment /* alignment */) -> MessagePtr
+{
+    // TODO Do not ignore alignment
+    return MessagePtr{new Message(&fMemoryResource, size)};
+}
+
 auto TransportFactory::CreateMessage(void* data,
                                      const size_t size,
                                      fairmq_free_fn* ffn,
@@ -85,12 +97,22 @@ auto TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQCha
     // return PollerPtr{new Poller(channelsMap, channelList)};
 }
 
-auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) const -> UnmanagedRegionPtr
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) -> UnmanagedRegionPtr
 {
     throw runtime_error{"Not yet implemented UMR."};
 }
 
-auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, const int64_t /*userFlags*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) const -> UnmanagedRegionPtr
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionBulkCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) -> UnmanagedRegionPtr
+{
+    throw runtime_error{"Not yet implemented UMR."};
+}
+
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, const int64_t /*userFlags*/, FairMQRegionCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) -> UnmanagedRegionPtr
+{
+    throw runtime_error{"Not yet implemented UMR."};
+}
+
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, const int64_t /*userFlags*/, FairMQRegionBulkCallback /*callback*/, const std::string& /* path = "" */, int /* flags = 0 */) -> UnmanagedRegionPtr
 {
     throw runtime_error{"Not yet implemented UMR."};
 }

fairmq/ofi/TransportFactory.h

@@ -36,7 +36,9 @@ class TransportFactory final : public FairMQTransportFactory
     TransportFactory operator=(const TransportFactory&) = delete;
 
     auto CreateMessage() -> MessagePtr override;
+    auto CreateMessage(Alignment alignment) -> MessagePtr override;
     auto CreateMessage(const std::size_t size) -> MessagePtr override;
+    auto CreateMessage(const std::size_t size, Alignment alignment) -> MessagePtr override;
     auto CreateMessage(void* data, const std::size_t size, fairmq_free_fn* ffn, void* hint = nullptr) -> MessagePtr override;
     auto CreateMessage(UnmanagedRegionPtr& region, void* data, const std::size_t size, void* hint = nullptr) -> MessagePtr override;
 
@@ -46,10 +48,13 @@ class TransportFactory final : public FairMQTransportFactory
     auto CreatePoller(const std::vector<FairMQChannel*>& channels) const -> PollerPtr override;
     auto CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const -> PollerPtr override;
 
-    auto CreateUnmanagedRegion(const size_t size, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
-    auto CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, RegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) -> UnmanagedRegionPtr override;
+    auto CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) -> UnmanagedRegionPtr override;
 
     void SubscribeToRegionEvents(RegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for OFI"; }
+    bool SubscribedToRegionEvents() override { LOG(error) << "Region event subscriptions not yet implemented for OFI"; return false; }
     void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for OFI"; }
     std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for OFI, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
 

fairmq/plugins/Control.cxx

@@ -24,7 +24,7 @@ namespace
     std::atomic<sig_atomic_t> gLastSignal(0);
     std::atomic<int> gSignalCount(0);
 
-    extern "C" auto signal_handler(int signal) -> void
+    extern "C" auto sigint_handler(int signal) -> void
     {
         ++gSignalCount;
         gLastSignal = signal;
@@ -33,6 +33,12 @@ namespace
             std::abort();
         }
     }
+
+    extern "C" auto sigterm_handler(int signal) -> void
+    {
+        ++gSignalCount;
+        gLastSignal = signal;
+    }
 }
 
 namespace fair
@@ -85,8 +91,8 @@ Control::Control(const string& name, const Plugin::Version version, const string
     if (GetProperty<int>("catch-signals") > 0) {
         LOG(debug) << "Plugin '" << name << "' is setting up signal handling for SIGINT and SIGTERM";
         fSignalHandlerThread = thread(&Control::SignalHandler, this);
-        signal(SIGINT, signal_handler);
-        signal(SIGTERM, signal_handler);
+        signal(SIGINT, sigint_handler);
+        signal(SIGTERM, sigterm_handler);
     } else {
         LOG(warn) << "Signal handling (e.g. Ctrl-C) has been deactivated.";
     }

fairmq/plugins/config/Config.cxx

@@ -63,12 +63,13 @@ Plugin::ProgOptions ConfigPluginProgramOptions()
     pluginOptions.add_options()
         ("id",                  po::value<string        >()->default_value(""),         "Device ID.")
         ("io-threads",          po::value<int           >()->default_value(1),          "Number of I/O threads.")
-        ("transport",         po::value<string        >()->default_value("zeromq"),   "Transport ('zeromq'/'nanomsg'/'shmem').")
+        ("transport",           po::value<string        >()->default_value("zeromq"),   "Transport ('zeromq'/'shmem').")
         ("network-interface",   po::value<string        >()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
         ("init-timeout",        po::value<int           >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
         ("max-run-time",        po::value<uint64_t      >()->default_value(0),          "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
         ("print-channels",      po::value<bool          >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
         ("shm-segment-size",    po::value<size_t        >()->default_value(2000000000), "Shared memory: size of the shared memory segment (in bytes).")
+        ("shm-throw-bad-alloc", po::value<bool          >()->default_value(true),       "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).")
         ("shm-monitor",         po::value<bool          >()->default_value(true),       "Shared memory: run monitor daemon.")
         ("ofi-size-hint",       po::value<size_t        >()->default_value(0),          "EXPERIMENTAL: OFI size hint for the allocator.")
         ("rate",                po::value<float         >()->default_value(0.),         "Rate for conditional run loop (Hz).")

fairmq/run/startMQBenchmark.sh.in

@@ -59,7 +59,7 @@ else
 fi
 
 echo ""
-echo "Usage: startBenchmark [message size=1000000] [number of iterations=0] [transport=zeromq/nanomsg/shmem] [affinity=false]"
+echo "Usage: startBenchmark [message size=1000000] [number of iterations=0] [transport=zeromq/shmem] [affinity=false]"
 
 SAMPLER="fairmq-bsampler"
 SAMPLER+=" --id bsampler1"
@@ -70,6 +70,7 @@ SAMPLER+=" --severity debug"
 SAMPLER+=" --msg-size $msgSize"
 SAMPLER+=" --multipart $multipart"
 SAMPLER+=" --num-parts $numParts"
+SAMPLER+=" --shm-throw-bad-alloc false"
 # SAMPLER+=" --msg-rate 1000"
 SAMPLER+=" --max-iterations $maxIterations"
 SAMPLER+=" --channel-config name=data,type=pair,method=bind,address=tcp://127.0.0.1:5555"

fairmq/shmem/Manager.cxx

@@ -1,332 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Manager.h"
-
-#include <fairmq/tools/CppSTL.h>
-#include <fairmq/tools/Strings.h>
-
-#include <boost/process.hpp>
-#include <boost/filesystem.hpp>
-
-using namespace std;
-using bie = ::boost::interprocess::interprocess_exception;
-namespace bipc = ::boost::interprocess;
-namespace bfs = ::boost::filesystem;
-namespace bpt = ::boost::posix_time;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-Manager::Manager(const string& id, size_t size)
-    : fShmId(id)
-    , fSegmentName("fmq_" + fShmId + "_main")
-    , fManagementSegmentName("fmq_" + fShmId + "_mng")
-    , fSegment(bipc::open_or_create, fSegmentName.c_str(), size)
-    , fManagementSegment(bipc::open_or_create, fManagementSegmentName.c_str(), 655360)
-    , fShmVoidAlloc(fManagementSegment.get_segment_manager())
-    , fShmMtx(bipc::open_or_create, string("fmq_" + fShmId + "_mtx").c_str())
-    , fRegionEventsCV(bipc::open_or_create, string("fmq_" + fShmId + "_cv").c_str())
-    , fRegionEventsSubscriptionActive(false)
-    , fDeviceCounter(nullptr)
-    , fRegionInfos(nullptr)
-    , fInterrupted(false)
-{
-    LOG(debug) << "created/opened shared memory segment '" << "fmq_" << fShmId << "_main" << "' of " << size << " bytes. Available are " << fSegment.get_free_memory() << " bytes.";
-
-    fRegionInfos = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(bipc::unique_instance)(fShmVoidAlloc);
-    // store info about the managed segment as region with id 0
-    fRegionInfos->emplace(0, RegionInfo("", 0, 0, fShmVoidAlloc));
-
-    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-
-    fDeviceCounter = fManagementSegment.find<DeviceCounter>(bipc::unique_instance).first;
-
-    if (fDeviceCounter) {
-        LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
-        (fDeviceCounter->fCount)++;
-        LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
-    } else {
-        LOG(debug) << "no device counter found, creating one and initializing with 1";
-        fDeviceCounter = fManagementSegment.construct<DeviceCounter>(bipc::unique_instance)(1);
-        LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
-    }
-}
-
-void Manager::StartMonitor(const string& id)
-{
-    try {
-        bipc::named_mutex monitorStatus(bipc::open_only, string("fmq_" + id + "_ms").c_str());
-        LOG(debug) << "Found fairmq-shmmonitor for shared memory id " << id;
-    } catch (bie&) {
-        LOG(debug) << "no fairmq-shmmonitor found for shared memory id " << id << ", starting...";
-        auto env = boost::this_process::environment();
-
-        vector<bfs::path> ownPath = boost::this_process::path();
-
-        if (const char* fmqp = getenv("FAIRMQ_PATH")) {
-            ownPath.insert(ownPath.begin(), bfs::path(fmqp));
-        }
-
-        bfs::path p = boost::process::search_path("fairmq-shmmonitor", ownPath);
-
-        if (!p.empty()) {
-            boost::process::spawn(p, "-x", "--shmid", id, "-d", "-t", "2000", env);
-            int numTries = 0;
-            do {
-                try {
-                    bipc::named_mutex monitorStatus(bipc::open_only, string("fmq_" + id + "_ms").c_str());
-                    LOG(debug) << "Started fairmq-shmmonitor for shared memory id " << id;
-                    break;
-                } catch (bie&) {
-                    this_thread::sleep_for(chrono::milliseconds(10));
-                    if (++numTries > 1000) {
-                        LOG(error) << "Did not get response from fairmq-shmmonitor after " << 10 * 1000 << " milliseconds. Exiting.";
-                        throw runtime_error(tools::ToString("Did not get response from fairmq-shmmonitor after ", 10 * 1000, " milliseconds. Exiting."));
-                    }
-                }
-            } while (true);
-        } else {
-            LOG(warn) << "could not find fairmq-shmmonitor in the path";
-        }
-    }
-}
-
-pair<bipc::mapped_region*, uint64_t> Manager::CreateRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const string& path /* = "" */, int flags /* = 0 */)
-{
-    try {
-
-        pair<bipc::mapped_region*, uint64_t> result;
-
-        {
-            uint64_t id = 0;
-            bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-
-            RegionCounter* rc = fManagementSegment.find<RegionCounter>(bipc::unique_instance).first;
-
-            if (rc) {
-                LOG(debug) << "region counter found, with value of " << rc->fCount << ". incrementing.";
-                (rc->fCount)++;
-                LOG(debug) << "incremented region counter, now: " << rc->fCount;
-            } else {
-                LOG(debug) << "no region counter found, creating one and initializing with 1";
-                rc = fManagementSegment.construct<RegionCounter>(bipc::unique_instance)(1);
-                LOG(debug) << "initialized region counter with: " << rc->fCount;
-            }
-
-            id = rc->fCount;
-
-            auto it = fRegions.find(id);
-            if (it != fRegions.end()) {
-                LOG(error) << "Trying to create a region that already exists";
-                return {nullptr, id};
-            }
-
-            // create region info
-            fRegionInfos->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
-
-            auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, size, false, callback, path, flags));
-            // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
-
-            r.first->second->StartReceivingAcks();
-            result.first = &(r.first->second->fRegion);
-            result.second = id;
-        }
-        fRegionEventsCV.notify_all();
-
-        return result;
-
-    } catch (bipc::interprocess_exception& e) {
-        LOG(error) << "cannot create region. Already created/not cleaned up?";
-        LOG(error) << e.what();
-        throw;
-    }
-}
-
-void Manager::RemoveRegion(const uint64_t id)
-{
-    {
-        bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-        fRegions.erase(id);
-        fRegionInfos->at(id).fDestroyed = true;
-    }
-    fRegionEventsCV.notify_all();
-}
-
-Region* Manager::GetRegion(const uint64_t id)
-{
-    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-    return GetRegionUnsafe(id);
-}
-
-Region* Manager::GetRegionUnsafe(const uint64_t id)
-{
-    // remote region could actually be a local one if a message originates from this device (has been sent out and returned)
-    auto it = fRegions.find(id);
-    if (it != fRegions.end()) {
-        return it->second.get();
-    } else {
-        try {
-            // get region info
-            RegionInfo regionInfo = fRegionInfos->at(id);
-            string path = regionInfo.fPath.c_str();
-            int flags = regionInfo.fFlags;
-            // LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
-
-            auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, 0, true, nullptr, path, flags));
-            return r.first->second.get();
-        } catch (bie& e) {
-            LOG(warn) << "Could not get remote region for id: " << id;
-            return nullptr;
-        }
-    }
-}
-
-vector<fair::mq::RegionInfo> Manager::GetRegionInfo()
-{
-    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-    return GetRegionInfoUnsafe();
-}
-
-vector<fair::mq::RegionInfo> Manager::GetRegionInfoUnsafe()
-{
-    vector<fair::mq::RegionInfo> result;
-
-    for (const auto& e : *fRegionInfos) {
-        fair::mq::RegionInfo info;
-        info.id = e.first;
-        info.flags = e.second.fUserFlags;
-        info.event = e.second.fDestroyed ? RegionEvent::destroyed : RegionEvent::created;
-        if (info.id != 0) {
-            if (!e.second.fDestroyed) {
-                auto region = GetRegionUnsafe(info.id);
-                info.ptr = region->fRegion.get_address();
-                info.size = region->fRegion.get_size();
-            } else {
-                info.ptr = nullptr;
-                info.size = 0;
-            }
-            result.push_back(info);
-        } else {
-            if (!e.second.fDestroyed) {
-                info.ptr = fSegment.get_address();
-                info.size = fSegment.get_size();
-            } else {
-                info.ptr = nullptr;
-                info.size = 0;
-            }
-            result.push_back(info);
-        }
-    }
-
-    return result;
-}
-
-void Manager::SubscribeToRegionEvents(RegionEventCallback callback)
-{
-    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-    if (fRegionEventThread.joinable()) {
-        fRegionEventsSubscriptionActive.store(false);
-        fRegionEventThread.join();
-    }
-    fRegionEventCallback = callback;
-    fRegionEventsSubscriptionActive.store(true);
-    fRegionEventThread = thread(&Manager::RegionEventsSubscription, this);
-}
-
-void Manager::UnsubscribeFromRegionEvents()
-{
-    if (fRegionEventThread.joinable()) {
-        fRegionEventsSubscriptionActive.store(false);
-        fRegionEventsCV.notify_all();
-        fRegionEventThread.join();
-    }
-    bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-    fRegionEventCallback = nullptr;
-}
-
-void Manager::RegionEventsSubscription()
-{
-    while (fRegionEventsSubscriptionActive.load()) {
-        bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-        auto infos = GetRegionInfoUnsafe();
-        for (const auto& i : infos) {
-            auto el = fObservedRegionEvents.find(i.id);
-            if (el == fObservedRegionEvents.end()) {
-                fRegionEventCallback(i);
-                fObservedRegionEvents.emplace(i.id, i.event);
-            } else {
-                if (el->second == RegionEvent::created && i.event == RegionEvent::destroyed) {
-                    fRegionEventCallback(i);
-                    el->second = i.event;
-                } else {
-                    // LOG(debug) << "ignoring event for id" << i.id << ":";
-                    // LOG(debug) << "incoming event: " << i.event;
-                    // LOG(debug) << "stored event: " << el->second;
-                }
-            }
-        }
-        fRegionEventsCV.wait(lock);
-    }
-}
-
-void Manager::RemoveSegments()
-{
-    if (bipc::shared_memory_object::remove(fSegmentName.c_str())) {
-        LOG(debug) << "successfully removed '" << fSegmentName << "' segment after the device has stopped.";
-    } else {
-        LOG(debug) << "did not remove " << fSegmentName << " segment after the device stopped. Already removed?";
-    }
-
-    if (bipc::shared_memory_object::remove(fManagementSegmentName.c_str())) {
-        LOG(debug) << "successfully removed '" << fManagementSegmentName << "' segment after the device has stopped.";
-    } else {
-        LOG(debug) << "did not remove '" << fManagementSegmentName << "' segment after the device stopped. Already removed?";
-    }
-}
-
-Manager::~Manager()
-{
-    bool lastRemoved = false;
-
-    if (fRegionEventThread.joinable()) {
-        fRegionEventsSubscriptionActive.store(false);
-        fRegionEventsCV.notify_all();
-        fRegionEventThread.join();
-    }
-
-    try {
-        bipc::scoped_lock<bipc::named_mutex> lock(fShmMtx);
-
-        (fDeviceCounter->fCount)--;
-
-        if (fDeviceCounter->fCount == 0) {
-            LOG(debug) << "last segment user, removing segment.";
-
-            RemoveSegments();
-            lastRemoved = true;
-        } else {
-            LOG(debug) << "other segment users present (" << fDeviceCounter->fCount << "), not removing it.";
-        }
-    } catch(bie& e) {
-        LOG(error) << "error while acquiring lock in Manager destructor: " << e.what();
-    }
-
-    if (lastRemoved) {
-        bipc::named_mutex::remove(string("fmq_" + fShmId + "_mtx").c_str());
-        bipc::named_condition::remove(string("fmq_" + fShmId + "_cv").c_str());
-    }
-}
-
-} // namespace shmem
-} // namespace mq
-} // namespace fair

fairmq/shmem/Manager.h

@@ -19,19 +19,25 @@
 #include "Region.h"
 
 #include <FairMQLogger.h>
-#include <FairMQUnmanagedRegion.h>
+#include <FairMQMessage.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
 
-#include <boost/interprocess/ipc/message_queue.hpp>
+#include <boost/process.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/sync/named_condition.hpp>
 #include <boost/interprocess/sync/named_mutex.hpp>
+#include <boost/interprocess/ipc/message_queue.hpp>
 
+#include <cstdlib> // getenv
 #include <set>
 #include <stdexcept>
 #include <string>
 #include <thread>
 #include <unordered_map>
-#include <utility>
+#include <utility> // pair
 #include <vector>
 
 namespace fair
@@ -45,46 +51,376 @@ struct SharedMemoryError : std::runtime_error { using std::runtime_error::runtim
 
 class Manager
 {
-    friend struct Region;
-
   public:
-    Manager(const std::string& id, size_t size);
+    Manager(std::string id, std::string deviceId, size_t size, bool throwOnBadAlloc)
+        : fShmId(std::move(id))
+        , fDeviceId(std::move(deviceId))
+        , fSegmentName("fmq_" + fShmId + "_main")
+        , fManagementSegmentName("fmq_" + fShmId + "_mng")
+        , fSegment(boost::interprocess::open_or_create, fSegmentName.c_str(), size)
+        , fManagementSegment(boost::interprocess::open_or_create, fManagementSegmentName.c_str(), 655360)
+        , fShmVoidAlloc(fManagementSegment.get_segment_manager())
+        , fShmMtx(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_mtx").c_str())
+        , fRegionEventsCV(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_cv").c_str())
+        , fRegionEventsSubscriptionActive(false)
+        , fDeviceCounter(nullptr)
+        , fRegionInfos(nullptr)
+        , fInterrupted(false)
+        , fMsgCounter(0)
+        , fHeartbeatThread()
+        , fSendHeartbeats(true)
+        , fThrowOnBadAlloc(throwOnBadAlloc)
+    {
+        using namespace boost::interprocess;
+        LOG(debug) << "created/opened shared memory segment '" << "fmq_" << fShmId << "_main" << "' of " << size << " bytes. Available are " << fSegment.get_free_memory() << " bytes.";
+
+        fRegionInfos = fManagementSegment.find_or_construct<Uint64RegionInfoMap>(unique_instance)(fShmVoidAlloc);
+        // store info about the managed segment as region with id 0
+        fRegionInfos->emplace(0, RegionInfo("", 0, 0, fShmVoidAlloc));
+
+        boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
+
+        fDeviceCounter = fManagementSegment.find<DeviceCounter>(unique_instance).first;
+
+        if (fDeviceCounter) {
+            LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
+            (fDeviceCounter->fCount)++;
+            LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
+        } else {
+            LOG(debug) << "no device counter found, creating one and initializing with 1";
+            fDeviceCounter = fManagementSegment.construct<DeviceCounter>(unique_instance)(1);
+            LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
+        }
+
+        fHeartbeatThread = std::thread(&Manager::SendHeartbeats, this);
+    }
 
     Manager() = delete;
 
     Manager(const Manager&) = delete;
     Manager operator=(const Manager&) = delete;
 
-    ~Manager();
+    ~Manager()
+    {
+        using namespace boost::interprocess;
+        bool lastRemoved = false;
+
+        UnsubscribeFromRegionEvents();
+
+        fSendHeartbeats = false;
+        fHeartbeatThread.join();
+
+        try {
+            boost::interprocess::scoped_lock<named_mutex> lock(fShmMtx);
+
+            (fDeviceCounter->fCount)--;
+
+            if (fDeviceCounter->fCount == 0) {
+                LOG(debug) << "last segment user, removing segment.";
+
+                RemoveSegments();
+                lastRemoved = true;
+            } else {
+                LOG(debug) << "other segment users present (" << fDeviceCounter->fCount << "), not removing it.";
+            }
+        } catch(interprocess_exception& e) {
+            LOG(error) << "error while acquiring lock in Manager destructor: " << e.what();
+        }
+
+        if (lastRemoved) {
+            named_mutex::remove(std::string("fmq_" + fShmId + "_mtx").c_str());
+            named_condition::remove(std::string("fmq_" + fShmId + "_cv").c_str());
+        }
+    }
 
     boost::interprocess::managed_shared_memory& Segment() { return fSegment; }
     boost::interprocess::managed_shared_memory& ManagementSegment() { return fManagementSegment; }
 
-    static void StartMonitor(const std::string&);
+    static void StartMonitor(const std::string& id)
+    {
+        using namespace boost::interprocess;
+        try {
+            named_mutex monitorStatus(open_only, std::string("fmq_" + id + "_ms").c_str());
+            LOG(debug) << "Found fairmq-shmmonitor for shared memory id " << id;
+        } catch (interprocess_exception&) {
+            LOG(debug) << "no fairmq-shmmonitor found for shared memory id " << id << ", starting...";
+            auto env = boost::this_process::environment();
+
+            std::vector<boost::filesystem::path> ownPath = boost::this_process::path();
+
+            if (const char* fmqp = getenv("FAIRMQ_PATH")) {
+                ownPath.insert(ownPath.begin(), boost::filesystem::path(fmqp));
+            }
+
+            boost::filesystem::path p = boost::process::search_path("fairmq-shmmonitor", ownPath);
+
+            if (!p.empty()) {
+                boost::process::spawn(p, "-x", "--shmid", id, "-d", "-t", "2000", env);
+                int numTries = 0;
+                do {
+                    try {
+                        named_mutex monitorStatus(open_only, std::string("fmq_" + id + "_ms").c_str());
+                        LOG(debug) << "Started fairmq-shmmonitor for shared memory id " << id;
+                        break;
+                    } catch (interprocess_exception&) {
+                        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+                        if (++numTries > 1000) {
+                            LOG(error) << "Did not get response from fairmq-shmmonitor after " << 10 * 1000 << " milliseconds. Exiting.";
+                            throw std::runtime_error(tools::ToString("Did not get response from fairmq-shmmonitor after ", 10 * 1000, " milliseconds. Exiting."));
+                        }
+                    }
+                } while (true);
+            } else {
+                LOG(warn) << "could not find fairmq-shmmonitor in the path";
+            }
+        }
+    }
 
     void Interrupt() { fInterrupted.store(true); }
     void Resume() { fInterrupted.store(false); }
+    void Reset()
+    {
+        if (fMsgCounter.load() != 0) {
+            LOG(error) << "Message counter during Reset expected to be 0, found: " << fMsgCounter.load();
+            throw MessageError(tools::ToString("Message counter during Reset expected to be 0, found: ", fMsgCounter.load()));
+        }
+    }
     bool Interrupted() { return fInterrupted.load(); }
 
-    int GetDeviceCounter();
-    int IncrementDeviceCounter();
-    int DecrementDeviceCounter();
+    std::pair<boost::interprocess::mapped_region*, uint64_t> CreateRegion(const size_t size,
+                                                                          const int64_t userFlags,
+                                                                          RegionCallback callback,
+                                                                          RegionBulkCallback bulkCallback,
+                                                                          const std::string& path = "",
+                                                                          int flags = 0)
+    {
+        using namespace boost::interprocess;
+        try {
+            std::pair<mapped_region*, uint64_t> result;
 
-    std::pair<boost::interprocess::mapped_region*, uint64_t> CreateRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const std::string& path = "", int flags = 0);
-    Region* GetRegion(const uint64_t id);
-    Region* GetRegionUnsafe(const uint64_t id);
-    void RemoveRegion(const uint64_t id);
+            {
+                uint64_t id = 0;
+                boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
 
-    std::vector<fair::mq::RegionInfo> GetRegionInfo();
-    std::vector<fair::mq::RegionInfo> GetRegionInfoUnsafe();
-    void SubscribeToRegionEvents(RegionEventCallback callback);
-    void UnsubscribeFromRegionEvents();
-    void RegionEventsSubscription();
+                RegionCounter* rc = fManagementSegment.find<RegionCounter>(unique_instance).first;
 
-    void RemoveSegments();
+                if (rc) {
+                    LOG(debug) << "region counter found, with value of " << rc->fCount << ". incrementing.";
+                    (rc->fCount)++;
+                    LOG(debug) << "incremented region counter, now: " << rc->fCount;
+                } else {
+                    LOG(debug) << "no region counter found, creating one and initializing with 1";
+                    rc = fManagementSegment.construct<RegionCounter>(unique_instance)(1);
+                    LOG(debug) << "initialized region counter with: " << rc->fCount;
+                }
+
+                id = rc->fCount;
+
+                auto it = fRegions.find(id);
+                if (it != fRegions.end()) {
+                    LOG(error) << "Trying to create a region that already exists";
+                    return {nullptr, id};
+                }
+
+                // create region info
+                fRegionInfos->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
+
+                auto r = fRegions.emplace(id, tools::make_unique<Region>(fShmId, id, size, false, callback, bulkCallback, path, flags));
+                // LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
+
+                r.first->second->StartReceivingAcks();
+                result.first = &(r.first->second->fRegion);
+                result.second = id;
+            }
+            fRegionEventsCV.notify_all();
+
+            return result;
+
+        } catch (interprocess_exception& e) {
+            LOG(error) << "cannot create region. Already created/not cleaned up?";
+            LOG(error) << e.what();
+            throw;
+        }
+    }
+
+    Region* GetRegion(const uint64_t id)
+    {
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        return GetRegionUnsafe(id);
+    }
+
+    Region* GetRegionUnsafe(const uint64_t id)
+    {
+        // remote region could actually be a local one if a message originates from this device (has been sent out and returned)
+        auto it = fRegions.find(id);
+        if (it != fRegions.end()) {
+            return it->second.get();
+        } else {
+            try {
+                // get region info
+                RegionInfo regionInfo = fRegionInfos->at(id);
+                std::string path = regionInfo.fPath.c_str();
+                int flags = regionInfo.fFlags;
+                // LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
+
+                auto r = fRegions.emplace(id, tools::make_unique<Region>(fShmId, id, 0, true, nullptr, nullptr, path, flags));
+                return r.first->second.get();
+            } catch (boost::interprocess::interprocess_exception& e) {
+                LOG(warn) << "Could not get remote region for id: " << id;
+                return nullptr;
+            }
+        }
+    }
+
+    void RemoveRegion(const uint64_t id)
+    {
+        {
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+            fRegions.erase(id);
+            fRegionInfos->at(id).fDestroyed = true;
+        }
+        fRegionEventsCV.notify_all();
+    }
+
+    std::vector<fair::mq::RegionInfo> GetRegionInfo()
+    {
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        return GetRegionInfoUnsafe();
+    }
+
+    std::vector<fair::mq::RegionInfo> GetRegionInfoUnsafe()
+    {
+        std::vector<fair::mq::RegionInfo> result;
+
+        for (const auto& e : *fRegionInfos) {
+            fair::mq::RegionInfo info;
+            info.id = e.first;
+            info.flags = e.second.fUserFlags;
+            info.event = e.second.fDestroyed ? RegionEvent::destroyed : RegionEvent::created;
+            if (info.id != 0) {
+                if (!e.second.fDestroyed) {
+                    auto region = GetRegionUnsafe(info.id);
+                    info.ptr = region->fRegion.get_address();
+                    info.size = region->fRegion.get_size();
+                } else {
+                    info.ptr = nullptr;
+                    info.size = 0;
+                }
+                result.push_back(info);
+            } else {
+                if (!e.second.fDestroyed) {
+                    info.ptr = fSegment.get_address();
+                    info.size = fSegment.get_size();
+                } else {
+                    info.ptr = nullptr;
+                    info.size = 0;
+                }
+                result.push_back(info);
+            }
+        }
+
+        return result;
+    }
+
+    void SubscribeToRegionEvents(RegionEventCallback callback)
+    {
+        if (fRegionEventThread.joinable()) {
+            LOG(debug) << "Already subscribed. Overwriting previous subscription.";
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+            fRegionEventsSubscriptionActive = false;
+            lock.unlock();
+            fRegionEventsCV.notify_all();
+            fRegionEventThread.join();
+        }
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        fRegionEventCallback = callback;
+        fRegionEventsSubscriptionActive = true;
+        fRegionEventThread = std::thread(&Manager::RegionEventsSubscription, this);
+    }
+
+    bool SubscribedToRegionEvents() { return fRegionEventThread.joinable(); }
+
+    void UnsubscribeFromRegionEvents()
+    {
+        if (fRegionEventThread.joinable()) {
+            boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+            fRegionEventsSubscriptionActive = false;
+            lock.unlock();
+            fRegionEventsCV.notify_all();
+            fRegionEventThread.join();
+            lock.lock();
+            fRegionEventCallback = nullptr;
+        }
+    }
+
+    void RegionEventsSubscription()
+    {
+        boost::interprocess::scoped_lock<boost::interprocess::named_mutex> lock(fShmMtx);
+        while (fRegionEventsSubscriptionActive) {
+            auto infos = GetRegionInfoUnsafe();
+            for (const auto& i : infos) {
+                auto el = fObservedRegionEvents.find(i.id);
+                if (el == fObservedRegionEvents.end()) {
+                    fRegionEventCallback(i);
+                    fObservedRegionEvents.emplace(i.id, i.event);
+                } else {
+                    if (el->second == RegionEvent::created && i.event == RegionEvent::destroyed) {
+                        fRegionEventCallback(i);
+                        el->second = i.event;
+                    } else {
+                        // LOG(debug) << "ignoring event for id" << i.id << ":";
+                        // LOG(debug) << "incoming event: " << i.event;
+                        // LOG(debug) << "stored event: " << el->second;
+                    }
+                }
+            }
+            fRegionEventsCV.wait(lock);
+        }
+    }
+
+    void IncrementMsgCounter() { ++fMsgCounter; }
+    void DecrementMsgCounter() { --fMsgCounter; }
+
+    void RemoveSegments()
+    {
+        using namespace boost::interprocess;
+        if (shared_memory_object::remove(fSegmentName.c_str())) {
+            LOG(debug) << "successfully removed '" << fSegmentName << "' segment after the device has stopped.";
+        } else {
+            LOG(debug) << "did not remove " << fSegmentName << " segment after the device stopped. Already removed?";
+        }
+
+        if (shared_memory_object::remove(fManagementSegmentName.c_str())) {
+            LOG(debug) << "successfully removed '" << fManagementSegmentName << "' segment after the device has stopped.";
+        } else {
+            LOG(debug) << "did not remove '" << fManagementSegmentName << "' segment after the device stopped. Already removed?";
+        }
+    }
+
+    void SendHeartbeats()
+    {
+        std::string controlQueueName("fmq_" + fShmId + "_cq");
+        while (fSendHeartbeats) {
+            try {
+                boost::interprocess::message_queue mq(boost::interprocess::open_only, controlQueueName.c_str());
+                boost::posix_time::ptime sndTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(100);
+                if (mq.timed_send(fDeviceId.c_str(), fDeviceId.size(), 0, sndTill)) {
+                    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+                } else {
+                    LOG(debug) << "control queue timeout";
+                }
+            } catch (boost::interprocess::interprocess_exception& ie) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(500));
+                // LOG(warn) << "no " << controlQueueName << " found";
+            }
+        }
+    }
+
+    bool ThrowingOnBadAlloc() const { return fThrowOnBadAlloc; }
 
   private:
     std::string fShmId;
+    std::string fDeviceId;
     std::string fSegmentName;
     std::string fManagementSegmentName;
     boost::interprocess::managed_shared_memory fSegment;
@@ -94,7 +430,7 @@ class Manager
 
     boost::interprocess::named_condition fRegionEventsCV;
     std::thread fRegionEventThread;
-    std::atomic<bool> fRegionEventsSubscriptionActive;
+    bool fRegionEventsSubscriptionActive;
     std::function<void(fair::mq::RegionInfo)> fRegionEventCallback;
     std::unordered_map<uint64_t, RegionEvent> fObservedRegionEvents;
 
@@ -103,6 +439,11 @@ class Manager
     std::unordered_map<uint64_t, std::unique_ptr<Region>> fRegions;
 
     std::atomic<bool> fInterrupted;
+    std::atomic<int32_t> fMsgCounter; // TODO: find a better lifetime solution instead of the counter
+
+    std::thread fHeartbeatThread;
+    std::atomic<bool> fSendHeartbeats;
+    bool fThrowOnBadAlloc;
 };
 
 } // namespace shmem

fairmq/shmem/Message.cxx

@@ -1,250 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Region.h"
-#include "Message.h"
-#include "UnmanagedRegion.h"
-#include "TransportFactory.h"
-
-#include <FairMQLogger.h>
-
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-#include <cstdlib>
-
-using namespace std;
-
-namespace bipc = ::boost::interprocess;
-namespace bpt = ::boost::posix_time;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-Transport Message::fTransportType = Transport::SHM;
-
-Message::Message(Manager& manager, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{0, 0, 0, -1}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(nullptr)
-{
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-Message::Message(Manager& manager, const size_t size, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{0, 0, 0, -1}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(nullptr)
-{
-    InitializeChunk(size);
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-Message::Message(Manager& manager, MetaHeader& hdr, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{hdr}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(nullptr)
-{
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-Message::Message(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{0, 0, 0, -1}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(nullptr)
-{
-    if (InitializeChunk(size)) {
-        std::memcpy(fLocalPtr, data, size);
-        if (ffn) {
-            ffn(data, hint);
-        } else {
-            free(data);
-        }
-    }
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-Message::Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint, FairMQTransportFactory* factory)
-    : fair::mq::Message{factory}
-    , fManager(manager)
-    , fQueued(false)
-    , fMeta{size, static_cast<UnmanagedRegion*>(region.get())->fRegionId, reinterpret_cast<size_t>(hint), -1}
-    , fRegionPtr(nullptr)
-    , fLocalPtr(static_cast<char*>(data))
-{
-    if (reinterpret_cast<const char*>(data) >= reinterpret_cast<const char*>(region->GetData()) ||
-        reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize()) {
-        fMeta.fHandle = (bipc::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
-    } else {
-        LOG(error) << "trying to create region message with data from outside the region";
-        throw runtime_error("trying to create region message with data from outside the region");
-    }
-    static_cast<TransportFactory*>(GetTransport())->IncrementMsgCounter();
-}
-
-bool Message::InitializeChunk(const size_t size)
-{
-    while (fMeta.fHandle < 0) {
-        try {
-            bipc::managed_shared_memory::size_type actualSize = size;
-            char* hint = 0; // unused for bipc::allocate_new
-            fLocalPtr = fManager.Segment().allocation_command<char>(bipc::allocate_new, size, actualSize, hint);
-        } catch (bipc::bad_alloc& ba) {
-            // LOG(warn) << "Shared memory full...";
-            this_thread::sleep_for(chrono::milliseconds(50));
-            if (fManager.Interrupted()) {
-                return false;
-            } else {
-                continue;
-            }
-        }
-        fMeta.fHandle = fManager.Segment().get_handle_from_address(fLocalPtr);
-    }
-
-    fMeta.fSize = size;
-    return true;
-}
-
-void Message::Rebuild()
-{
-    CloseMessage();
-    fQueued = false;
-}
-
-void Message::Rebuild(const size_t size)
-{
-    CloseMessage();
-    fQueued = false;
-    InitializeChunk(size);
-}
-
-void Message::Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    CloseMessage();
-    fQueued = false;
-
-    if (InitializeChunk(size)) {
-        std::memcpy(fLocalPtr, data, size);
-        if (ffn) {
-            ffn(data, hint);
-        } else {
-            free(data);
-        }
-    }
-}
-
-void* Message::GetData() const
-{
-    if (!fLocalPtr) {
-        if (fMeta.fRegionId == 0) {
-            if (fMeta.fSize > 0) {
-                fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fMeta.fHandle));
-            } else {
-                fLocalPtr = nullptr;
-            }
-        } else {
-            fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
-            if (fRegionPtr) {
-                fLocalPtr = reinterpret_cast<char*>(fRegionPtr->fRegion.get_address()) + fMeta.fHandle;
-            } else {
-                // LOG(warn) << "could not get pointer from a region message";
-                fLocalPtr = nullptr;
-            }
-        }
-    }
-
-    return fLocalPtr;
-}
-
-bool Message::SetUsedSize(const size_t size)
-{
-    if (size == fMeta.fSize) {
-        return true;
-    } else if (size <= fMeta.fSize) {
-        try {
-            bipc::managed_shared_memory::size_type shrunkSize = size;
-            fLocalPtr = fManager.Segment().allocation_command<char>(bipc::shrink_in_place, fMeta.fSize + 128, shrunkSize, fLocalPtr);
-            fMeta.fSize = size;
-            return true;
-        } catch (bipc::interprocess_exception& e) {
-            LOG(info) << "could not set used size: " << e.what();
-            return false;
-        }
-    } else {
-        LOG(error) << "cannot set used size higher than original.";
-        return false;
-    }
-}
-
-void Message::Copy(const fair::mq::Message& msg)
-{
-    if (fMeta.fHandle < 0) {
-        bipc::managed_shared_memory::handle_t otherHandle = static_cast<const Message&>(msg).fMeta.fHandle;
-        if (otherHandle) {
-            if (InitializeChunk(msg.GetSize())) {
-                std::memcpy(GetData(), msg.GetData(), msg.GetSize());
-            }
-        } else {
-            LOG(error) << "copy fail: source message not initialized!";
-        }
-    } else {
-        LOG(error) << "copy fail: target message already initialized!";
-    }
-}
-
-void Message::CloseMessage()
-{
-    if (fMeta.fHandle >= 0 && !fQueued) {
-        if (fMeta.fRegionId == 0) {
-            fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fMeta.fHandle));
-            fMeta.fHandle = -1;
-        } else {
-            if (!fRegionPtr) {
-                fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
-            }
-
-            if (fRegionPtr) {
-                fRegionPtr->ReleaseBlock({fMeta.fHandle, fMeta.fSize, fMeta.fHint});
-            } else {
-                LOG(warn) << "region ack queue for id " << fMeta.fRegionId << " no longer exist. Not sending ack";
-            }
-        }
-    }
-
-    static_cast<TransportFactory*>(GetTransport())->DecrementMsgCounter();
-}
-
-Message::~Message()
-{
-    try {
-        CloseMessage();
-    } catch(SharedMemoryError& sme) {
-        LOG(error) << "error closing message: " << sme.what();
-    } catch(bipc::lock_exception& le) {
-        LOG(error) << "error closing message: " << le.what();
-    }
-}
-
-}
-}
-}

fairmq/shmem/Message.h

@@ -10,7 +10,10 @@
 
 #include "Common.h"
 #include "Manager.h"
-
+#include "Region.h"
+#include "UnmanagedRegion.h"
+#include <fairmq/Tools.h>
+#include <FairMQLogger.h>
 #include <FairMQMessage.h>
 #include <FairMQUnmanagedRegion.h>
 
@@ -33,30 +36,204 @@ class Message final : public fair::mq::Message
     friend class Socket;
 
   public:
-    Message(Manager& manager, FairMQTransportFactory* factory = nullptr);
-    Message(Manager& manager, const size_t size, FairMQTransportFactory* factory = nullptr);
-    Message(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr);
-    Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr);
+    Message(Manager& manager, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        fManager.IncrementMsgCounter();
+    }
 
-    Message(Manager& manager, MetaHeader& hdr, FairMQTransportFactory* factory = nullptr);
+    Message(Manager& manager, Alignment /* alignment */, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, const size_t size, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        InitializeChunk(size);
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, const size_t size, Alignment alignment, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        InitializeChunk(size, static_cast<size_t>(alignment));
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{0, 0, 0, -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        if (InitializeChunk(size)) {
+            std::memcpy(fLocalPtr, data, size);
+            if (ffn) {
+                ffn(data, hint);
+            } else {
+                free(data);
+            }
+        }
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{size, static_cast<UnmanagedRegion*>(region.get())->fRegionId, reinterpret_cast<size_t>(hint), -1}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(static_cast<char*>(data))
+    {
+        if (reinterpret_cast<const char*>(data) >= reinterpret_cast<const char*>(region->GetData()) ||
+            reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize()) {
+            fMeta.fHandle = (boost::interprocess::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
+        } else {
+            LOG(error) << "trying to create region message with data from outside the region";
+            throw std::runtime_error("trying to create region message with data from outside the region");
+        }
+        fManager.IncrementMsgCounter();
+    }
+
+    Message(Manager& manager, MetaHeader& hdr, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fManager(manager)
+        , fQueued(false)
+        , fMeta{hdr}
+        , fRegionPtr(nullptr)
+        , fLocalPtr(nullptr)
+    {
+        fManager.IncrementMsgCounter();
+    }
 
     Message(const Message&) = delete;
     Message operator=(const Message&) = delete;
 
-    void Rebuild() override;
-    void Rebuild(const size_t size) override;
-    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
+    void Rebuild() override
+    {
+        CloseMessage();
+        fQueued = false;
+    }
+
+    void Rebuild(const size_t size) override
+    {
+        CloseMessage();
+        fQueued = false;
+        InitializeChunk(size);
+    }
+
+    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override
+    {
+        CloseMessage();
+        fQueued = false;
+
+        if (InitializeChunk(size)) {
+            std::memcpy(fLocalPtr, data, size);
+            if (ffn) {
+                ffn(data, hint);
+            } else {
+                free(data);
+            }
+        }
+    }
+
+    void* GetData() const override
+    {
+        if (!fLocalPtr) {
+            if (fMeta.fRegionId == 0) {
+                if (fMeta.fSize > 0) {
+                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+                } else {
+                    fLocalPtr = nullptr;
+                }
+            } else {
+                fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
+                if (fRegionPtr) {
+                    fLocalPtr = reinterpret_cast<char*>(fRegionPtr->fRegion.get_address()) + fMeta.fHandle;
+                } else {
+                    // LOG(warn) << "could not get pointer from a region message";
+                    fLocalPtr = nullptr;
+                }
+            }
+        }
+
+        return fLocalPtr;
+    }
 
-    void* GetData() const override;
     size_t GetSize() const override { return fMeta.fSize; }
 
-    bool SetUsedSize(const size_t size) override;
+    bool SetUsedSize(const size_t size) override
+    {
+        if (size == fMeta.fSize) {
+            return true;
+        } else if (size <= fMeta.fSize) {
+            try {
+                boost::interprocess::managed_shared_memory::size_type shrunkSize = size;
+                fLocalPtr = fManager.Segment().allocation_command<char>(boost::interprocess::shrink_in_place, fMeta.fSize + 128, shrunkSize, fLocalPtr);
+                fMeta.fSize = size;
+                return true;
+            } catch (boost::interprocess::interprocess_exception& e) {
+                LOG(info) << "could not set used size: " << e.what();
+                return false;
+            }
+        } else {
+            LOG(error) << "cannot set used size higher than original.";
+            return false;
+        }
+    }
 
-    Transport GetType() const override { return fTransportType; }
+    Transport GetType() const override { return fair::mq::Transport::SHM; }
 
-    void Copy(const fair::mq::Message& msg) override;
+    void Copy(const fair::mq::Message& msg) override
+    {
+        if (fMeta.fHandle < 0) {
+            boost::interprocess::managed_shared_memory::handle_t otherHandle = static_cast<const Message&>(msg).fMeta.fHandle;
+            if (otherHandle) {
+                if (InitializeChunk(msg.GetSize())) {
+                    std::memcpy(GetData(), msg.GetData(), msg.GetSize());
+                }
+            } else {
+                LOG(error) << "copy fail: source message not initialized!";
+            }
+        } else {
+            LOG(error) << "copy fail: target message already initialized!";
+        }
+    }
 
-    ~Message() override;
+    ~Message() override
+    {
+        try {
+            CloseMessage();
+        } catch(SharedMemoryError& sme) {
+            LOG(error) << "error closing message: " << sme.what();
+        } catch(boost::interprocess::lock_exception& le) {
+            LOG(error) << "error closing message: " << le.what();
+        }
+    }
 
   private:
     Manager& fManager;
@@ -65,10 +242,60 @@ class Message final : public fair::mq::Message
     mutable Region* fRegionPtr;
     mutable char* fLocalPtr;
 
-    static Transport fTransportType;
+    bool InitializeChunk(const size_t size, size_t alignment = 0)
+    {
+        tools::RateLimiter rateLimiter(20);
 
-    bool InitializeChunk(const size_t size);
-    void CloseMessage();
+        while (fMeta.fHandle < 0) {
+            try {
+                // boost::interprocess::managed_shared_memory::size_type actualSize = size;
+                // char* hint = 0; // unused for boost::interprocess::allocate_new
+                // fLocalPtr = fManager.Segment().allocation_command<char>(boost::interprocess::allocate_new, size, actualSize, hint);
+                if (alignment == 0) {
+                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().allocate(size));
+                } else {
+                    fLocalPtr = reinterpret_cast<char*>(fManager.Segment().allocate_aligned(size, alignment));
+                }
+            } catch (boost::interprocess::bad_alloc& ba) {
+                // LOG(warn) << "Shared memory full...";
+                if (fManager.ThrowingOnBadAlloc()) {
+                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size, ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default"));
+                }
+                rateLimiter.maybe_sleep();
+                if (fManager.Interrupted()) {
+                    return false;
+                } else {
+                    continue;
+                }
+            }
+            fMeta.fHandle = fManager.Segment().get_handle_from_address(fLocalPtr);
+        }
+
+        fMeta.fSize = size;
+        return true;
+    }
+
+    void CloseMessage()
+    {
+        if (fMeta.fHandle >= 0 && !fQueued) {
+            if (fMeta.fRegionId == 0) {
+                fManager.Segment().deallocate(fManager.Segment().get_address_from_handle(fMeta.fHandle));
+                fMeta.fHandle = -1;
+            } else {
+                if (!fRegionPtr) {
+                    fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
+                }
+
+                if (fRegionPtr) {
+                    fRegionPtr->ReleaseBlock({fMeta.fHandle, fMeta.fSize, fMeta.fHint});
+                } else {
+                    LOG(warn) << "region ack queue for id " << fMeta.fRegionId << " no longer exist. Not sending ack";
+                }
+            }
+        }
+
+        fManager.DecrementMsgCounter();
+    }
 };
 
 }

fairmq/shmem/Poller.cxx

@@ -1,222 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * Poller.cxx
- *
- * @since 2014-01-23
- * @author A. Rybalchenko
- */
-
-#include "Poller.h"
-#include "Socket.h"
-
-#include <FairMQLogger.h>
-
-#include <zmq.h>
-
-using namespace std;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-Poller::Poller(const vector<FairMQChannel>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new zmq_pollitem_t[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
-        fItems[i].fd = 0;
-        fItems[i].revents = 0;
-
-        int type = 0;
-        size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-Poller::Poller(const vector<FairMQChannel*>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new zmq_pollitem_t[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
-        fItems[i].fd = 0;
-        fItems[i].revents = 0;
-
-        int type = 0;
-        size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-Poller::Poller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    try
-    {
-        int offset = 0;
-        // calculate offsets and the total size of the poll item set
-        for (string channel : channelList)
-        {
-            fOffsetMap[channel] = offset;
-            offset += channelsMap.at(channel).size();
-            fNumItems += channelsMap.at(channel).size();
-        }
-
-        fItems = new zmq_pollitem_t[fNumItems];
-
-        int index = 0;
-        for (string channel : channelList)
-        {
-            for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i)
-            {
-                index = fOffsetMap[channel] + i;
-
-                fItems[index].socket = static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
-                fItems[index].fd = 0;
-                fItems[index].revents = 0;
-
-                int type = 0;
-                size_t size = sizeof(type);
-                zmq_getsockopt(static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-                SetItemEvents(fItems[index], type);
-            }
-        }
-    }
-    catch (const out_of_range& oor)
-    {
-        LOG(error) << "at least one of the provided channel keys for poller initialization is invalid";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-void Poller::SetItemEvents(zmq_pollitem_t& item, const int type)
-{
-    if (type == ZMQ_REQ || type == ZMQ_REP || type == ZMQ_PAIR || type == ZMQ_DEALER || type == ZMQ_ROUTER)
-    {
-        item.events = ZMQ_POLLIN|ZMQ_POLLOUT;
-    }
-    else if (type == ZMQ_PUSH || type == ZMQ_PUB || type == ZMQ_XPUB)
-    {
-        item.events = ZMQ_POLLOUT;
-    }
-    else if (type == ZMQ_PULL || type == ZMQ_SUB || type == ZMQ_XSUB)
-    {
-        item.events = ZMQ_POLLIN;
-    }
-    else
-    {
-        LOG(error) << "invalid poller configuration, exiting.";
-        exit(EXIT_FAILURE);
-    }
-}
-
-void Poller::Poll(const int timeout)
-{
-    if (zmq_poll(fItems, fNumItems, timeout) < 0)
-    {
-        if (errno == ETERM)
-        {
-            LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
-        }
-        else
-        {
-            LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
-            throw runtime_error("polling failed");
-        }
-    }
-}
-
-bool Poller::CheckInput(const int index)
-{
-    if (fItems[index].revents & ZMQ_POLLIN)
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool Poller::CheckOutput(const int index)
-{
-    if (fItems[index].revents & ZMQ_POLLOUT)
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool Poller::CheckInput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN)
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const out_of_range& oor)
-    {
-        LOG(error) << "invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-bool Poller::CheckOutput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT)
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const out_of_range& oor)
-    {
-        LOG(error) << "Invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-Poller::~Poller()
-{
-    delete[] fItems;
-}
-
-}
-}
-}

fairmq/shmem/Poller.h

@@ -8,42 +8,184 @@
 #ifndef FAIR_MQ_SHMEM_POLLER_H_
 #define FAIR_MQ_SHMEM_POLLER_H_
 
+#include "Socket.h"
+#include <fairmq/Tools.h>
+#include <FairMQChannel.h>
+#include <FairMQLogger.h>
+#include <FairMQPoller.h>
+
 #include <zmq.h>
 
-#include <FairMQPoller.h>
-#include <FairMQChannel.h>
-
-#include <vector>
 #include <unordered_map>
+#include <vector>
 
 class FairMQChannel;
 
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
+namespace fair {
+namespace mq {
+namespace shmem {
 
 class Poller final : public fair::mq::Poller
 {
   public:
-    Poller(const std::vector<FairMQChannel>& channels);
-    Poller(const std::vector<FairMQChannel*>& channels);
-    Poller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList);
+    Poller(const std::vector<FairMQChannel>& channels)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        fNumItems = channels.size();
+        fItems = new zmq_pollitem_t[fNumItems];
+
+        for (int i = 0; i < fNumItems; ++i) {
+            fItems[i].socket = static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
+            fItems[i].fd = 0;
+            fItems[i].revents = 0;
+
+            int type = 0;
+            size_t size = sizeof(type);
+            zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+            SetItemEvents(fItems[i], type);
+        }
+    }
+
+    Poller(const std::vector<FairMQChannel*>& channels)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        fNumItems = channels.size();
+        fItems = new zmq_pollitem_t[fNumItems];
+
+        for (int i = 0; i < fNumItems; ++i) {
+            fItems[i].socket = static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
+            fItems[i].fd = 0;
+            fItems[i].revents = 0;
+
+            int type = 0;
+            size_t size = sizeof(type);
+            zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+            SetItemEvents(fItems[i], type);
+        }
+    }
+
+    Poller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        try {
+            int offset = 0;
+            // calculate offsets and the total size of the poll item set
+            for (std::string channel : channelList) {
+                fOffsetMap[channel] = offset;
+                offset += channelsMap.at(channel).size();
+                fNumItems += channelsMap.at(channel).size();
+            }
+
+            fItems = new zmq_pollitem_t[fNumItems];
+
+            int index = 0;
+            for (std::string channel : channelList) {
+                for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i) {
+                    index = fOffsetMap[channel] + i;
+
+                    fItems[index].socket = static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
+                    fItems[index].fd = 0;
+                    fItems[index].revents = 0;
+
+                    int type = 0;
+                    size_t size = sizeof(type);
+                    zmq_getsockopt(static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+                    SetItemEvents(fItems[index], type);
+                }
+            }
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "At least one of the provided channel keys for poller initialization is invalid." << " Out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("At least one of the provided channel keys for poller initialization is invalid. ", "Out of range error: ", oor.what()));
+        }
+    }
 
     Poller(const Poller&) = delete;
     Poller operator=(const Poller&) = delete;
 
-    void SetItemEvents(zmq_pollitem_t& item, const int type);
+    void SetItemEvents(zmq_pollitem_t& item, const int type)
+    {
+        if (type == ZMQ_REQ || type == ZMQ_REP || type == ZMQ_PAIR || type == ZMQ_DEALER || type == ZMQ_ROUTER) {
+            item.events = ZMQ_POLLIN | ZMQ_POLLOUT;
+        } else if (type == ZMQ_PUSH || type == ZMQ_PUB || type == ZMQ_XPUB) {
+            item.events = ZMQ_POLLOUT;
+        } else if (type == ZMQ_PULL || type == ZMQ_SUB || type == ZMQ_XSUB) {
+            item.events = ZMQ_POLLIN;
+        } else {
+            LOG(error) << "invalid poller configuration, exiting.";
+            throw fair::mq::PollerError("Invalid poller configuration, exiting.");
+        }
+    }
 
-    void Poll(const int timeout) override;
-    bool CheckInput(const int index) override;
-    bool CheckOutput(const int index) override;
-    bool CheckInput(const std::string& channelKey, const int index) override;
-    bool CheckOutput(const std::string& channelKey, const int index) override;
+    void Poll(const int timeout) override
+    {
+        if (zmq_poll(fItems, fNumItems, timeout) < 0) {
+            if (errno == ETERM) {
+                LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
+            } else {
+                LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
+                throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+            }
+        }
+    }
 
-    ~Poller() override;
+    bool CheckInput(const int index) override
+    {
+        if (fItems[index].revents & ZMQ_POLLIN) {
+            return true;
+        }
+
+        return false;
+    }
+
+    bool CheckOutput(const int index) override
+    {
+        if (fItems[index].revents & ZMQ_POLLOUT) {
+            return true;
+        }
+
+        return false;
+    }
+
+    bool CheckInput(const std::string& channelKey, const int index) override
+    {
+        try {
+            if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN) {
+                return true;
+            }
+
+            return false;
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "invalid channel key: '" << channelKey << "'";
+            LOG(error) << "out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("Invalid channel key '", channelKey, "'. Out of range error: ", oor.what()));
+        }
+    }
+
+    bool CheckOutput(const std::string& channelKey, const int index) override
+    {
+        try {
+            if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT) {
+                return true;
+            }
+
+            return false;
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "invalid channel key: '" << channelKey << "'";
+            LOG(error) << "out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("Invalid channel key '", channelKey, "'. Out of range error: ", oor.what()));
+        }
+    }
+
+    ~Poller() override { delete[] fItems; }
 
   private:
     zmq_pollitem_t* fItems;
@@ -52,8 +194,8 @@ class Poller final : public fair::mq::Poller
     std::unordered_map<std::string, int> fOffsetMap;
 };
 
-}
-}
-}
+}   // namespace shmem
+}   // namespace mq
+}   // namespace fair
 
 #endif /* FAIR_MQ_SHMEM_POLLER_H_ */

fairmq/shmem/Region.cxx

@@ -1,217 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Region.h"
-#include "Common.h"
-#include "Manager.h"
-
-#include <fairmq/tools/CppSTL.h>
-#include <fairmq/tools/Strings.h>
-
-#include <boost/filesystem.hpp>
-#include <boost/process.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-#include <cerrno>
-#include <chrono>
-#include <ios>
-
-using namespace std;
-
-namespace bipc = ::boost::interprocess;
-namespace bpt = ::boost::posix_time;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-Region::Region(Manager& manager, uint64_t id, uint64_t size, bool remote, RegionCallback callback, const string& path /* = "" */, int flags /* = 0 */)
-    : fManager(manager)
-    , fRemote(remote)
-    , fStop(false)
-    , fName("fmq_" + fManager.fShmId + "_rg_" + to_string(id))
-    , fQueueName("fmq_" + fManager.fShmId + "_rgq_" + to_string(id))
-    , fShmemObject()
-    , fFile(nullptr)
-    , fFileMapping()
-    , fQueue(nullptr)
-    , fReceiveAcksWorker()
-    , fSendAcksWorker()
-    , fCallback(callback)
-{
-    if (path != "") {
-        fName = string(path + fName);
-
-        if (!fRemote) {
-            // create a file
-            filebuf fbuf;
-            if (fbuf.open(fName, ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary)) {
-                // set the size
-                fbuf.pubseekoff(size - 1, ios_base::beg);
-                fbuf.sputc(0);
-            }
-        }
-
-        fFile = fopen(fName.c_str(), "r+");
-
-        if (!fFile) {
-            LOG(error) << "Failed to initialize file: " << fName;
-            LOG(error) << "errno: " << errno << ": " << strerror(errno);
-            throw runtime_error(tools::ToString("Failed to initialize file for shared memory region: ", strerror(errno)));
-        }
-        fFileMapping = bipc::file_mapping(fName.c_str(), bipc::read_write);
-        LOG(debug) << "shmem: initialized file: " << fName;
-        fRegion = bipc::mapped_region(fFileMapping, bipc::read_write, 0, size, 0, flags);
-    } else {
-        if (fRemote) {
-            fShmemObject = bipc::shared_memory_object(bipc::open_only, fName.c_str(), bipc::read_write);
-        } else {
-            fShmemObject = bipc::shared_memory_object(bipc::create_only, fName.c_str(), bipc::read_write);
-            fShmemObject.truncate(size);
-        }
-        fRegion = bipc::mapped_region(fShmemObject, bipc::read_write, 0, 0, 0, flags);
-    }
-
-    InitializeQueues();
-    StartSendingAcks();
-    LOG(debug) << "shmem: initialized region: " << fName;
-}
-
-void Region::InitializeQueues()
-{
-    if (fRemote) {
-        fQueue = tools::make_unique<bipc::message_queue>(bipc::open_only, fQueueName.c_str());
-    } else {
-        fQueue = tools::make_unique<bipc::message_queue>(bipc::create_only, fQueueName.c_str(), 1024, fAckBunchSize * sizeof(RegionBlock));
-    }
-    LOG(debug) << "shmem: initialized region queue: " << fQueueName;
-}
-
-void Region::StartSendingAcks()
-{
-    fSendAcksWorker = thread(&Region::SendAcks, this);
-}
-
-void Region::StartReceivingAcks()
-{
-    fReceiveAcksWorker = thread(&Region::ReceiveAcks, this);
-}
-
-void Region::ReceiveAcks()
-{
-    unsigned int priority;
-    bipc::message_queue::size_type recvdSize;
-    unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
-
-    while (!fStop) { // end thread condition (should exist until region is destroyed)
-        auto rcvTill = bpt::microsec_clock::universal_time() + bpt::milliseconds(500);
-
-        while (fQueue->timed_receive(blocks.get(), fAckBunchSize * sizeof(RegionBlock), recvdSize, priority, rcvTill)) {
-            // LOG(debug) << "received: " << block.fHandle << " " << block.fSize << " " << block.fMessageId;
-            if (fCallback) {
-                const auto numBlocks = recvdSize / sizeof(RegionBlock);
-                for (size_t i = 0; i < numBlocks; i++) {
-                    fCallback(reinterpret_cast<char*>(fRegion.get_address()) + blocks[i].fHandle, blocks[i].fSize, reinterpret_cast<void*>(blocks[i].fHint));
-                }
-            }
-        }
-    } // while !fStop
-
-    LOG(debug) << "receive ack worker for " << fName << " leaving.";
-}
-
-void Region::ReleaseBlock(const RegionBlock &block)
-{
-    unique_lock<mutex> lock(fBlockMtx);
-
-    fBlocksToFree.emplace_back(block);
-
-    if (fBlocksToFree.size() >= fAckBunchSize) {
-        lock.unlock(); // reduces contention on fBlockMtx
-        fBlockSendCV.notify_one();
-    }
-}
-
-void Region::SendAcks()
-{
-    unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
-
-    while (true) { // we'll try to send all acks before stopping
-        size_t blocksToSend = 0;
-
-        {   // mutex locking block
-            unique_lock<mutex> lock(fBlockMtx);
-
-            // try to get more blocks without waiting (we can miss a notify from CloseMessage())
-            if (!fStop && (fBlocksToFree.size() < fAckBunchSize)) {
-                // cv.wait() timeout: send whatever blocks we have
-                fBlockSendCV.wait_for(lock, chrono::milliseconds(500));
-            }
-
-            blocksToSend = min(fBlocksToFree.size(), fAckBunchSize);
-
-            copy_n(fBlocksToFree.end() - blocksToSend, blocksToSend, blocks.get());
-            fBlocksToFree.resize(fBlocksToFree.size() - blocksToSend);
-        } // unlock the block mutex here while sending over IPC
-
-        if (blocksToSend > 0) {
-            while (!fQueue->try_send(blocks.get(), blocksToSend * sizeof(RegionBlock), 0) && !fStop) {
-                // receiver slow? yield and try again...
-                this_thread::yield();
-            }
-        } else { // blocksToSend == 0
-            if (fStop) {
-                break;
-            }
-        }
-    }
-
-    LOG(debug) << "send ack worker for " << fName << " leaving.";
-}
-
-Region::~Region()
-{
-    fStop = true;
-
-    if (fSendAcksWorker.joinable()) {
-        fBlockSendCV.notify_one();
-        fSendAcksWorker.join();
-    }
-
-    if (!fRemote) {
-        if (fReceiveAcksWorker.joinable()) {
-            fReceiveAcksWorker.join();
-        }
-
-        if (bipc::shared_memory_object::remove(fName.c_str())) {
-            LOG(debug) << "shmem: destroyed region " << fName;
-        }
-
-        if (bipc::file_mapping::remove(fName.c_str())) {
-            LOG(debug) << "shmem: destroyed file mapping " << fName;
-        }
-
-        if (fFile) {
-            fclose(fFile);
-        }
-
-        if (bipc::message_queue::remove(fQueueName.c_str())) {
-            LOG(debug) << "shmem: removed region queue " << fQueueName;
-        }
-    } else {
-        // LOG(debug) << "shmem: region '" << fName << "' is remote, no cleanup necessary.";
-        LOG(debug) << "shmem: region queue '" << fQueueName << "' is remote, no cleanup necessary";
-    }
-}
-
-} // namespace shmem
-} // namespace mq
-} // namespace fair

fairmq/shmem/Region.h

@@ -19,15 +19,24 @@
 
 #include <FairMQLogger.h>
 #include <FairMQUnmanagedRegion.h>
+#include <fairmq/tools/CppSTL.h>
+#include <fairmq/tools/Strings.h>
 
+#include <boost/filesystem.hpp>
+#include <boost/process.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/file_mapping.hpp>
 #include <boost/interprocess/ipc/message_queue.hpp>
 
+#include <algorithm> // min
 #include <thread>
 #include <mutex>
 #include <condition_variable>
 #include <unordered_map>
+#include <cerrno>
+#include <chrono>
+#include <ios>
 
 namespace fair
 {
@@ -36,28 +45,204 @@ namespace mq
 namespace shmem
 {
 
-class Manager;
-
 struct Region
 {
-    Region(Manager& manager, uint64_t id, uint64_t size, bool remote, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0);
+    Region(const std::string& shmId, uint64_t id, uint64_t size, bool remote, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string& path, int flags)
+        : fRemote(remote)
+        , fStop(false)
+        , fName("fmq_" + shmId + "_rg_" + std::to_string(id))
+        , fQueueName("fmq_" + shmId + "_rgq_" + std::to_string(id))
+        , fShmemObject()
+        , fFile(nullptr)
+        , fFileMapping()
+        , fQueue(nullptr)
+        , fReceiveAcksWorker()
+        , fSendAcksWorker()
+        , fCallback(callback)
+        , fBulkCallback(bulkCallback)
+    {
+        using namespace boost::interprocess;
+
+        if (path != "") {
+            fName = std::string(path + fName);
+
+            if (!fRemote) {
+                // create a file
+                std::filebuf fbuf;
+                if (fbuf.open(fName, std::ios_base::in | std::ios_base::out | std::ios_base::trunc | std::ios_base::binary)) {
+                    // set the size
+                    fbuf.pubseekoff(size - 1, std::ios_base::beg);
+                    fbuf.sputc(0);
+                }
+            }
+
+            fFile = fopen(fName.c_str(), "r+");
+
+            if (!fFile) {
+                LOG(error) << "Failed to initialize file: " << fName;
+                LOG(error) << "errno: " << errno << ": " << strerror(errno);
+                throw std::runtime_error(tools::ToString("Failed to initialize file for shared memory region: ", strerror(errno)));
+            }
+            fFileMapping = file_mapping(fName.c_str(), read_write);
+            LOG(debug) << "shmem: initialized file: " << fName;
+            fRegion = mapped_region(fFileMapping, read_write, 0, size, 0, flags);
+        } else {
+            if (fRemote) {
+                fShmemObject = shared_memory_object(open_only, fName.c_str(), read_write);
+            } else {
+                fShmemObject = shared_memory_object(create_only, fName.c_str(), read_write);
+                fShmemObject.truncate(size);
+            }
+            fRegion = mapped_region(fShmemObject, read_write, 0, 0, 0, flags);
+        }
+
+        InitializeQueues();
+        StartSendingAcks();
+        LOG(debug) << "shmem: initialized region: " << fName;
+    }
 
     Region() = delete;
 
     Region(const Region&) = delete;
     Region(Region&&) = delete;
 
-    void InitializeQueues();
+    void InitializeQueues()
+    {
+        using namespace boost::interprocess;
 
-    void StartSendingAcks();
-    void SendAcks();
-    void StartReceivingAcks();
-    void ReceiveAcks();
-    void ReleaseBlock(const RegionBlock &);
+        if (fRemote) {
+            fQueue = tools::make_unique<message_queue>(open_only, fQueueName.c_str());
+        } else {
+            fQueue = tools::make_unique<message_queue>(create_only, fQueueName.c_str(), 1024, fAckBunchSize * sizeof(RegionBlock));
+        }
+        LOG(debug) << "shmem: initialized region queue: " << fQueueName;
+    }
 
-    ~Region();
+    void StartSendingAcks()
+    {
+        fSendAcksWorker = std::thread(&Region::SendAcks, this);
+    }
+
+    void SendAcks()
+    {
+        std::unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
+
+        while (true) { // we'll try to send all acks before stopping
+            size_t blocksToSend = 0;
+
+            {   // mutex locking block
+                std::unique_lock<std::mutex> lock(fBlockMtx);
+
+                // try to get more blocks without waiting (we can miss a notify from CloseMessage())
+                if (!fStop && (fBlocksToFree.size() < fAckBunchSize)) {
+                    // cv.wait() timeout: send whatever blocks we have
+                    fBlockSendCV.wait_for(lock, std::chrono::milliseconds(500));
+                }
+
+                blocksToSend = std::min(fBlocksToFree.size(), fAckBunchSize);
+
+                copy_n(fBlocksToFree.end() - blocksToSend, blocksToSend, blocks.get());
+                fBlocksToFree.resize(fBlocksToFree.size() - blocksToSend);
+            } // unlock the block mutex here while sending over IPC
+
+            if (blocksToSend > 0) {
+                while (!fQueue->try_send(blocks.get(), blocksToSend * sizeof(RegionBlock), 0) && !fStop) {
+                    // receiver slow? yield and try again...
+                    std::this_thread::yield();
+                }
+            } else { // blocksToSend == 0
+                if (fStop) {
+                    break;
+                }
+            }
+        }
+
+        LOG(debug) << "send ack worker for " << fName << " leaving.";
+    }
+
+    void StartReceivingAcks()
+    {
+        fReceiveAcksWorker = std::thread(&Region::ReceiveAcks, this);
+    }
+
+    void ReceiveAcks()
+    {
+        unsigned int priority;
+        boost::interprocess::message_queue::size_type recvdSize;
+        std::unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
+        std::vector<fair::mq::RegionBlock> result;
+        result.reserve(fAckBunchSize);
+
+        while (!fStop) { // end thread condition (should exist until region is destroyed)
+            auto rcvTill = boost::posix_time::microsec_clock::universal_time() + boost::posix_time::milliseconds(500);
+
+            while (fQueue->timed_receive(blocks.get(), fAckBunchSize * sizeof(RegionBlock), recvdSize, priority, rcvTill)) {
+                // LOG(debug) << "received: " << block.fHandle << " " << block.fSize << " " << block.fMessageId;
+                const auto numBlocks = recvdSize / sizeof(RegionBlock);
+                if (fBulkCallback) {
+                    result.clear();
+                    for (size_t i = 0; i < numBlocks; i++) {
+                        result.emplace_back(reinterpret_cast<char*>(fRegion.get_address()) + blocks[i].fHandle, blocks[i].fSize, reinterpret_cast<void*>(blocks[i].fHint));
+                    }
+                    fBulkCallback(result);
+                } else if (fCallback) {
+                    for (size_t i = 0; i < numBlocks; i++) {
+                        fCallback(reinterpret_cast<char*>(fRegion.get_address()) + blocks[i].fHandle, blocks[i].fSize, reinterpret_cast<void*>(blocks[i].fHint));
+                    }
+                }
+            }
+        } // while !fStop
+
+        LOG(debug) << "ReceiveAcks() worker for " << fName << " leaving.";
+    }
+
+    void ReleaseBlock(const RegionBlock& block)
+    {
+        std::unique_lock<std::mutex> lock(fBlockMtx);
+
+        fBlocksToFree.emplace_back(block);
+
+        if (fBlocksToFree.size() >= fAckBunchSize) {
+            lock.unlock(); // reduces contention on fBlockMtx
+            fBlockSendCV.notify_one();
+        }
+    }
+
+    ~Region()
+    {
+        fStop = true;
+
+        if (fSendAcksWorker.joinable()) {
+            fBlockSendCV.notify_one();
+            fSendAcksWorker.join();
+        }
+
+        if (!fRemote) {
+            if (fReceiveAcksWorker.joinable()) {
+                fReceiveAcksWorker.join();
+            }
+
+            if (boost::interprocess::shared_memory_object::remove(fName.c_str())) {
+                LOG(debug) << "shmem: destroyed region " << fName;
+            }
+
+            if (boost::interprocess::file_mapping::remove(fName.c_str())) {
+                LOG(debug) << "shmem: destroyed file mapping " << fName;
+            }
+
+            if (fFile) {
+                fclose(fFile);
+            }
+
+            if (boost::interprocess::message_queue::remove(fQueueName.c_str())) {
+                LOG(debug) << "shmem: removed region queue " << fQueueName;
+            }
+        } else {
+            // LOG(debug) << "shmem: region '" << fName << "' is remote, no cleanup necessary.";
+            LOG(debug) << "shmem: region queue '" << fQueueName << "' is remote, no cleanup necessary";
+        }
+    }
 
-    Manager& fManager;
     bool fRemote;
     bool fStop;
     std::string fName;
@@ -76,6 +261,7 @@ struct Region
     std::thread fReceiveAcksWorker;
     std::thread fSendAcksWorker;
     RegionCallback fCallback;
+    RegionBulkCallback fBulkCallback;
 };
 
 } // namespace shmem

fairmq/shmem/Socket.cxx

@@ -1,496 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Common.h"
-#include "Socket.h"
-#include "Message.h"
-#include "UnmanagedRegion.h"
-#include "TransportFactory.h"
-
-#include <FairMQLogger.h>
-#include <fairmq/Tools.h>
-
-#include <zmq.h>
-
-#include <stdexcept>
-
-using namespace std;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-struct ZMsg
-{
-    ZMsg()            { int rc __attribute__((unused)) = zmq_msg_init(&fMsg);            assert(rc == 0); }
-    explicit ZMsg(size_t size) { int rc __attribute__((unused)) = zmq_msg_init_size(&fMsg, size); assert(rc == 0); }
-    ~ZMsg()           { int rc __attribute__((unused)) = zmq_msg_close(&fMsg);           assert(rc == 0); }
-
-    void* Data()      { return zmq_msg_data(&fMsg); }
-    size_t Size()     { return zmq_msg_size(&fMsg); }
-    zmq_msg_t* Msg()  { return &fMsg; }
-
-    zmq_msg_t fMsg;
-};
-
-Socket::Socket(Manager& manager, const string& type, const string& name, const string& id /*= ""*/, void* context, FairMQTransportFactory* fac /*=nullptr*/)
-    : fair::mq::Socket{fac}
-    , fSocket(nullptr)
-    , fManager(manager)
-    , fId(id + "." + name + "." + type)
-    , fBytesTx(0)
-    , fBytesRx(0)
-    , fMessagesTx(0)
-    , fMessagesRx(0)
-    , fSndTimeout(100)
-    , fRcvTimeout(100)
-{
-    assert(context);
-    fSocket = zmq_socket(context, GetConstant(type));
-
-    if (fSocket == nullptr) {
-        LOG(error) << "Failed creating socket " << fId << ", reason: " << zmq_strerror(errno);
-        throw SocketError(tools::ToString("Failed creating socket ", fId, ", reason: ", zmq_strerror(errno)));
-    }
-
-    if (zmq_setsockopt(fSocket, ZMQ_IDENTITY, fId.c_str(), fId.length()) != 0) {
-        LOG(error) << "Failed setting ZMQ_IDENTITY socket option, reason: " << zmq_strerror(errno);
-    }
-
-    // Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
-    // Default value for ZeroMQ is -1, which is to wait forever.
-    int linger = 1000;
-    if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) {
-        LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
-    }
-
-    if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0) {
-        LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
-    }
-
-    if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0) {
-        LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
-    }
-
-    // if (type == "sub")
-    // {
-    //     if (zmq_setsockopt(fSocket, ZMQ_SUBSCRIBE, nullptr, 0) != 0)
-    //     {
-    //         LOG(error) << "Failed setting ZMQ_SUBSCRIBE socket option, reason: " << zmq_strerror(errno);
-    //     }
-    // }
-
-    if (type == "sub" || type == "pub") {
-        LOG(error) << "PUB/SUB socket type is not supported for shared memory transport";
-        throw SocketError("PUB/SUB socket type is not supported for shared memory transport");
-    }
-    LOG(debug) << "Created socket " << GetId();
-}
-
-bool Socket::Bind(const string& address)
-{
-    // LOG(info) << "binding socket " << fId << " on " << address;
-    if (zmq_bind(fSocket, address.c_str()) != 0) {
-        if (errno == EADDRINUSE) {
-            // do not print error in this case, this is handled by FairMQDevice in case no connection could be established after trying a number of random ports from a range.
-            return false;
-        }
-        LOG(error) << "Failed binding socket " << fId << ", reason: " << zmq_strerror(errno);
-        return false;
-    }
-    return true;
-}
-
-bool Socket::Connect(const string& address)
-{
-    // LOG(info) << "connecting socket " << fId << " on " << address;
-    if (zmq_connect(fSocket, address.c_str()) != 0) {
-        LOG(error) << "Failed connecting socket " << fId << ", reason: " << zmq_strerror(errno);
-        return false;
-    }
-    return true;
-}
-
-int Socket::Send(MessagePtr& msg, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0) {
-        flags = ZMQ_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    Message* shmMsg = static_cast<Message*>(msg.get());
-    ZMsg zmqMsg(sizeof(MetaHeader));
-    std::memcpy(zmqMsg.Data(), &(shmMsg->fMeta), sizeof(MetaHeader));
-
-    while (true && !fManager.Interrupted()) {
-        int nbytes = zmq_msg_send(zmqMsg.Msg(), fSocket, flags);
-        if (nbytes > 0) {
-            shmMsg->fQueued = true;
-            ++fMessagesTx;
-            size_t size = msg->GetSize();
-            fBytesTx += size;
-            return size;
-        } else if (zmq_errno() == EAGAIN) {
-            if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                if (timeout > 0) {
-                    elapsed += fSndTimeout;
-                    if (elapsed >= timeout) {
-                        return -2;
-                    }
-                }
-                continue;
-            } else {
-                return -2;
-            }
-        } else if (zmq_errno() == ETERM) {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        } else if (zmq_errno() == EINTR) {
-            LOG(debug) << "Send interrupted by system call";
-            return nbytes;
-        }else {
-            LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-            return nbytes;
-        }
-    }
-
-    return -1;
-}
-
-int Socket::Receive(MessagePtr& msg, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0) {
-        flags = ZMQ_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    ZMsg zmqMsg;
-
-    while (true) {
-        Message* shmMsg = static_cast<Message*>(msg.get());
-        int nbytes = zmq_msg_recv(zmqMsg.Msg(), fSocket, flags);
-        if (nbytes > 0) {
-            // check for number of received messages. must be 1
-            if (nbytes != sizeof(MetaHeader)) {
-                throw SocketError(
-                    tools::ToString("Received message is not a valid FairMQ shared memory message. ",
-                        "Possibly due to a misconfigured transport on the sender side. ",
-                        "Expected size of ", sizeof(MetaHeader), " bytes, received ", nbytes));
-            }
-
-            MetaHeader* hdr = static_cast<MetaHeader*>(zmqMsg.Data());
-            size_t size = hdr->fSize;
-            shmMsg->fMeta = *hdr;
-
-            fBytesRx += size;
-            ++fMessagesRx;
-            return size;
-        } else if (zmq_errno() == EAGAIN) {
-            if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                if (timeout > 0) {
-                    elapsed += fRcvTimeout;
-                    if (elapsed >= timeout) {
-                        return -2;
-                    }
-                }
-                continue;
-            } else {
-                return -2;
-            }
-        } else if (zmq_errno() == ETERM) {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        } else if (zmq_errno() == EINTR) {
-            LOG(debug) << "Receive interrupted by system call";
-            return nbytes;
-        }else {
-            LOG(error) << "Failed receiving on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-            return nbytes;
-        }
-    }
-}
-
-int64_t Socket::Send(vector<MessagePtr>& msgVec, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0) {
-        flags = ZMQ_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    // put it into zmq message
-    const unsigned int vecSize = msgVec.size();
-    ZMsg zmqMsg(vecSize * sizeof(MetaHeader));
-
-    // prepare the message with shm metas
-    MetaHeader* metas = static_cast<MetaHeader*>(zmqMsg.Data());
-
-    for (auto& msg : msgVec) {
-        Message* shmMsg = static_cast<Message*>(msg.get());
-        std::memcpy(metas++, &(shmMsg->fMeta), sizeof(MetaHeader));
-    }
-
-    while (!fManager.Interrupted()) {
-        int64_t totalSize = 0;
-        int nbytes = zmq_msg_send(zmqMsg.Msg(), fSocket, flags);
-        if (nbytes > 0) {
-            assert(static_cast<unsigned int>(nbytes) == (vecSize * sizeof(MetaHeader))); // all or nothing
-
-            for (auto& msg : msgVec) {
-                Message* shmMsg = static_cast<Message*>(msg.get());
-                shmMsg->fQueued = true;
-                totalSize += shmMsg->fMeta.fSize;
-            }
-
-            // store statistics on how many messages have been sent
-            fMessagesTx++;
-            fBytesTx += totalSize;
-
-            return totalSize;
-        } else if (zmq_errno() == EAGAIN) {
-            if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                if (timeout > 0) {
-                    elapsed += fSndTimeout;
-                    if (elapsed >= timeout) {
-                        return -2;
-                    }
-                }
-                continue;
-            } else {
-                return -2;
-            }
-        } else if (zmq_errno() == ETERM) {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        } else if (zmq_errno() == EINTR) {
-            LOG(debug) << "Send interrupted by system call";
-            return nbytes;
-        }else {
-            LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-            return nbytes;
-        }
-    }
-
-    return -1;
-}
-
-int64_t Socket::Receive(vector<MessagePtr>& msgVec, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0) {
-        flags = ZMQ_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    ZMsg zmqMsg;
-
-    while (!fManager.Interrupted()) {
-        int64_t totalSize = 0;
-        int nbytes = zmq_msg_recv(zmqMsg.Msg(), fSocket, flags);
-        if (nbytes > 0) {
-            MetaHeader* hdrVec = static_cast<MetaHeader*>(zmqMsg.Data());
-            const auto hdrVecSize = zmqMsg.Size();
-
-            assert(hdrVecSize > 0);
-            if (hdrVecSize % sizeof(MetaHeader) != 0) {
-                throw SocketError(
-                    tools::ToString("Received message is not a valid FairMQ shared memory message. ",
-                        "Possibly due to a misconfigured transport on the sender side. ",
-                        "Expected size of ", sizeof(MetaHeader), " bytes, received ", nbytes));
-            }
-
-            const auto numMessages = hdrVecSize / sizeof(MetaHeader);
-            msgVec.reserve(numMessages);
-
-            for (size_t m = 0; m < numMessages; m++) {
-                // create new message (part)
-                msgVec.emplace_back(tools::make_unique<Message>(fManager, hdrVec[m], GetTransport()));
-                Message* shmMsg = static_cast<Message*>(msgVec.back().get());
-                totalSize += shmMsg->GetSize();
-            }
-
-            // store statistics on how many messages have been received (handle all parts as a single message)
-            fMessagesRx++;
-            fBytesRx += totalSize;
-
-            return totalSize;
-        } else if (zmq_errno() == EAGAIN) {
-            if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
-                if (timeout > 0) {
-                    elapsed += fRcvTimeout;
-                    if (elapsed >= timeout) {
-                        return -2;
-                    }
-                }
-                continue;
-            } else {
-                return -2;
-            }
-        } else if (zmq_errno() == EINTR) {
-            LOG(debug) << "Receive interrupted by system call";
-            return nbytes;
-        } else {
-            LOG(error) << "Failed receiving on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
-            return nbytes;
-        }
-    }
-
-    return -1;
-}
-
-void Socket::Close()
-{
-    // LOG(debug) << "Closing socket " << fId;
-
-    if (fSocket == nullptr) {
-        return;
-    }
-
-    if (zmq_close(fSocket) != 0) {
-        LOG(error) << "Failed closing socket " << fId << ", reason: " << zmq_strerror(errno);
-    }
-
-    fSocket = nullptr;
-}
-
-void Socket::SetOption(const string& option, const void* value, size_t valueSize)
-{
-    if (zmq_setsockopt(fSocket, GetConstant(option), value, valueSize) < 0) {
-        LOG(error) << "Failed setting socket option, reason: " << zmq_strerror(errno);
-    }
-}
-
-void Socket::GetOption(const string& option, void* value, size_t* valueSize)
-{
-    if (zmq_getsockopt(fSocket, GetConstant(option), value, valueSize) < 0) {
-        LOG(error) << "Failed getting socket option, reason: " << zmq_strerror(errno);
-    }
-}
-
-void Socket::SetLinger(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_LINGER, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int Socket::GetLinger() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_LINGER, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-void Socket::SetSndBufSize(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_SNDHWM, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int Socket::GetSndBufSize() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_SNDHWM, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-void Socket::SetRcvBufSize(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_RCVHWM, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int Socket::GetRcvBufSize() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_RCVHWM, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-void Socket::SetSndKernelSize(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_SNDBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int Socket::GetSndKernelSize() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_SNDBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-void Socket::SetRcvKernelSize(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_RCVBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int Socket::GetRcvKernelSize() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_RCVBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-int Socket::GetConstant(const string& constant)
-{
-    if (constant == "") return 0;
-    if (constant == "sub") return ZMQ_SUB;
-    if (constant == "pub") return ZMQ_PUB;
-    if (constant == "xsub") return ZMQ_XSUB;
-    if (constant == "xpub") return ZMQ_XPUB;
-    if (constant == "push") return ZMQ_PUSH;
-    if (constant == "pull") return ZMQ_PULL;
-    if (constant == "req") return ZMQ_REQ;
-    if (constant == "rep") return ZMQ_REP;
-    if (constant == "dealer") return ZMQ_DEALER;
-    if (constant == "router") return ZMQ_ROUTER;
-    if (constant == "pair") return ZMQ_PAIR;
-
-    if (constant == "snd-hwm") return ZMQ_SNDHWM;
-    if (constant == "rcv-hwm") return ZMQ_RCVHWM;
-    if (constant == "snd-size") return ZMQ_SNDBUF;
-    if (constant == "rcv-size") return ZMQ_RCVBUF;
-    if (constant == "snd-more") return ZMQ_SNDMORE;
-    if (constant == "rcv-more") return ZMQ_RCVMORE;
-
-    if (constant == "linger") return ZMQ_LINGER;
-    if (constant == "no-block") return ZMQ_DONTWAIT;
-    if (constant == "snd-more no-block") return ZMQ_DONTWAIT|ZMQ_SNDMORE;
-
-    return -1;
-}
-
-}
-}
-}

fairmq/shmem/Socket.h

@@ -8,13 +8,18 @@
 #ifndef FAIR_MQ_SHMEM_SOCKET_H_
 #define FAIR_MQ_SHMEM_SOCKET_H_
 
+#include "Common.h"
 #include "Manager.h"
+#include "Message.h"
 
 #include <FairMQSocket.h>
 #include <FairMQMessage.h>
+#include <FairMQLogger.h>
+#include <fairmq/Tools.h>
+
+#include <zmq.h>
 
 #include <atomic>
-#include <memory> // unique_ptr
 
 class FairMQTransportFactory;
 
@@ -25,47 +30,483 @@ namespace mq
 namespace shmem
 {
 
+struct ZMsg
+{
+    ZMsg() { int rc __attribute__((unused)) = zmq_msg_init(&fMsg); assert(rc == 0); }
+    explicit ZMsg(size_t size) { int rc __attribute__((unused)) = zmq_msg_init_size(&fMsg, size); assert(rc == 0); }
+    ~ZMsg() { int rc __attribute__((unused)) = zmq_msg_close(&fMsg); assert(rc == 0); }
+
+    void* Data() { return zmq_msg_data(&fMsg); }
+    size_t Size() { return zmq_msg_size(&fMsg); }
+    zmq_msg_t* Msg() { return &fMsg; }
+
+    zmq_msg_t fMsg;
+};
+
 class Socket final : public fair::mq::Socket
 {
   public:
-    Socket(Manager& manager, const std::string& type, const std::string& name, const std::string& id = "", void* context = nullptr, FairMQTransportFactory* fac = nullptr);
+    Socket(Manager& manager, const std::string& type, const std::string& name, const std::string& id, void* context, FairMQTransportFactory* fac = nullptr)
+        : fair::mq::Socket(fac)
+        , fSocket(nullptr)
+        , fManager(manager)
+        , fId(id + "." + name + "." + type)
+        , fBytesTx(0)
+        , fBytesRx(0)
+        , fMessagesTx(0)
+        , fMessagesRx(0)
+        , fSndTimeout(100)
+        , fRcvTimeout(100)
+    {
+        assert(context);
+        fSocket = zmq_socket(context, GetConstant(type));
+
+        if (fSocket == nullptr) {
+            LOG(error) << "Failed creating socket " << fId << ", reason: " << zmq_strerror(errno);
+            throw SocketError(tools::ToString("Failed creating socket ", fId, ", reason: ", zmq_strerror(errno)));
+        }
+
+        if (zmq_setsockopt(fSocket, ZMQ_IDENTITY, fId.c_str(), fId.length()) != 0) {
+            LOG(error) << "Failed setting ZMQ_IDENTITY socket option, reason: " << zmq_strerror(errno);
+        }
+
+        // Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
+        // Default value for ZeroMQ is -1, which is to wait forever.
+        int linger = 1000;
+        if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) {
+            LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
+        }
+
+        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0) {
+            LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
+        }
+
+        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0) {
+            LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
+        }
+
+        // if (type == "sub")
+        // {
+        //     if (zmq_setsockopt(fSocket, ZMQ_SUBSCRIBE, nullptr, 0) != 0)
+        //     {
+        //         LOG(error) << "Failed setting ZMQ_SUBSCRIBE socket option, reason: " << zmq_strerror(errno);
+        //     }
+        // }
+
+        if (type == "sub" || type == "pub") {
+            LOG(error) << "PUB/SUB socket type is not supported for shared memory transport";
+            throw SocketError("PUB/SUB socket type is not supported for shared memory transport");
+        }
+        LOG(debug) << "Created socket " << GetId();
+    }
+
     Socket(const Socket&) = delete;
     Socket operator=(const Socket&) = delete;
 
     std::string GetId() const override { return fId; }
 
-    bool Bind(const std::string& address) override;
-    bool Connect(const std::string& address) override;
+    bool Bind(const std::string& address) override
+    {
+        // LOG(info) << "binding socket " << fId << " on " << address;
+        if (zmq_bind(fSocket, address.c_str()) != 0) {
+            if (errno == EADDRINUSE) {
+                // do not print error in this case, this is handled by FairMQDevice in case no connection could be established after trying a number of random ports from a range.
+                return false;
+            }
+            LOG(error) << "Failed binding socket " << fId << ", reason: " << zmq_strerror(errno);
+            return false;
+        }
+        return true;
+    }
 
-    int Send(MessagePtr& msg, const int timeout = -1) override;
-    int Receive(MessagePtr& msg, const int timeout = -1) override;
-    int64_t Send(std::vector<MessagePtr>& msgVec, const int timeout = -1) override;
-    int64_t Receive(std::vector<MessagePtr>& msgVec, const int timeout = -1) override;
+    bool Connect(const std::string& address) override
+    {
+        // LOG(info) << "connecting socket " << fId << " on " << address;
+        if (zmq_connect(fSocket, address.c_str()) != 0) {
+            LOG(error) << "Failed connecting socket " << fId << ", reason: " << zmq_strerror(errno);
+            return false;
+        }
+        return true;
+    }
+
+    int Send(MessagePtr& msg, const int timeout = -1) override
+    {
+        int flags = 0;
+        if (timeout == 0) {
+            flags = ZMQ_DONTWAIT;
+        }
+        int elapsed = 0;
+
+        Message* shmMsg = static_cast<Message*>(msg.get());
+        ZMsg zmqMsg(sizeof(MetaHeader));
+        std::memcpy(zmqMsg.Data(), &(shmMsg->fMeta), sizeof(MetaHeader));
+
+        while (true && !fManager.Interrupted()) {
+            int nbytes = zmq_msg_send(zmqMsg.Msg(), fSocket, flags);
+            if (nbytes > 0) {
+                shmMsg->fQueued = true;
+                ++fMessagesTx;
+                size_t size = msg->GetSize();
+                fBytesTx += size;
+                return size;
+            } else if (zmq_errno() == EAGAIN) {
+                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
+                    if (timeout > 0) {
+                        elapsed += fSndTimeout;
+                        if (elapsed >= timeout) {
+                            return -2;
+                        }
+                    }
+                    continue;
+                } else {
+                    return -2;
+                }
+            } else if (zmq_errno() == ETERM) {
+                LOG(info) << "terminating socket " << fId;
+                return -1;
+            } else if (zmq_errno() == EINTR) {
+                LOG(debug) << "Send interrupted by system call";
+                return nbytes;
+            }else {
+                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
+                return nbytes;
+            }
+        }
+
+        return -1;
+    }
+
+    int Receive(MessagePtr& msg, const int timeout = -1) override
+    {
+        int flags = 0;
+        if (timeout == 0) {
+            flags = ZMQ_DONTWAIT;
+        }
+        int elapsed = 0;
+
+        ZMsg zmqMsg;
+
+        while (true) {
+            Message* shmMsg = static_cast<Message*>(msg.get());
+            int nbytes = zmq_msg_recv(zmqMsg.Msg(), fSocket, flags);
+            if (nbytes > 0) {
+                // check for number of received messages. must be 1
+                if (nbytes != sizeof(MetaHeader)) {
+                    throw SocketError(
+                        tools::ToString("Received message is not a valid FairMQ shared memory message. ",
+                            "Possibly due to a misconfigured transport on the sender side. ",
+                            "Expected size of ", sizeof(MetaHeader), " bytes, received ", nbytes));
+                }
+
+                MetaHeader* hdr = static_cast<MetaHeader*>(zmqMsg.Data());
+                size_t size = hdr->fSize;
+                shmMsg->fMeta = *hdr;
+
+                fBytesRx += size;
+                ++fMessagesRx;
+                return size;
+            } else if (zmq_errno() == EAGAIN) {
+                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
+                    if (timeout > 0) {
+                        elapsed += fRcvTimeout;
+                        if (elapsed >= timeout) {
+                            return -2;
+                        }
+                    }
+                    continue;
+                } else {
+                    return -2;
+                }
+            } else if (zmq_errno() == ETERM) {
+                LOG(info) << "terminating socket " << fId;
+                return -1;
+            } else if (zmq_errno() == EINTR) {
+                LOG(debug) << "Receive interrupted by system call";
+                return nbytes;
+            }else {
+                LOG(error) << "Failed receiving on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
+                return nbytes;
+            }
+        }
+    }
+
+    int64_t Send(std::vector<MessagePtr>& msgVec, const int timeout = -1) override
+    {
+        int flags = 0;
+        if (timeout == 0) {
+            flags = ZMQ_DONTWAIT;
+        }
+        int elapsed = 0;
+
+        // put it into zmq message
+        const unsigned int vecSize = msgVec.size();
+        ZMsg zmqMsg(vecSize * sizeof(MetaHeader));
+
+        // prepare the message with shm metas
+        MetaHeader* metas = static_cast<MetaHeader*>(zmqMsg.Data());
+
+        for (auto& msg : msgVec) {
+            Message* shmMsg = static_cast<Message*>(msg.get());
+            std::memcpy(metas++, &(shmMsg->fMeta), sizeof(MetaHeader));
+        }
+
+        while (!fManager.Interrupted()) {
+            int64_t totalSize = 0;
+            int nbytes = zmq_msg_send(zmqMsg.Msg(), fSocket, flags);
+            if (nbytes > 0) {
+                assert(static_cast<unsigned int>(nbytes) == (vecSize * sizeof(MetaHeader))); // all or nothing
+
+                for (auto& msg : msgVec) {
+                    Message* shmMsg = static_cast<Message*>(msg.get());
+                    shmMsg->fQueued = true;
+                    totalSize += shmMsg->fMeta.fSize;
+                }
+
+                // store statistics on how many messages have been sent
+                fMessagesTx++;
+                fBytesTx += totalSize;
+
+                return totalSize;
+            } else if (zmq_errno() == EAGAIN) {
+                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
+                    if (timeout > 0) {
+                        elapsed += fSndTimeout;
+                        if (elapsed >= timeout) {
+                            return -2;
+                        }
+                    }
+                    continue;
+                } else {
+                    return -2;
+                }
+            } else if (zmq_errno() == ETERM) {
+                LOG(info) << "terminating socket " << fId;
+                return -1;
+            } else if (zmq_errno() == EINTR) {
+                LOG(debug) << "Send interrupted by system call";
+                return nbytes;
+            }else {
+                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
+                return nbytes;
+            }
+        }
+
+        return -1;
+    }
+
+    int64_t Receive(std::vector<MessagePtr>& msgVec, const int timeout = -1) override
+    {
+        int flags = 0;
+        if (timeout == 0) {
+            flags = ZMQ_DONTWAIT;
+        }
+        int elapsed = 0;
+
+        ZMsg zmqMsg;
+
+        while (!fManager.Interrupted()) {
+            int64_t totalSize = 0;
+            int nbytes = zmq_msg_recv(zmqMsg.Msg(), fSocket, flags);
+            if (nbytes > 0) {
+                MetaHeader* hdrVec = static_cast<MetaHeader*>(zmqMsg.Data());
+                const auto hdrVecSize = zmqMsg.Size();
+
+                assert(hdrVecSize > 0);
+                if (hdrVecSize % sizeof(MetaHeader) != 0) {
+                    throw SocketError(
+                        tools::ToString("Received message is not a valid FairMQ shared memory message. ",
+                            "Possibly due to a misconfigured transport on the sender side. ",
+                            "Expected size of ", sizeof(MetaHeader), " bytes, received ", nbytes));
+                }
+
+                const auto numMessages = hdrVecSize / sizeof(MetaHeader);
+                msgVec.reserve(numMessages);
+
+                for (size_t m = 0; m < numMessages; m++) {
+                    // create new message (part)
+                    msgVec.emplace_back(tools::make_unique<Message>(fManager, hdrVec[m], GetTransport()));
+                    Message* shmMsg = static_cast<Message*>(msgVec.back().get());
+                    totalSize += shmMsg->GetSize();
+                }
+
+                // store statistics on how many messages have been received (handle all parts as a single message)
+                fMessagesRx++;
+                fBytesRx += totalSize;
+
+                return totalSize;
+            } else if (zmq_errno() == EAGAIN) {
+                if (!fManager.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
+                    if (timeout > 0) {
+                        elapsed += fRcvTimeout;
+                        if (elapsed >= timeout) {
+                            return -2;
+                        }
+                    }
+                    continue;
+                } else {
+                    return -2;
+                }
+            } else if (zmq_errno() == EINTR) {
+                LOG(debug) << "Receive interrupted by system call";
+                return nbytes;
+            } else {
+                LOG(error) << "Failed receiving on socket " << fId << ", errno: " << errno << ", reason: " << zmq_strerror(errno) << ", nbytes = " << nbytes;
+                return nbytes;
+            }
+        }
+
+        return -1;
+    }
 
     void* GetSocket() const { return fSocket; }
 
-    void Close() override;
+    void Close() override
+    {
+        // LOG(debug) << "Closing socket " << fId;
 
-    void SetOption(const std::string& option, const void* value, size_t valueSize) override;
-    void GetOption(const std::string& option, void* value, size_t* valueSize) override;
+        if (fSocket == nullptr) {
+            return;
+        }
 
-    void SetLinger(const int value) override;
-    int GetLinger() const override;
-    void SetSndBufSize(const int value) override;
-    int GetSndBufSize() const override;
-    void SetRcvBufSize(const int value) override;
-    int GetRcvBufSize() const override;
-    void SetSndKernelSize(const int value) override;
-    int GetSndKernelSize() const override;
-    void SetRcvKernelSize(const int value) override;
-    int GetRcvKernelSize() const override;
+        if (zmq_close(fSocket) != 0) {
+            LOG(error) << "Failed closing socket " << fId << ", reason: " << zmq_strerror(errno);
+        }
+
+        fSocket = nullptr;
+    }
+
+    void SetOption(const std::string& option, const void* value, size_t valueSize) override
+    {
+        if (zmq_setsockopt(fSocket, GetConstant(option), value, valueSize) < 0) {
+            LOG(error) << "Failed setting socket option, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    void GetOption(const std::string& option, void* value, size_t* valueSize) override
+    {
+        if (zmq_getsockopt(fSocket, GetConstant(option), value, valueSize) < 0) {
+            LOG(error) << "Failed getting socket option, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    void SetLinger(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_LINGER, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed setting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetLinger() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_LINGER, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    void SetSndBufSize(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_SNDHWM, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed setting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetSndBufSize() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_SNDHWM, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    void SetRcvBufSize(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_RCVHWM, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed setting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetRcvBufSize() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_RCVHWM, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    void SetSndKernelSize(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_SNDBUF, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetSndKernelSize() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_SNDBUF, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    void SetRcvKernelSize(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_RCVBUF, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetRcvKernelSize() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_RCVBUF, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
 
     unsigned long GetBytesTx() const override { return fBytesTx; }
     unsigned long GetBytesRx() const override { return fBytesRx; }
     unsigned long GetMessagesTx() const override { return fMessagesTx; }
     unsigned long GetMessagesRx() const override { return fMessagesRx; }
 
-    static int GetConstant(const std::string& constant);
+    static int GetConstant(const std::string& constant)
+    {
+        if (constant == "") return 0;
+        if (constant == "sub") return ZMQ_SUB;
+        if (constant == "pub") return ZMQ_PUB;
+        if (constant == "xsub") return ZMQ_XSUB;
+        if (constant == "xpub") return ZMQ_XPUB;
+        if (constant == "push") return ZMQ_PUSH;
+        if (constant == "pull") return ZMQ_PULL;
+        if (constant == "req") return ZMQ_REQ;
+        if (constant == "rep") return ZMQ_REP;
+        if (constant == "dealer") return ZMQ_DEALER;
+        if (constant == "router") return ZMQ_ROUTER;
+        if (constant == "pair") return ZMQ_PAIR;
+
+        if (constant == "snd-hwm") return ZMQ_SNDHWM;
+        if (constant == "rcv-hwm") return ZMQ_RCVHWM;
+        if (constant == "snd-size") return ZMQ_SNDBUF;
+        if (constant == "rcv-size") return ZMQ_RCVBUF;
+        if (constant == "snd-more") return ZMQ_SNDMORE;
+        if (constant == "rcv-more") return ZMQ_RCVMORE;
+
+        if (constant == "linger") return ZMQ_LINGER;
+        if (constant == "no-block") return ZMQ_DONTWAIT;
+        if (constant == "snd-more no-block") return ZMQ_DONTWAIT|ZMQ_SNDMORE;
+
+        return -1;
+    }
 
     ~Socket() override { Close(); }
 

fairmq/shmem/TransportFactory.cxx

@@ -1,223 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2016-2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "TransportFactory.h"
-
-#include <FairMQLogger.h>
-#include <fairmq/Tools.h>
-
-#include <zmq.h>
-
-#include <boost/version.hpp>
-#include <boost/interprocess/managed_shared_memory.hpp>
-
-#include <boost/interprocess/ipc/message_queue.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-#include <boost/interprocess/sync/scoped_lock.hpp>
-
-#include <chrono>
-#include <thread>
-#include <cstdlib> // getenv
-
-using namespace std;
-
-namespace bpt = ::boost::posix_time;
-namespace bipc = ::boost::interprocess;
-
-namespace fair
-{
-namespace mq
-{
-namespace shmem
-{
-
-Transport TransportFactory::fTransportType = Transport::SHM;
-
-TransportFactory::TransportFactory(const string& id, const ProgOptions* config)
-    : fair::mq::TransportFactory(id)
-    , fDeviceId(id)
-    , fShmId()
-    , fZMQContext(nullptr)
-    , fManager(nullptr)
-    , fHeartbeatThread()
-    , fSendHeartbeats(true)
-    , fMsgCounter(0)
-{
-    int major, minor, patch;
-    zmq_version(&major, &minor, &patch);
-    LOG(debug) << "Transport: Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
-               << "boost::interprocess (" << (BOOST_VERSION / 100000) << "." << (BOOST_VERSION / 100 % 1000) << "." << (BOOST_VERSION % 100) << ")";
-
-    fZMQContext = zmq_ctx_new();
-    if (!fZMQContext) {
-        throw runtime_error(tools::ToString("failed creating context, reason: ", zmq_strerror(errno)));
-    }
-
-    int numIoThreads = 1;
-    string sessionName = "default";
-    size_t segmentSize = 2000000000;
-    bool autolaunchMonitor = false;
-    if (config) {
-        numIoThreads = config->GetProperty<int>("io-threads", numIoThreads);
-        sessionName = config->GetProperty<string>("session", sessionName);
-        segmentSize = config->GetProperty<size_t>("shm-segment-size", segmentSize);
-        autolaunchMonitor = config->GetProperty<bool>("shm-monitor", autolaunchMonitor);
-    } else {
-        LOG(debug) << "ProgOptions not available! Using defaults.";
-    }
-
-    fShmId = buildShmIdFromSessionIdAndUserId(sessionName);
-
-    try {
-        if (zmq_ctx_set(fZMQContext, ZMQ_IO_THREADS, numIoThreads) != 0) {
-            LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
-        }
-
-        // Set the maximum number of allowed sockets on the context.
-        if (zmq_ctx_set(fZMQContext, ZMQ_MAX_SOCKETS, 10000) != 0) {
-            LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
-        }
-
-        if (autolaunchMonitor) {
-            Manager::StartMonitor(fShmId);
-        }
-
-        fManager = tools::make_unique<Manager>(fShmId, segmentSize);
-
-    } catch (bipc::interprocess_exception& e) {
-        LOG(error) << "Could not initialize shared memory transport: " << e.what();
-        throw runtime_error(tools::ToString("Could not initialize shared memory transport: ", e.what()));
-    }
-
-    fSendHeartbeats = true;
-    fHeartbeatThread = thread(&TransportFactory::SendHeartbeats, this);
-}
-
-void TransportFactory::SendHeartbeats()
-{
-    string controlQueueName("fmq_" + fShmId + "_cq");
-    while (fSendHeartbeats) {
-        try {
-            bipc::message_queue mq(bipc::open_only, controlQueueName.c_str());
-            bpt::ptime sndTill = bpt::microsec_clock::universal_time() + bpt::milliseconds(100);
-            if (mq.timed_send(fDeviceId.c_str(), fDeviceId.size(), 0, sndTill)) {
-                this_thread::sleep_for(chrono::milliseconds(100));
-            } else {
-                LOG(debug) << "control queue timeout";
-            }
-        } catch (bipc::interprocess_exception& ie) {
-            this_thread::sleep_for(chrono::milliseconds(500));
-            // LOG(warn) << "no " << controlQueueName << " found";
-        }
-    }
-}
-
-MessagePtr TransportFactory::CreateMessage()
-{
-    return tools::make_unique<Message>(*fManager, this);
-}
-
-MessagePtr TransportFactory::CreateMessage(const size_t size)
-{
-    return tools::make_unique<Message>(*fManager, size, this);
-}
-
-MessagePtr TransportFactory::CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    return tools::make_unique<Message>(*fManager, data, size, ffn, hint, this);
-}
-
-MessagePtr TransportFactory::CreateMessage(UnmanagedRegionPtr& region, void* data, const size_t size, void* hint)
-{
-    return tools::make_unique<Message>(*fManager, region, data, size, hint, this);
-}
-
-SocketPtr TransportFactory::CreateSocket(const string& type, const string& name)
-{
-    assert(fZMQContext);
-    return tools::make_unique<Socket>(*fManager, type, name, GetId(), fZMQContext, this);
-}
-
-PollerPtr TransportFactory::CreatePoller(const vector<FairMQChannel>& channels) const
-{
-    return tools::make_unique<Poller>(channels);
-}
-
-PollerPtr TransportFactory::CreatePoller(const vector<FairMQChannel*>& channels) const
-{
-    return tools::make_unique<Poller>(channels);
-}
-
-PollerPtr TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList) const
-{
-    return tools::make_unique<Poller>(channelsMap, channelList);
-}
-
-UnmanagedRegionPtr TransportFactory::CreateUnmanagedRegion(const size_t size, RegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
-{
-    return tools::make_unique<UnmanagedRegion>(*fManager, size, callback, path, flags);
-}
-
-UnmanagedRegionPtr TransportFactory::CreateUnmanagedRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
-{
-    return tools::make_unique<UnmanagedRegion>(*fManager, size, userFlags, callback, path, flags);
-}
-
-void TransportFactory::SubscribeToRegionEvents(RegionEventCallback callback)
-{
-    fManager->SubscribeToRegionEvents(callback);
-}
-
-void TransportFactory::UnsubscribeFromRegionEvents()
-{
-    fManager->UnsubscribeFromRegionEvents();
-}
-
-vector<fair::mq::RegionInfo> TransportFactory::GetRegionInfo()
-{
-    return fManager->GetRegionInfo();
-}
-
-Transport TransportFactory::GetType() const
-{
-    return fTransportType;
-}
-
-void TransportFactory::Reset()
-{
-    if (fMsgCounter.load() != 0) {
-        LOG(error) << "Message counter during Reset expected to be 0, found: " << fMsgCounter.load();
-        throw MessageError(tools::ToString("Message counter during Reset expected to be 0, found: ", fMsgCounter.load()));
-    }
-}
-
-
-TransportFactory::~TransportFactory()
-{
-    LOG(debug) << "Destroying Shared Memory transport...";
-    fSendHeartbeats = false;
-    fHeartbeatThread.join();
-
-    if (fZMQContext) {
-        if (zmq_ctx_term(fZMQContext) != 0) {
-            if (errno == EINTR) {
-                LOG(error) << "failed closing context, reason: " << zmq_strerror(errno);
-            } else {
-                fZMQContext = nullptr;
-                return;
-            }
-        }
-    } else {
-        LOG(error) << "context not available for shutdown";
-    }
-}
-
-} // namespace shmem
-} // namespace mq
-} // namespace fair

fairmq/shmem/TransportFactory.h

@@ -18,11 +18,16 @@
 
 #include <FairMQTransportFactory.h>
 #include <fairmq/ProgOptions.h>
+#include <FairMQLogger.h>
+#include <fairmq/Tools.h>
 
-#include <vector>
+#include <boost/version.hpp>
+
+#include <zmq.h>
+
+#include <memory> // unique_ptr
 #include <string>
-#include <thread>
-#include <atomic>
+#include <vector>
 
 namespace fair
 {
@@ -34,50 +39,172 @@ namespace shmem
 class TransportFactory final : public fair::mq::TransportFactory
 {
   public:
-    TransportFactory(const std::string& id = "", const ProgOptions* config = nullptr);
+    TransportFactory(const std::string& id = "", const ProgOptions* config = nullptr)
+        : fair::mq::TransportFactory(id)
+        , fDeviceId(id)
+        , fShmId()
+        , fZMQContext(zmq_ctx_new())
+        , fManager(nullptr)
+    {
+        int major, minor, patch;
+        zmq_version(&major, &minor, &patch);
+        LOG(debug) << "Transport: Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
+                << "boost::interprocess (" << (BOOST_VERSION / 100000) << "." << (BOOST_VERSION / 100 % 1000) << "." << (BOOST_VERSION % 100) << ")";
+
+        if (!fZMQContext) {
+            throw std::runtime_error(tools::ToString("failed creating context, reason: ", zmq_strerror(errno)));
+        }
+
+        int numIoThreads = 1;
+        std::string sessionName = "default";
+        size_t segmentSize = 2000000000;
+        bool autolaunchMonitor = false;
+        bool throwOnBadAlloc = true;
+        if (config) {
+            numIoThreads = config->GetProperty<int>("io-threads", numIoThreads);
+            sessionName = config->GetProperty<std::string>("session", sessionName);
+            segmentSize = config->GetProperty<size_t>("shm-segment-size", segmentSize);
+            autolaunchMonitor = config->GetProperty<bool>("shm-monitor", autolaunchMonitor);
+            throwOnBadAlloc = config->GetProperty<bool>("shm-throw-bad-alloc", throwOnBadAlloc);
+        } else {
+            LOG(debug) << "ProgOptions not available! Using defaults.";
+        }
+
+        fShmId = buildShmIdFromSessionIdAndUserId(sessionName);
+
+        try {
+            if (zmq_ctx_set(fZMQContext, ZMQ_IO_THREADS, numIoThreads) != 0) {
+                LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
+            }
+
+            // Set the maximum number of allowed sockets on the context.
+            if (zmq_ctx_set(fZMQContext, ZMQ_MAX_SOCKETS, 10000) != 0) {
+                LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
+            }
+
+            if (autolaunchMonitor) {
+                Manager::StartMonitor(fShmId);
+            }
+
+            fManager = tools::make_unique<Manager>(fShmId, fDeviceId, segmentSize, throwOnBadAlloc);
+        } catch (boost::interprocess::interprocess_exception& e) {
+            LOG(error) << "Could not initialize shared memory transport: " << e.what();
+            throw std::runtime_error(tools::ToString("Could not initialize shared memory transport: ", e.what()));
+        }
+    }
+
     TransportFactory(const TransportFactory&) = delete;
     TransportFactory operator=(const TransportFactory&) = delete;
 
-    MessagePtr CreateMessage() override;
-    MessagePtr CreateMessage(const size_t size) override;
-    MessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-    MessagePtr CreateMessage(UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0) override;
+    MessagePtr CreateMessage() override
+    {
+        return tools::make_unique<Message>(*fManager, this);
+    }
 
-    SocketPtr CreateSocket(const std::string& type, const std::string& name) override;
+    MessagePtr CreateMessage(Alignment alignment) override
+    {
+        return tools::make_unique<Message>(*fManager, alignment, this);
+    }
 
-    PollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override;
-    PollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const override;
-    PollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
+    MessagePtr CreateMessage(const size_t size) override
+    {
+        return tools::make_unique<Message>(*fManager, size, this);
+    }
 
-    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const override;
-    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const override;
+    MessagePtr CreateMessage(const size_t size, Alignment alignment) override
+    {
+        return tools::make_unique<Message>(*fManager, size, alignment, this);
+    }
 
-    void SubscribeToRegionEvents(RegionEventCallback callback) override;
-    void UnsubscribeFromRegionEvents() override;
-    std::vector<fair::mq::RegionInfo> GetRegionInfo() override;
+    MessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override
+    {
+        return tools::make_unique<Message>(*fManager, data, size, ffn, hint, this);
+    }
 
-    Transport GetType() const override;
+    MessagePtr CreateMessage(UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0) override
+    {
+        return tools::make_unique<Message>(*fManager, region, data, size, hint, this);
+    }
+
+    SocketPtr CreateSocket(const std::string& type, const std::string& name) override
+    {
+        return tools::make_unique<Socket>(*fManager, type, name, GetId(), fZMQContext, this);
+    }
+
+    PollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override
+    {
+        return tools::make_unique<Poller>(channels);
+    }
+
+    PollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const override
+    {
+        return tools::make_unique<Poller>(channels);
+    }
+
+    PollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override
+    {
+        return tools::make_unique<Poller>(channelsMap, channelList);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) override
+    {
+        return CreateUnmanagedRegion(size, 0, callback, nullptr, path, flags);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, RegionBulkCallback bulkCallback = nullptr, const std::string& path = "", int flags = 0) override
+    {
+        return CreateUnmanagedRegion(size, 0, nullptr, bulkCallback, path, flags);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) override
+    {
+        return CreateUnmanagedRegion(size, userFlags, callback, nullptr, path, flags);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionBulkCallback bulkCallback = nullptr, const std::string& path = "", int flags = 0) override
+    {
+        return CreateUnmanagedRegion(size, userFlags, nullptr, bulkCallback, path, flags);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string& path, int flags)
+    {
+        return tools::make_unique<UnmanagedRegion>(*fManager, size, userFlags, callback, bulkCallback, path, flags, this);
+    }
+
+    void SubscribeToRegionEvents(RegionEventCallback callback) override { fManager->SubscribeToRegionEvents(callback); }
+    bool SubscribedToRegionEvents() override { return fManager->SubscribedToRegionEvents(); }
+    void UnsubscribeFromRegionEvents() override { fManager->UnsubscribeFromRegionEvents(); }
+    std::vector<fair::mq::RegionInfo> GetRegionInfo() override { return fManager->GetRegionInfo(); }
+
+    Transport GetType() const override { return fair::mq::Transport::SHM; }
 
     void Interrupt() override { fManager->Interrupt(); }
     void Resume() override { fManager->Resume(); }
-    void Reset() override;
+    void Reset() override { fManager->Reset(); }
 
-    void IncrementMsgCounter() { ++fMsgCounter; }
-    void DecrementMsgCounter() { --fMsgCounter; }
+    ~TransportFactory() override
+    {
+        LOG(debug) << "Destroying Shared Memory transport...";
 
-    ~TransportFactory() override;
+        if (fZMQContext) {
+            if (zmq_ctx_term(fZMQContext) != 0) {
+                if (errno == EINTR) {
+                    LOG(error) << "failed closing context, reason: " << zmq_strerror(errno);
+                } else {
+                    fZMQContext = nullptr;
+                    return;
+                }
+            }
+        } else {
+            LOG(error) << "context not available for shutdown";
+        }
+    }
 
   private:
-    void SendHeartbeats();
-
-    static Transport fTransportType;
     std::string fDeviceId;
     std::string fShmId;
     void* fZMQContext;
     std::unique_ptr<Manager> fManager;
-    std::thread fHeartbeatThread;
-    std::atomic<bool> fSendHeartbeats;
-    std::atomic<int32_t> fMsgCounter;
 };
 
 } // namespace shmem

fairmq/shmem/UnmanagedRegion.h

@@ -36,22 +36,27 @@ class UnmanagedRegion final : public fair::mq::UnmanagedRegion
     friend class Socket;
 
   public:
-    UnmanagedRegion(Manager& manager, const size_t size, RegionCallback callback, const std::string& path = "", int flags = 0)
-        : UnmanagedRegion(manager, size, 0, callback, path, flags)
-    {}
-
-    UnmanagedRegion(Manager& manager, const size_t size, const int64_t userFlags, RegionCallback callback, const std::string& path = "", int flags = 0)
-        : fManager(manager)
+    UnmanagedRegion(Manager& manager,
+                    const size_t size,
+                    const int64_t userFlags,
+                    RegionCallback callback,
+                    RegionBulkCallback bulkCallback,
+                    const std::string& path = "",
+                    int flags = 0,
+                    FairMQTransportFactory* factory = nullptr)
+        : FairMQUnmanagedRegion(factory)
+        , fManager(manager)
         , fRegion(nullptr)
         , fRegionId(0)
     {
-        auto result = fManager.CreateRegion(size, userFlags, callback, path, flags);
+        auto result = fManager.CreateRegion(size, userFlags, callback, bulkCallback, path, flags);
         fRegion = result.first;
         fRegionId = result.second;
     }
 
     void* GetData() const override { return fRegion->get_address(); }
     size_t GetSize() const override { return fRegion->get_size(); }
+    uint64_t GetId() const override { return fRegionId; }
 
     ~UnmanagedRegion() override { fManager.RemoveRegion(fRegionId); }
 

fairmq/tools/RateLimit.h

@@ -40,7 +40,7 @@ class RateLimiter
 {
     using clock = std::chrono::steady_clock;
 
-public:
+  public:
     /**
      * Constructs a rate limiter.
      *
@@ -59,15 +59,15 @@ public:
         }
         skip_check_count = std::max(1, int(std::chrono::milliseconds(5) / tw_req));
         count = skip_check_count;
-    //std::cerr << "skip_check_count: " << skip_check_count << '\n';
+        // std::cerr << "skip_check_count: " << skip_check_count << '\n';
     }
 
     /**
      * Call this function at the end of the iteration rate limited loop.
      *
-   * This function might use `std::this_thread::sleep_for` to limit the iteration rate. If no sleeps
-   * are necessary, the function will back off checking for the time to further allow increased
-   * iteration rates (until the requested rate or 1s between rechecks is reached).
+     * This function might use `std::this_thread::sleep_for` to limit the iteration rate. If no
+     * sleeps are necessary, the function will back off checking for the time to further allow
+     * increased iteration rates (until the requested rate or 1s between rechecks is reached).
      */
     void maybe_sleep()
     {
@@ -79,20 +79,23 @@ public:
             } else {
                 tw = (1 * tw + 3 * (now - start_time) / skip_check_count) / 4;
             }
-      //std::ostringstream s; s << "tw = " << std::setw(10) << duration_cast<nanoseconds>(tw).count() << "ns, req = " << duration_cast<nanoseconds>(tw_req).count() << "ns, ";
+            // std::ostringstream s; s << "tw = " << std::setw(10) <<
+            // duration_cast<nanoseconds>(tw).count() << "ns, req = " <<
+            // duration_cast<nanoseconds>(tw_req).count() << "ns, ";
             if (tw > tw_req * 65 / 64) {
                 // the time between maybe_sleep calls is more than 1% too long
                 // fix it by reducing ts towards 0 and if ts = 0 doesn't suffice, increase
                 // skip_check_count
                 if (ts > clock::duration::zero()) {
-          ts = std::max(clock::duration::zero(),
-                        ts - (tw - tw_req) * skip_check_count * 1 / 2);
-          //std::cerr << s.str() << "maybe_sleep: going too slow; sleep less: " << duration_cast<microseconds>(ts).count() << "µs\n";
+                    ts = std::max(clock::duration::zero(), ts - (tw - tw_req) * skip_check_count * 1 / 2);
+                    // std::cerr << s.str() << "maybe_sleep: going too slow; sleep less: " <<
+                    // duration_cast<microseconds>(ts).count() << "µs\n";
                 } else {
                     skip_check_count =
                         std::min(int(seconds(1) / tw_req),   // recheck at least every second
                                  (skip_check_count * 5 + 3) / 4);
-          //std::cerr << s.str() << "maybe_sleep: going too slow; work more: " << skip_check_count << "\n";
+                    // std::cerr << s.str() << "maybe_sleep: going too slow; work more: " <<
+                    // skip_check_count << "\n";
                 }
             } else if (tw < tw_req * 63 / 64) {
                 // the time between maybe_sleep calls is more than 1% too short
@@ -107,10 +110,12 @@ public:
                 if (skip_check_count > min_skip_count) {
                     assert(ts == clock::duration::zero());
                     skip_check_count = std::max(min_skip_count, skip_check_count * 3 / 4);
-          //std::cerr << s.str() << "maybe_sleep: going too fast; work less: " << skip_check_count << "\n";
+                    // std::cerr << s.str() << "maybe_sleep: going too fast; work less: " <<
+                    // skip_check_count << "\n";
                 } else {
                     ts += (tw_req - tw) * (skip_check_count * 7) / 8;
-          //std::cerr << s.str() << "maybe_sleep: going too fast; sleep more: " << duration_cast<microseconds>(ts).count() << "µs\n";
+                    // std::cerr << s.str() << "maybe_sleep: going too fast; sleep more: " <<
+                    // duration_cast<microseconds>(ts).count() << "µs\n";
                 }
             }
 
@@ -122,7 +127,7 @@ public:
         }
     }
 
-private:
+  private:
     clock::duration tw{},   //! deduced duration between maybe_sleep calls
         ts{},               //! sleep duration
         tw_req;             //! requested duration between maybe_sleep calls

fairmq/zeromq/Context.h

@@ -0,0 +1,195 @@
+/********************************************************************************
+ *    Copyright (C) 2020 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_ZMQ_CONTEXT_H_
+#define FAIR_MQ_ZMQ_CONTEXT_H_
+
+#include <fairmq/Tools.h>
+#include <FairMQLogger.h>
+#include <FairMQUnmanagedRegion.h>
+
+#include <zmq.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <queue>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace fair
+{
+namespace mq
+{
+namespace zmq
+{
+
+struct ContextError : std::runtime_error { using std::runtime_error::runtime_error; };
+
+class Context
+{
+  public:
+    Context(int numIoThreads)
+        : fZmqCtx(zmq_ctx_new())
+        , fInterrupted(false)
+        , fRegionCounter(1)
+    {
+        if (!fZmqCtx) {
+            throw ContextError(tools::ToString("failed creating context, reason: ", zmq_strerror(errno)));
+        }
+
+        if (zmq_ctx_set(fZmqCtx, ZMQ_MAX_SOCKETS, 10000) != 0) {
+            LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
+            throw ContextError(tools::ToString("failed configuring context, reason: ", zmq_strerror(errno)));
+        }
+
+        if (zmq_ctx_set(fZmqCtx, ZMQ_IO_THREADS, numIoThreads) != 0) {
+            LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
+            throw ContextError(tools::ToString("failed configuring context, reason: ", zmq_strerror(errno)));
+        }
+
+        fRegionEvents.emplace(0, nullptr, 0, 0, RegionEvent::local_only);
+    }
+
+    Context(const Context&) = delete;
+    Context operator=(const Context&) = delete;
+
+    void SubscribeToRegionEvents(RegionEventCallback callback)
+    {
+        if (fRegionEventThread.joinable()) {
+            LOG(debug) << "Already subscribed. Overwriting previous subscription.";
+            {
+                std::lock_guard<std::mutex> lock(fMtx);
+                fRegionEventsSubscriptionActive = false;
+            }
+            fRegionEventsCV.notify_one();
+            fRegionEventThread.join();
+        }
+        std::lock_guard<std::mutex> lock(fMtx);
+        fRegionEventCallback = callback;
+        fRegionEventsSubscriptionActive = true;
+        fRegionEventThread = std::thread(&Context::RegionEventsSubscription, this);
+    }
+
+    bool SubscribedToRegionEvents() const { return fRegionEventThread.joinable(); }
+
+    void UnsubscribeFromRegionEvents()
+    {
+        if (fRegionEventThread.joinable()) {
+            std::unique_lock<std::mutex> lock(fMtx);
+            fRegionEventsSubscriptionActive = false;
+            lock.unlock();
+            fRegionEventsCV.notify_one();
+            fRegionEventThread.join();
+            lock.lock();
+            fRegionEventCallback = nullptr;
+        }
+    }
+
+    void RegionEventsSubscription()
+    {
+        std::unique_lock<std::mutex> lock(fMtx);
+        while (fRegionEventsSubscriptionActive) {
+
+            while (!fRegionEvents.empty()) {
+                auto i = fRegionEvents.front();
+                fRegionEventCallback(i);
+                fRegionEvents.pop();
+            }
+            fRegionEventsCV.wait(lock, [&]() { return !fRegionEventsSubscriptionActive || !fRegionEvents.empty(); });
+        }
+    }
+
+    std::vector<RegionInfo> GetRegionInfo() const
+    {
+        std::lock_guard<std::mutex> lock(fMtx);
+        return fRegionInfos;
+    }
+
+    uint64_t RegionCount() const
+    {
+        std::lock_guard<std::mutex> lock(fMtx);
+        return fRegionCounter;
+    }
+
+    void AddRegion(uint64_t id, void* ptr, size_t size, int64_t userFlags, RegionEvent event)
+    {
+        {
+            std::lock_guard<std::mutex> lock(fMtx);
+            ++fRegionCounter;
+            fRegionInfos.emplace_back(id, ptr, size, userFlags, event);
+            fRegionEvents.emplace(id, ptr, size, userFlags, event);
+        }
+        fRegionEventsCV.notify_one();
+    }
+
+    void RemoveRegion(uint64_t id)
+    {
+        {
+            std::lock_guard<std::mutex> lock(fMtx);
+            auto it = find_if(fRegionInfos.begin(), fRegionInfos.end(), [id](const RegionInfo& i) {
+                return i.id == id;
+            });
+            if (it != fRegionInfos.end()) {
+                fRegionEvents.push(*it);
+                fRegionEvents.back().event = RegionEvent::destroyed;
+                fRegionInfos.erase(it);
+            } else {
+                LOG(error) << "RemoveRegion: given id (" << id << ") not found.";
+            }
+        }
+        fRegionEventsCV.notify_one();
+    }
+
+    void Interrupt() { fInterrupted.store(true); }
+    void Resume() { fInterrupted.store(false); }
+    void Reset() {}
+    bool Interrupted() { return fInterrupted.load(); }
+
+    void* GetZmqCtx() { return fZmqCtx; }
+
+    ~Context()
+    {
+        UnsubscribeFromRegionEvents();
+
+        if (fZmqCtx) {
+            if (zmq_ctx_term(fZmqCtx) != 0) {
+                if (errno == EINTR) {
+                    LOG(error) << " failed closing context, reason: " << zmq_strerror(errno);
+                } else {
+                    fZmqCtx = nullptr;
+                    return;
+                }
+            }
+        } else {
+            LOG(error) << "context not available for shutdown";
+        }
+    }
+
+  private:
+    void* fZmqCtx;
+    mutable std::mutex fMtx;
+    std::atomic<bool> fInterrupted;
+
+    uint64_t fRegionCounter;
+    std::condition_variable fRegionEventsCV;
+    std::vector<RegionInfo> fRegionInfos;
+    std::queue<RegionInfo> fRegionEvents;
+    std::thread fRegionEventThread;
+    std::function<void(RegionInfo)> fRegionEventCallback;
+    bool fRegionEventsSubscriptionActive;
+};
+
+}   // namespace zmq
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_ZMQ_CONTEXT_H_ */

fairmq/zeromq/FairMQMessageZMQ.cxx

@@ -1,250 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQMessageZMQ.cxx
- *
- * @since 2012-12-05
- * @author D. Klein, A. Rybalchenko, N. Winckler
- */
-
-
-#include "FairMQMessageZMQ.h"
-#include "FairMQLogger.h"
-#include <fairmq/Tools.h>
-#include "FairMQUnmanagedRegionZMQ.h"
-#include <FairMQTransportFactory.h>
-
-#include <cstring>
-
-using namespace std;
-
-fair::mq::Transport FairMQMessageZMQ::fTransportType = fair::mq::Transport::ZMQ;
-
-FairMQMessageZMQ::FairMQMessageZMQ(FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fUsedSizeModified(false)
-    , fUsedSize()
-    , fMsg(fair::mq::tools::make_unique<zmq_msg_t>())
-    , fViewMsg(nullptr)
-{
-    if (zmq_msg_init(fMsg.get()) != 0)
-    {
-        LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
-    }
-}
-
-FairMQMessageZMQ::FairMQMessageZMQ(const size_t size, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fUsedSizeModified(false)
-    , fUsedSize(size)
-    , fMsg(fair::mq::tools::make_unique<zmq_msg_t>())
-    , fViewMsg(nullptr)
-{
-    if (zmq_msg_init_size(fMsg.get(), size) != 0)
-    {
-        LOG(error) << "failed initializing message with size, reason: " << zmq_strerror(errno);
-    }
-}
-
-FairMQMessageZMQ::FairMQMessageZMQ(void* data, const size_t size, fairmq_free_fn* ffn, void* hint, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fUsedSizeModified(false)
-    , fUsedSize()
-    , fMsg(fair::mq::tools::make_unique<zmq_msg_t>())
-    , fViewMsg(nullptr)
-{
-    if (zmq_msg_init_data(fMsg.get(), data, size, ffn, hint) != 0)
-    {
-        LOG(error) << "failed initializing message with data, reason: " << zmq_strerror(errno);
-    }
-}
-
-FairMQMessageZMQ::FairMQMessageZMQ(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint, FairMQTransportFactory* factory)
-    : FairMQMessage{factory}
-    , fUsedSizeModified(false)
-    , fUsedSize()
-    , fMsg(fair::mq::tools::make_unique<zmq_msg_t>())
-    , fViewMsg(nullptr)
-{
-    // FIXME: make this zero-copy:
-    // simply taking over the provided buffer can casue premature delete, since region could be destroyed before the message is sent out.
-    // Needs lifetime extension for the ZMQ region.
-    if (zmq_msg_init_size(fMsg.get(), size) != 0)
-    {
-        LOG(error) << "failed initializing message with size, reason: " << zmq_strerror(errno);
-    }
-
-    memcpy(zmq_msg_data(fMsg.get()), data, size);
-    // call region callback
-    static_cast<FairMQUnmanagedRegionZMQ*>(region.get())->fCallback(data, size, hint);
-
-    // if (zmq_msg_init_data(fMsg.get(), data, size, [](void*, void*){}, nullptr) != 0)
-    // {
-    //     LOG(error) << "failed initializing message with data, reason: " << zmq_strerror(errno);
-    // }
-}
-
-void FairMQMessageZMQ::Rebuild()
-{
-    CloseMessage();
-    fMsg = fair::mq::tools::make_unique<zmq_msg_t>();
-    if (zmq_msg_init(fMsg.get()) != 0)
-    {
-        LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
-    }
-}
-
-void FairMQMessageZMQ::Rebuild(const size_t size)
-{
-    CloseMessage();
-    fMsg = fair::mq::tools::make_unique<zmq_msg_t>();
-    if (zmq_msg_init_size(fMsg.get(), size) != 0)
-    {
-        LOG(error) << "failed initializing message with size, reason: " << zmq_strerror(errno);
-    }
-}
-
-void FairMQMessageZMQ::Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    CloseMessage();
-    fMsg = fair::mq::tools::make_unique<zmq_msg_t>();
-    if (zmq_msg_init_data(fMsg.get(), data, size, ffn, hint) != 0)
-    {
-        LOG(error) << "failed initializing message with data, reason: " << zmq_strerror(errno);
-    }
-}
-
-zmq_msg_t* FairMQMessageZMQ::GetMessage() const
-{
-    if (!fViewMsg)
-    {
-        return fMsg.get();
-    }
-    else
-    {
-        return fViewMsg.get();
-    }
-}
-
-void* FairMQMessageZMQ::GetData() const
-{
-    if (!fViewMsg)
-    {
-        return zmq_msg_data(fMsg.get());
-    }
-    else
-    {
-        return zmq_msg_data(fViewMsg.get());
-    }
-}
-
-size_t FairMQMessageZMQ::GetSize() const
-{
-    if (fUsedSizeModified)
-    {
-        return fUsedSize;
-    }
-    else
-    {
-        return zmq_msg_size(fMsg.get());
-    }
-}
-
-// To emulate shrinking, a new message is created with the new size (ViewMsg), that points to the original buffer with the new size.
-// Once the "view message" is transfered, the original is destroyed.
-// Used size is applied only once in ApplyUsedSize, which is called by the socket before sending.
-// This function just updates the desired size until the actual "resizing" happens.
-bool FairMQMessageZMQ::SetUsedSize(const size_t size)
-{
-    if (size <= zmq_msg_size(fMsg.get()))
-    {
-        fUsedSize = size;
-        fUsedSizeModified = true;
-        return true;
-    }
-    else
-    {
-        LOG(error) << "cannot set used size higher than original.";
-        return false;
-    }
-}
-
-void FairMQMessageZMQ::ApplyUsedSize()
-{
-    // Apply only once (before actual send).
-    // The check is needed because a send could fail and can be reattempted by the user, in this case we do not want to modify buffer again.
-    if (fUsedSizeModified && !fViewMsg)
-    {
-        fViewMsg = fair::mq::tools::make_unique<zmq_msg_t>();
-        void* ptr = zmq_msg_data(fMsg.get());
-        if (zmq_msg_init_data(fViewMsg.get(),
-                                ptr,
-                                fUsedSize,
-                                [](void* /* data */, void* obj)
-                                {
-                                    zmq_msg_close(static_cast<zmq_msg_t*>(obj));
-                                    delete static_cast<zmq_msg_t*>(obj);
-                                },
-                                fMsg.release()) != 0)
-        {
-            LOG(error) << "failed initializing view message, reason: " << zmq_strerror(errno);
-        }
-    }
-}
-
-fair::mq::Transport FairMQMessageZMQ::GetType() const
-{
-    return fTransportType;
-}
-
-void FairMQMessageZMQ::Copy(const FairMQMessage& msg)
-{
-    const FairMQMessageZMQ& zMsg = static_cast<const FairMQMessageZMQ&>(msg);
-    // Shares the message buffer between msg and this fMsg.
-    if (zmq_msg_copy(fMsg.get(), zMsg.GetMessage()) != 0)
-    {
-        LOG(error) << "failed copying message, reason: " << zmq_strerror(errno);
-        return;
-    }
-
-    // if the target message has been resized, apply same to this message also
-    if (zMsg.fUsedSizeModified)
-    {
-        fUsedSizeModified = true;
-        fUsedSize = zMsg.fUsedSize;
-    }
-}
-
-void FairMQMessageZMQ::CloseMessage()
-{
-    if (!fViewMsg)
-    {
-        if (zmq_msg_close(fMsg.get()) != 0)
-        {
-            LOG(error) << "failed closing message, reason: " << zmq_strerror(errno);
-        }
-        // reset the message object to allow reuse in Rebuild
-        fMsg.reset(nullptr);
-    }
-    else
-    {
-        if (zmq_msg_close(fViewMsg.get()) != 0)
-        {
-            LOG(error) << "failed closing message, reason: " << zmq_strerror(errno);
-        }
-        // reset the message object to allow reuse in Rebuild
-        fViewMsg.reset(nullptr);
-    }
-    fUsedSizeModified = false;
-    fUsedSize = 0;
-}
-
-FairMQMessageZMQ::~FairMQMessageZMQ()
-{
-    CloseMessage();
-}

fairmq/zeromq/FairMQMessageZMQ.h

@@ -1,67 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQMessageZMQ.h
- *
- * @since 2014-01-17
- * @author A. Rybalchenko
- */
-
-#ifndef FAIRMQMESSAGEZMQ_H_
-#define FAIRMQMESSAGEZMQ_H_
-
-#include <cstddef>
-#include <string>
-#include <memory>
-
-#include <zmq.h>
-
-#include "FairMQMessage.h"
-#include "FairMQUnmanagedRegion.h"
-class FairMQTransportFactory;
-
-class FairMQSocketZMQ;
-
-class FairMQMessageZMQ final : public FairMQMessage
-{
-    friend class FairMQSocketZMQ;
-
-  public:
-    FairMQMessageZMQ(FairMQTransportFactory* = nullptr);
-    FairMQMessageZMQ(const size_t size, FairMQTransportFactory* = nullptr);
-    FairMQMessageZMQ(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* = nullptr);
-    FairMQMessageZMQ(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* = nullptr);
-
-    void Rebuild() override;
-    void Rebuild(const size_t size) override;
-    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-
-    void* GetData() const override;
-    size_t GetSize() const override;
-
-    bool SetUsedSize(const size_t size) override;
-    void ApplyUsedSize();
-
-    fair::mq::Transport GetType() const override;
-
-    void Copy(const FairMQMessage& msg) override;
-
-    ~FairMQMessageZMQ() override;
-
-  private:
-    bool fUsedSizeModified;
-    size_t fUsedSize;
-    std::unique_ptr<zmq_msg_t> fMsg;
-    std::unique_ptr<zmq_msg_t> fViewMsg; // view on a subset of fMsg (treating it as user buffer)
-    static fair::mq::Transport fTransportType;
-
-    zmq_msg_t* GetMessage() const;
-    void CloseMessage();
-};
-
-#endif /* FAIRMQMESSAGEZMQ_H_ */

fairmq/zeromq/FairMQPollerZMQ.cxx

@@ -1,211 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQPollerZMQ.cxx
- *
- * @since 2014-01-23
- * @author A. Rybalchenko
- */
-
-#include <zmq.h>
-
-#include "FairMQPollerZMQ.h"
-#include "FairMQSocketZMQ.h"
-#include "FairMQLogger.h"
-
-using namespace std;
-
-FairMQPollerZMQ::FairMQPollerZMQ(const vector<FairMQChannel>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new zmq_pollitem_t[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].socket = static_cast<const FairMQSocketZMQ*>(&(channels.at(i).GetSocket()))->GetSocket();
-        fItems[i].fd = 0;
-        fItems[i].revents = 0;
-
-        int type = 0;
-        size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const FairMQSocketZMQ*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-
-FairMQPollerZMQ::FairMQPollerZMQ(const std::vector<FairMQChannel*>& channels)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    fNumItems = channels.size();
-    fItems = new zmq_pollitem_t[fNumItems];
-
-    for (int i = 0; i < fNumItems; ++i)
-    {
-        fItems[i].socket = static_cast<const FairMQSocketZMQ*>(&(channels.at(i)->GetSocket()))->GetSocket();
-        fItems[i].fd = 0;
-        fItems[i].revents = 0;
-
-        int type = 0;
-        size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const FairMQSocketZMQ*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-        SetItemEvents(fItems[i], type);
-    }
-}
-
-FairMQPollerZMQ::FairMQPollerZMQ(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList)
-    : fItems()
-    , fNumItems(0)
-    , fOffsetMap()
-{
-    try
-    {
-        int offset = 0;
-        // calculate offsets and the total size of the poll item set
-        for (string channel : channelList)
-        {
-            fOffsetMap[channel] = offset;
-            offset += channelsMap.at(channel).size();
-            fNumItems += channelsMap.at(channel).size();
-        }
-
-        fItems = new zmq_pollitem_t[fNumItems];
-
-        int index = 0;
-        for (string channel : channelList)
-        {
-            for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i)
-            {
-                index = fOffsetMap[channel] + i;
-
-                fItems[index].socket = static_cast<const FairMQSocketZMQ*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
-                fItems[index].fd = 0;
-                fItems[index].revents = 0;
-
-                int type = 0;
-                size_t size = sizeof(type);
-                zmq_getsockopt(static_cast<const FairMQSocketZMQ*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
-
-                SetItemEvents(fItems[index], type);
-            }
-        }
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "at least one of the provided channel keys for poller initialization is invalid";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        throw std::out_of_range("invalid channel during poller initialization");
-    }
-}
-
-void FairMQPollerZMQ::SetItemEvents(zmq_pollitem_t& item, const int type)
-{
-    if (type == ZMQ_REQ || type == ZMQ_REP || type == ZMQ_PAIR || type == ZMQ_DEALER || type == ZMQ_ROUTER)
-    {
-        item.events = ZMQ_POLLIN|ZMQ_POLLOUT;
-    }
-    else if (type == ZMQ_PUSH || type == ZMQ_PUB || type == ZMQ_XPUB)
-    {
-        item.events = ZMQ_POLLOUT;
-    }
-    else if (type == ZMQ_PULL || type == ZMQ_SUB || type == ZMQ_XSUB)
-    {
-        item.events = ZMQ_POLLIN;
-    }
-    else
-    {
-        LOG(error) << "invalid poller configuration, exiting.";
-        exit(EXIT_FAILURE);
-    }
-}
-
-void FairMQPollerZMQ::Poll(const int timeout)
-{
-    if (zmq_poll(fItems, fNumItems, timeout) < 0)
-    {
-        if (errno == ETERM)
-        {
-            LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
-        }
-        else
-        {
-            LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
-            throw std::runtime_error("polling failed");
-        }
-    }
-}
-
-bool FairMQPollerZMQ::CheckInput(const int index)
-{
-    if (fItems[index].revents & ZMQ_POLLIN)
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool FairMQPollerZMQ::CheckOutput(const int index)
-{
-    if (fItems[index].revents & ZMQ_POLLOUT)
-    {
-        return true;
-    }
-
-    return false;
-}
-
-bool FairMQPollerZMQ::CheckInput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN)
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-bool FairMQPollerZMQ::CheckOutput(const string& channelKey, const int index)
-{
-    try
-    {
-        if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT)
-        {
-            return true;
-        }
-
-        return false;
-    }
-    catch (const std::out_of_range& oor)
-    {
-        LOG(error) << "invalid channel key: \"" << channelKey << "\"";
-        LOG(error) << "out of range error: " << oor.what() << '\n';
-        exit(EXIT_FAILURE);
-    }
-}
-
-FairMQPollerZMQ::~FairMQPollerZMQ()
-{
-    delete[] fItems;
-}

fairmq/zeromq/FairMQPollerZMQ.h

@@ -1,59 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQPollerZMQ.h
- *
- * @since 2014-01-23
- * @author A. Rybalchenko
- */
-
-#ifndef FAIRMQPOLLERZMQ_H_
-#define FAIRMQPOLLERZMQ_H_
-
-#include <vector>
-#include <unordered_map>
-
-#include <zmq.h>
-
-#include "FairMQPoller.h"
-#include "FairMQChannel.h"
-#include "FairMQTransportFactoryZMQ.h"
-
-class FairMQChannel;
-
-class FairMQPollerZMQ final : public FairMQPoller
-{
-    friend class FairMQChannel;
-    friend class FairMQTransportFactoryZMQ;
-
-  public:
-    FairMQPollerZMQ(const std::vector<FairMQChannel>& channels);
-    FairMQPollerZMQ(const std::vector<FairMQChannel*>& channels);
-    FairMQPollerZMQ(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList);
-
-    FairMQPollerZMQ(const FairMQPollerZMQ&) = delete;
-    FairMQPollerZMQ operator=(const FairMQPollerZMQ&) = delete;
-
-    void SetItemEvents(zmq_pollitem_t& item, const int type);
-
-    void Poll(const int timeout) override;
-    bool CheckInput(const int index) override;
-    bool CheckOutput(const int index) override;
-    bool CheckInput(const std::string& channelKey, const int index) override;
-    bool CheckOutput(const std::string& channelKey, const int index) override;
-
-    ~FairMQPollerZMQ() override;
-
-  private:
-    zmq_pollitem_t* fItems;
-    int fNumItems;
-
-    std::unordered_map<std::string, int> fOffsetMap;
-};
-
-#endif /* FAIRMQPOLLERZMQ_H_ */

fairmq/zeromq/FairMQSocketZMQ.cxx

@@ -1,498 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQSocketZMQ.h"
-#include "FairMQMessageZMQ.h"
-#include "FairMQLogger.h"
-#include <fairmq/Tools.h>
-
-#include <zmq.h>
-
-#include <cassert>
-
-using namespace std;
-using namespace fair::mq;
-
-atomic<bool> FairMQSocketZMQ::fInterrupted(false);
-
-FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const string& id /*= ""*/, void* context, FairMQTransportFactory* fac)
-    : FairMQSocket{fac}
-    , fSocket(nullptr)
-    , fId(id + "." + name + "." + type)
-    , fBytesTx(0)
-    , fBytesRx(0)
-    , fMessagesTx(0)
-    , fMessagesRx(0)
-    , fSndTimeout(100)
-    , fRcvTimeout(100)
-{
-    assert(context);
-    fSocket = zmq_socket(context, GetConstant(type));
-
-    if (fSocket == nullptr)
-    {
-        LOG(error) << "Failed creating socket " << fId << ", reason: " << zmq_strerror(errno);
-        exit(EXIT_FAILURE);
-    }
-
-    if (zmq_setsockopt(fSocket, ZMQ_IDENTITY, fId.c_str(), fId.length()) != 0)
-    {
-        LOG(error) << "Failed setting ZMQ_IDENTITY socket option, reason: " << zmq_strerror(errno);
-    }
-
-    // Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
-    // Default value for ZeroMQ is -1, which is to wait forever.
-    int linger = 1000;
-    if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0)
-    {
-        LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
-    }
-
-    if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0)
-    {
-        LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
-    }
-
-    if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0)
-    {
-        LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
-    }
-
-    if (type == "sub")
-    {
-        if (zmq_setsockopt(fSocket, ZMQ_SUBSCRIBE, nullptr, 0) != 0)
-        {
-            LOG(error) << "Failed setting ZMQ_SUBSCRIBE socket option, reason: " << zmq_strerror(errno);
-        }
-    }
-
-    LOG(debug) << "Created socket " << GetId();
-}
-
-bool FairMQSocketZMQ::Bind(const string& address)
-{
-    // LOG(info) << "bind socket " << fId << " on " << address;
-
-    if (zmq_bind(fSocket, address.c_str()) != 0)
-    {
-        if (errno == EADDRINUSE) {
-            // do not print error in this case, this is handled by FairMQDevice in case no connection could be established after trying a number of random ports from a range.
-            return false;
-        }
-        LOG(error) << "Failed binding socket " << fId << ", reason: " << zmq_strerror(errno);
-        return false;
-    }
-
-    return true;
-}
-
-bool FairMQSocketZMQ::Connect(const string& address)
-{
-    // LOG(info) << "connect socket " << fId << " on " << address;
-
-    if (zmq_connect(fSocket, address.c_str()) != 0)
-    {
-        LOG(error) << "Failed connecting socket " << fId << ", reason: " << zmq_strerror(errno);
-        return false;
-    }
-
-    return true;
-}
-
-int FairMQSocketZMQ::Send(FairMQMessagePtr& msg, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0) {
-        flags = ZMQ_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    static_cast<FairMQMessageZMQ*>(msg.get())->ApplyUsedSize();
-
-    while (true) {
-        int nbytes = zmq_msg_send(static_cast<FairMQMessageZMQ*>(msg.get())->GetMessage(), fSocket, flags);
-        if (nbytes >= 0) {
-            fBytesTx += nbytes;
-            ++fMessagesTx;
-
-            return nbytes;
-        } else if (zmq_errno() == EAGAIN) {
-            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0)) {
-                if (timeout > 0) {
-                    elapsed += fSndTimeout;
-                    if (elapsed >= timeout) {
-                        return -2;
-                    }
-                }
-                continue;
-            } else {
-                return -2;
-            }
-        } else if (zmq_errno() == ETERM) {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        } else if (zmq_errno() == EINTR) {
-            LOG(debug) << "Send interrupted by system call";
-            return nbytes;
-        } else {
-            LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-int FairMQSocketZMQ::Receive(FairMQMessagePtr& msg, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0) {
-        flags = ZMQ_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    while (true) {
-        int nbytes = zmq_msg_recv(static_cast<FairMQMessageZMQ*>(msg.get())->GetMessage(), fSocket, flags);
-        if (nbytes >= 0) {
-            fBytesRx += nbytes;
-            ++fMessagesRx;
-            return nbytes;
-        } else if (zmq_errno() == EAGAIN) {
-            if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0)) {
-                if (timeout > 0) {
-                    elapsed += fRcvTimeout;
-                    if (elapsed >= timeout) {
-                        return -2;
-                    }
-                }
-                continue;
-            } else {
-                return -2;
-            }
-        } else if (zmq_errno() == ETERM) {
-            LOG(info) << "terminating socket " << fId;
-            return -1;
-        } else if (zmq_errno() == EINTR) {
-            LOG(debug) << "Receive interrupted by system call";
-            return nbytes;
-        } else {
-            LOG(error) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
-            return nbytes;
-        }
-    }
-}
-
-int64_t FairMQSocketZMQ::Send(vector<FairMQMessagePtr>& msgVec, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0) {
-        flags = ZMQ_DONTWAIT;
-    }
-
-    const unsigned int vecSize = msgVec.size();
-
-    // Sending vector typicaly handles more then one part
-    if (vecSize > 1) {
-        int elapsed = 0;
-
-        while (true) {
-            int64_t totalSize = 0;
-            bool repeat = false;
-
-            for (unsigned int i = 0; i < vecSize; ++i) {
-                static_cast<FairMQMessageZMQ*>(msgVec[i].get())->ApplyUsedSize();
-
-                int nbytes = zmq_msg_send(static_cast<FairMQMessageZMQ*>(msgVec[i].get())->GetMessage(),
-                                      fSocket,
-                                      (i < vecSize - 1) ? ZMQ_SNDMORE|flags : flags);
-                if (nbytes >= 0) {
-                    totalSize += nbytes;
-                } else {
-                    // according to ZMQ docs, this can only occur for the first part
-                    if (zmq_errno() == EAGAIN) {
-                        if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0)) {
-                            if (timeout > 0) {
-                                elapsed += fSndTimeout;
-                                if (elapsed >= timeout) {
-                                    return -2;
-                                }
-                            }
-                            repeat = true;
-                            break;
-                        } else {
-                            return -2;
-                        }
-                    }
-                    if (zmq_errno() == ETERM) {
-                        LOG(info) << "terminating socket " << fId;
-                        return -1;
-                    } else if (zmq_errno() == EINTR) {
-                        LOG(debug) << "Receive interrupted by system call";
-                        return nbytes;
-                    } else {
-                        LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-                        return nbytes;
-                    }
-                }
-            }
-
-            if (repeat) {
-                continue;
-            }
-
-            // store statistics on how many messages have been sent (handle all parts as a single message)
-            ++fMessagesTx;
-            fBytesTx += totalSize;
-            return totalSize;
-        }
-    } // If there's only one part, send it as a regular message
-    else if (vecSize == 1) {
-        return Send(msgVec.back(), timeout);
-    } else { // if the vector is empty, something might be wrong
-        LOG(warn) << "Will not send empty vector";
-        return -1;
-    }
-}
-
-int64_t FairMQSocketZMQ::Receive(vector<FairMQMessagePtr>& msgVec, const int timeout)
-{
-    int flags = 0;
-    if (timeout == 0) {
-        flags = ZMQ_DONTWAIT;
-    }
-    int elapsed = 0;
-
-    while (true) {
-        int64_t totalSize = 0;
-        int64_t more = 0;
-        bool repeat = false;
-
-        do {
-            unique_ptr<FairMQMessage> part(new FairMQMessageZMQ(GetTransport()));
-
-            int nbytes = zmq_msg_recv(static_cast<FairMQMessageZMQ*>(part.get())->GetMessage(), fSocket, flags);
-            if (nbytes >= 0) {
-                msgVec.push_back(move(part));
-                totalSize += nbytes;
-            } else if (zmq_errno() == EAGAIN) {
-                if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0)) {
-                    if (timeout > 0) {
-                        elapsed += fRcvTimeout;
-                        if (elapsed >= timeout) {
-                            return -2;
-                        }
-                    }
-                    repeat = true;
-                    break;
-                } else {
-                    return -2;
-                }
-            } else if (zmq_errno() == EINTR) {
-                LOG(debug) << "Receive interrupted by system call";
-                return nbytes;
-            } else {
-                return nbytes;
-            }
-
-            size_t moreSize = sizeof(more);
-            zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &moreSize);
-        } while (more);
-
-        if (repeat) {
-            continue;
-        }
-
-        // store statistics on how many messages have been received (handle all parts as a single message)
-        ++fMessagesRx;
-        fBytesRx += totalSize;
-        return totalSize;
-    }
-}
-
-void FairMQSocketZMQ::Close()
-{
-    // LOG(debug) << "Closing socket " << fId;
-
-    if (fSocket == nullptr)
-    {
-        return;
-    }
-
-    if (zmq_close(fSocket) != 0)
-    {
-        LOG(error) << "Failed closing socket " << fId << ", reason: " << zmq_strerror(errno);
-    }
-
-    fSocket = nullptr;
-}
-
-void FairMQSocketZMQ::Interrupt()
-{
-    fInterrupted = true;
-}
-
-void FairMQSocketZMQ::Resume()
-{
-    fInterrupted = false;
-}
-
-void* FairMQSocketZMQ::GetSocket() const
-{
-    return fSocket;
-}
-
-void FairMQSocketZMQ::SetOption(const string& option, const void* value, size_t valueSize)
-{
-    if (zmq_setsockopt(fSocket, GetConstant(option), value, valueSize) < 0)
-    {
-        LOG(error) << "Failed setting socket option, reason: " << zmq_strerror(errno);
-    }
-}
-
-void FairMQSocketZMQ::GetOption(const string& option, void* value, size_t* valueSize)
-{
-    if (zmq_getsockopt(fSocket, GetConstant(option), value, valueSize) < 0)
-    {
-        LOG(error) << "Failed getting socket option, reason: " << zmq_strerror(errno);
-    }
-}
-
-void FairMQSocketZMQ::SetLinger(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_LINGER, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int FairMQSocketZMQ::GetLinger() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_LINGER, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-void FairMQSocketZMQ::SetSndBufSize(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_SNDHWM, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int FairMQSocketZMQ::GetSndBufSize() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_SNDHWM, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-void FairMQSocketZMQ::SetRcvBufSize(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_RCVHWM, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int FairMQSocketZMQ::GetRcvBufSize() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_RCVHWM, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-void FairMQSocketZMQ::SetSndKernelSize(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_SNDBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int FairMQSocketZMQ::GetSndKernelSize() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_SNDBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-void FairMQSocketZMQ::SetRcvKernelSize(const int value)
-{
-    if (zmq_setsockopt(fSocket, ZMQ_RCVBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
-    }
-}
-
-int FairMQSocketZMQ::GetRcvKernelSize() const
-{
-    int value = 0;
-    size_t valueSize = sizeof(value);
-    if (zmq_getsockopt(fSocket, ZMQ_RCVBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
-    }
-    return value;
-}
-
-unsigned long FairMQSocketZMQ::GetBytesTx() const
-{
-    return fBytesTx;
-}
-
-unsigned long FairMQSocketZMQ::GetBytesRx() const
-{
-    return fBytesRx;
-}
-
-unsigned long FairMQSocketZMQ::GetMessagesTx() const
-{
-    return fMessagesTx;
-}
-
-unsigned long FairMQSocketZMQ::GetMessagesRx() const
-{
-    return fMessagesRx;
-}
-
-int FairMQSocketZMQ::GetConstant(const string& constant)
-{
-    if (constant == "") return 0;
-    if (constant == "sub") return ZMQ_SUB;
-    if (constant == "pub") return ZMQ_PUB;
-    if (constant == "xsub") return ZMQ_XSUB;
-    if (constant == "xpub") return ZMQ_XPUB;
-    if (constant == "push") return ZMQ_PUSH;
-    if (constant == "pull") return ZMQ_PULL;
-    if (constant == "req") return ZMQ_REQ;
-    if (constant == "rep") return ZMQ_REP;
-    if (constant == "dealer") return ZMQ_DEALER;
-    if (constant == "router") return ZMQ_ROUTER;
-    if (constant == "pair") return ZMQ_PAIR;
-
-    if (constant == "snd-hwm") return ZMQ_SNDHWM;
-    if (constant == "rcv-hwm") return ZMQ_RCVHWM;
-    if (constant == "snd-size") return ZMQ_SNDBUF;
-    if (constant == "rcv-size") return ZMQ_RCVBUF;
-    if (constant == "snd-more") return ZMQ_SNDMORE;
-    if (constant == "rcv-more") return ZMQ_RCVMORE;
-
-    if (constant == "linger") return ZMQ_LINGER;
-
-    return -1;
-}
-
-FairMQSocketZMQ::~FairMQSocketZMQ()
-{
-    Close();
-}

fairmq/zeromq/FairMQSocketZMQ.h

@@ -1,81 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQSOCKETZMQ_H_
-#define FAIRMQSOCKETZMQ_H_
-
-#include <atomic>
-
-#include <memory> // unique_ptr
-
-#include "FairMQSocket.h"
-#include "FairMQMessage.h"
-class FairMQTransportFactory;
-
-class FairMQSocketZMQ final : public FairMQSocket
-{
-  public:
-    FairMQSocketZMQ(const std::string& type, const std::string& name, const std::string& id = "", void* context = nullptr, FairMQTransportFactory* factory = nullptr);
-    FairMQSocketZMQ(const FairMQSocketZMQ&) = delete;
-    FairMQSocketZMQ operator=(const FairMQSocketZMQ&) = delete;
-
-    std::string GetId() const override { return fId; }
-
-    bool Bind(const std::string& address) override;
-    bool Connect(const std::string& address) override;
-
-    int Send(FairMQMessagePtr& msg, const int timeout = -1) override;
-    int Receive(FairMQMessagePtr& msg, const int timeout = -1) override;
-    int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
-    int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
-
-    void* GetSocket() const;
-
-    void Close() override;
-
-    static void Interrupt();
-    static void Resume();
-
-    void SetOption(const std::string& option, const void* value, size_t valueSize) override;
-    void GetOption(const std::string& option, void* value, size_t* valueSize) override;
-
-    void SetLinger(const int value) override;
-    int GetLinger() const override;
-    void SetSndBufSize(const int value) override;
-    int GetSndBufSize() const override;
-    void SetRcvBufSize(const int value) override;
-    int GetRcvBufSize() const override;
-    void SetSndKernelSize(const int value) override;
-    int GetSndKernelSize() const override;
-    void SetRcvKernelSize(const int value) override;
-    int GetRcvKernelSize() const override;
-
-    unsigned long GetBytesTx() const override;
-    unsigned long GetBytesRx() const override;
-    unsigned long GetMessagesTx() const override;
-    unsigned long GetMessagesRx() const override;
-
-    static int GetConstant(const std::string& constant);
-
-    ~FairMQSocketZMQ() override;
-
-  private:
-    void* fSocket;
-    std::string fId;
-    std::atomic<unsigned long> fBytesTx;
-    std::atomic<unsigned long> fBytesRx;
-    std::atomic<unsigned long> fMessagesTx;
-    std::atomic<unsigned long> fMessagesRx;
-
-    static std::atomic<bool> fInterrupted;
-
-    int fSndTimeout;
-    int fRcvTimeout;
-};
-
-#endif /* FAIRMQSOCKETZMQ_H_ */

fairmq/zeromq/FairMQTransportFactoryZMQ.cxx

@@ -1,130 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2014-2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQTransportFactoryZMQ.h"
-#include <zmq.h>
-
-using namespace std;
-
-fair::mq::Transport FairMQTransportFactoryZMQ::fTransportType = fair::mq::Transport::ZMQ;
-
-FairMQTransportFactoryZMQ::FairMQTransportFactoryZMQ(const string& id, const fair::mq::ProgOptions* config)
-    : FairMQTransportFactory(id)
-    , fContext(zmq_ctx_new())
-{
-    int major, minor, patch;
-    zmq_version(&major, &minor, &patch);
-    LOG(debug) << "Transport: Using ZeroMQ library, version: " << major << "." << minor << "." << patch;
-
-    if (!fContext)
-    {
-        LOG(error) << "failed creating context, reason: " << zmq_strerror(errno);
-        exit(EXIT_FAILURE);
-    }
-
-    if (zmq_ctx_set(fContext, ZMQ_MAX_SOCKETS, 10000) != 0)
-    {
-        LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
-    }
-
-    int numIoThreads = 1;
-    if (config)
-    {
-        numIoThreads = config->GetProperty<int>("io-threads", numIoThreads);
-    }
-    else
-    {
-        LOG(debug) << "fair::mq::ProgOptions not available! Using defaults.";
-    }
-
-    if (zmq_ctx_set(fContext, ZMQ_IO_THREADS, numIoThreads) != 0)
-    {
-        LOG(error) << "failed configuring context, reason: " << zmq_strerror(errno);
-    }
-
-}
-
-FairMQMessagePtr FairMQTransportFactoryZMQ::CreateMessage()
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageZMQ(this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryZMQ::CreateMessage(const size_t size)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageZMQ(size, this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryZMQ::CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageZMQ(data, size, ffn, hint, this));
-}
-
-FairMQMessagePtr FairMQTransportFactoryZMQ::CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint)
-{
-    return unique_ptr<FairMQMessage>(new FairMQMessageZMQ(region, data, size, hint, this));
-}
-
-FairMQSocketPtr FairMQTransportFactoryZMQ::CreateSocket(const string& type, const string& name)
-{
-    assert(fContext);
-    return unique_ptr<FairMQSocket>(new FairMQSocketZMQ(type, name, GetId(), fContext, this));
-}
-
-FairMQPollerPtr FairMQTransportFactoryZMQ::CreatePoller(const vector<FairMQChannel>& channels) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerZMQ(channels));
-}
-
-FairMQPollerPtr FairMQTransportFactoryZMQ::CreatePoller(const std::vector<FairMQChannel*>& channels) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerZMQ(channels));
-}
-
-FairMQPollerPtr FairMQTransportFactoryZMQ::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList) const
-{
-    return unique_ptr<FairMQPoller>(new FairMQPollerZMQ(channelsMap, channelList));
-}
-
-FairMQUnmanagedRegionPtr FairMQTransportFactoryZMQ::CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
-{
-    return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionZMQ(size, callback, path, flags));
-}
-
-FairMQUnmanagedRegionPtr FairMQTransportFactoryZMQ::CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */) const
-{
-    return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionZMQ(size, userFlags, callback, path, flags));
-}
-
-fair::mq::Transport FairMQTransportFactoryZMQ::GetType() const
-{
-    return fTransportType;
-}
-
-FairMQTransportFactoryZMQ::~FairMQTransportFactoryZMQ()
-{
-    LOG(debug) << "Destroying ZeroMQ transport...";
-    if (fContext)
-    {
-        if (zmq_ctx_term(fContext) != 0)
-        {
-            if (errno == EINTR)
-            {
-                LOG(error) << " failed closing context, reason: " << zmq_strerror(errno);
-            }
-            else
-            {
-                fContext = nullptr;
-                return;
-            }
-        }
-    }
-    else
-    {
-        LOG(error) << "context not available for shutdown";
-    }
-}

fairmq/zeromq/FairMQTransportFactoryZMQ.h

@@ -1,66 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQTransportFactoryZMQ.h
- *
- * @since 2014-01-20
- * @author: A. Rybalchenko
- */
-
-#ifndef FAIRMQTRANSPORTFACTORYZMQ_H_
-#define FAIRMQTRANSPORTFACTORYZMQ_H_
-
-#include <vector>
-#include <string>
-
-#include "FairMQTransportFactory.h"
-#include "FairMQMessageZMQ.h"
-#include "FairMQSocketZMQ.h"
-#include "FairMQPollerZMQ.h"
-#include "FairMQUnmanagedRegionZMQ.h"
-#include <fairmq/ProgOptions.h>
-
-class FairMQTransportFactoryZMQ final : public FairMQTransportFactory
-{
-  public:
-    FairMQTransportFactoryZMQ(const std::string& id = "", const fair::mq::ProgOptions* config = nullptr);
-    FairMQTransportFactoryZMQ(const FairMQTransportFactoryZMQ&) = delete;
-    FairMQTransportFactoryZMQ operator=(const FairMQTransportFactoryZMQ&) = delete;
-
-    ~FairMQTransportFactoryZMQ() override;
-
-    FairMQMessagePtr CreateMessage() override;
-    FairMQMessagePtr CreateMessage(const size_t size) override;
-    FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
-    FairMQMessagePtr CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0) override;
-
-    FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name) override;
-
-    FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override;
-    FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const override;
-    FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
-
-    FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0) const override;
-    FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) const override;
-
-    void SubscribeToRegionEvents(FairMQRegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for ZeroMQ"; }
-    void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for ZeroMQ"; }
-    std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for ZeroMQ, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
-
-    fair::mq::Transport GetType() const override;
-
-    void Interrupt() override { FairMQSocketZMQ::Interrupt(); }
-    void Resume() override { FairMQSocketZMQ::Resume(); }
-    void Reset() override {}
-
-  private:
-    static fair::mq::Transport fTransportType;
-    void* fContext;
-};
-
-#endif /* FAIRMQTRANSPORTFACTORYZMQ_H_ */

fairmq/zeromq/FairMQUnmanagedRegionZMQ.cxx

@@ -1,42 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "FairMQUnmanagedRegionZMQ.h"
-#include "FairMQLogger.h"
-
-using namespace std;
-
-FairMQUnmanagedRegionZMQ::FairMQUnmanagedRegionZMQ(const size_t size, FairMQRegionCallback callback, const std::string& /* path = "" */, int /* flags = 0 */)
-    : fBuffer(malloc(size))
-    , fSize(size)
-    , fCallback(callback)
-{
-}
-
-FairMQUnmanagedRegionZMQ::FairMQUnmanagedRegionZMQ(const size_t size, int64_t /* userFlags */, FairMQRegionCallback callback, const std::string& /* path = "" */, int /* flags = 0 */)
-    : fBuffer(malloc(size))
-    , fSize(size)
-    , fCallback(callback)
-{
-}
-
-void* FairMQUnmanagedRegionZMQ::GetData() const
-{
-    return fBuffer;
-}
-
-size_t FairMQUnmanagedRegionZMQ::GetSize() const
-{
-    return fSize;
-}
-
-FairMQUnmanagedRegionZMQ::~FairMQUnmanagedRegionZMQ()
-{
-    LOG(debug) << "destroying region";
-    free(fBuffer);
-}

fairmq/zeromq/FairMQUnmanagedRegionZMQ.h

@@ -1,39 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIRMQUNMANAGEDREGIONZMQ_H_
-#define FAIRMQUNMANAGEDREGIONZMQ_H_
-
-#include "FairMQUnmanagedRegion.h"
-
-#include <cstddef> // size_t
-#include <string>
-
-class FairMQUnmanagedRegionZMQ final : public FairMQUnmanagedRegion
-{
-    friend class FairMQSocketZMQ;
-    friend class FairMQMessageZMQ;
-
-  public:
-    FairMQUnmanagedRegionZMQ(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
-    FairMQUnmanagedRegionZMQ(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path = "", int flags = 0);
-    FairMQUnmanagedRegionZMQ(const FairMQUnmanagedRegionZMQ&) = delete;
-    FairMQUnmanagedRegionZMQ operator=(const FairMQUnmanagedRegionZMQ&) = delete;
-
-    virtual void* GetData() const override;
-    virtual size_t GetSize() const override;
-
-    virtual ~FairMQUnmanagedRegionZMQ();
-
-  private:
-    void* fBuffer;
-    size_t fSize;
-    FairMQRegionCallback fCallback;
-};
-
-#endif /* FAIRMQUNMANAGEDREGIONZMQ_H_ */

fairmq/zeromq/Message.h

@@ -0,0 +1,266 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_ZMQ_MESSAGE_H
+#define FAIR_MQ_ZMQ_MESSAGE_H
+
+#include <fairmq/Tools.h>
+#include <fairmq/zeromq/UnmanagedRegion.h>
+#include <FairMQLogger.h>
+#include <FairMQMessage.h>
+#include <FairMQUnmanagedRegion.h>
+
+#include <zmq.h>
+
+#include <cstddef>
+#include <cstring>
+#include <memory>
+#include <string>
+
+namespace fair
+{
+namespace mq
+{
+namespace zmq
+{
+
+class Socket;
+
+class Message final : public fair::mq::Message
+{
+    friend class Socket;
+
+  public:
+    Message(FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fUsedSizeModified(false)
+        , fUsedSize()
+        , fMsg(tools::make_unique<zmq_msg_t>())
+        , fViewMsg(nullptr)
+    {
+        if (zmq_msg_init(fMsg.get()) != 0) {
+            LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
+        }
+    }
+    Message(Alignment /* alignment */, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fUsedSizeModified(false)
+        , fUsedSize()
+        , fMsg(tools::make_unique<zmq_msg_t>())
+        , fViewMsg(nullptr)
+    {
+        if (zmq_msg_init(fMsg.get()) != 0) {
+            LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    Message(const size_t size, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fUsedSizeModified(false)
+        , fUsedSize(size)
+        , fMsg(tools::make_unique<zmq_msg_t>())
+        , fViewMsg(nullptr)
+    {
+        if (zmq_msg_init_size(fMsg.get(), size) != 0) {
+            LOG(error) << "failed initializing message with size, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    Message(const size_t size, Alignment /* alignment */, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fUsedSizeModified(false)
+        , fUsedSize(size)
+        , fMsg(tools::make_unique<zmq_msg_t>())
+        , fViewMsg(nullptr)
+    {
+        if (zmq_msg_init_size(fMsg.get(), size) != 0) {
+            LOG(error) << "failed initializing message with size, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    Message(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fUsedSizeModified(false)
+        , fUsedSize()
+        , fMsg(tools::make_unique<zmq_msg_t>())
+        , fViewMsg(nullptr)
+    {
+        if (zmq_msg_init_data(fMsg.get(), data, size, ffn, hint) != 0) {
+            LOG(error) << "failed initializing message with data, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    Message(UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Message(factory)
+        , fUsedSizeModified(false)
+        , fUsedSize()
+        , fMsg(tools::make_unique<zmq_msg_t>())
+        , fViewMsg(nullptr)
+    {
+        // FIXME: make this zero-copy:
+        // simply taking over the provided buffer can casue premature delete, since region could be
+        // destroyed before the message is sent out. Needs lifetime extension for the ZMQ region.
+        if (zmq_msg_init_size(fMsg.get(), size) != 0) {
+            LOG(error) << "failed initializing message with size, reason: " << zmq_strerror(errno);
+        }
+
+        std::memcpy(zmq_msg_data(fMsg.get()), data, size);
+        // call region callback
+        auto ptr = static_cast<UnmanagedRegion*>(region.get());
+        if (ptr->fBulkCallback) {
+            ptr->fBulkCallback({{data, size, hint}});
+        } else if (ptr->fCallback) {
+            ptr->fCallback(data, size, hint);
+        }
+
+        // if (zmq_msg_init_data(fMsg.get(), data, size, [](void*, void*){}, nullptr) != 0)
+        // {
+        //     LOG(error) << "failed initializing message with data, reason: " <<
+        //     zmq_strerror(errno);
+        // }
+    }
+
+    void Rebuild() override
+    {
+        CloseMessage();
+        fMsg = tools::make_unique<zmq_msg_t>();
+        if (zmq_msg_init(fMsg.get()) != 0) {
+            LOG(error) << "failed initializing message, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    void Rebuild(const size_t size) override
+    {
+        CloseMessage();
+        fMsg = tools::make_unique<zmq_msg_t>();
+        if (zmq_msg_init_size(fMsg.get(), size) != 0) {
+            LOG(error) << "failed initializing message with size, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override
+    {
+        CloseMessage();
+        fMsg = tools::make_unique<zmq_msg_t>();
+        if (zmq_msg_init_data(fMsg.get(), data, size, ffn, hint) != 0) {
+            LOG(error) << "failed initializing message with data, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    void* GetData() const override
+    {
+        if (!fViewMsg) {
+            return zmq_msg_data(fMsg.get());
+        } else {
+            return zmq_msg_data(fViewMsg.get());
+        }
+    }
+
+    size_t GetSize() const override
+    {
+        if (fUsedSizeModified) {
+            return fUsedSize;
+        } else {
+            return zmq_msg_size(fMsg.get());
+        }
+    }
+
+    // To emulate shrinking, a new message is created with the new size (ViewMsg), that points to
+    // the original buffer with the new size. Once the "view message" is transfered, the original is
+    // destroyed. Used size is applied only once in ApplyUsedSize, which is called by the socket
+    // before sending. This function just updates the desired size until the actual "resizing"
+    // happens.
+    bool SetUsedSize(const size_t size) override
+    {
+        if (size <= zmq_msg_size(fMsg.get())) {
+            fUsedSize = size;
+            fUsedSizeModified = true;
+            return true;
+        } else {
+            LOG(error) << "cannot set used size higher than original.";
+            return false;
+        }
+    }
+
+    void ApplyUsedSize()
+    {
+        // Apply only once (before actual send).
+        // The check is needed because a send could fail and can be reattempted by the user, in this
+        // case we do not want to modify buffer again.
+        if (fUsedSizeModified && !fViewMsg) {
+            fViewMsg = tools::make_unique<zmq_msg_t>();
+            void* ptr = zmq_msg_data(fMsg.get());
+            if (zmq_msg_init_data(fViewMsg.get(), ptr, fUsedSize, [](void* /* data */, void* obj) {
+                    zmq_msg_close(static_cast<zmq_msg_t*>(obj));
+                    delete static_cast<zmq_msg_t*>(obj);
+                }, fMsg.release()) != 0) {
+                LOG(error) << "failed initializing view message, reason: " << zmq_strerror(errno);
+            }
+        }
+    }
+
+    Transport GetType() const override { return Transport::ZMQ; }
+
+    void Copy(const fair::mq::Message& msg) override
+    {
+        const Message& zMsg = static_cast<const Message&>(msg);
+        // Shares the message buffer between msg and this fMsg.
+        if (zmq_msg_copy(fMsg.get(), zMsg.GetMessage()) != 0) {
+            LOG(error) << "failed copying message, reason: " << zmq_strerror(errno);
+            return;
+        }
+
+        // if the target message has been resized, apply same to this message also
+        if (zMsg.fUsedSizeModified) {
+            fUsedSizeModified = true;
+            fUsedSize = zMsg.fUsedSize;
+        }
+    }
+
+    ~Message() override { CloseMessage(); }
+
+  private:
+    bool fUsedSizeModified;
+    size_t fUsedSize;
+    std::unique_ptr<zmq_msg_t> fMsg;
+    std::unique_ptr<zmq_msg_t> fViewMsg;   // view on a subset of fMsg (treating it as user buffer)
+
+    zmq_msg_t* GetMessage() const
+    {
+        if (!fViewMsg) {
+            return fMsg.get();
+        } else {
+            return fViewMsg.get();
+        }
+    }
+
+    void CloseMessage()
+    {
+        if (!fViewMsg) {
+            if (zmq_msg_close(fMsg.get()) != 0) {
+                LOG(error) << "failed closing message, reason: " << zmq_strerror(errno);
+            }
+            // reset the message object to allow reuse in Rebuild
+            fMsg.reset(nullptr);
+        } else {
+            if (zmq_msg_close(fViewMsg.get()) != 0) {
+                LOG(error) << "failed closing message, reason: " << zmq_strerror(errno);
+            }
+            // reset the message object to allow reuse in Rebuild
+            fViewMsg.reset(nullptr);
+        }
+        fUsedSizeModified = false;
+        fUsedSize = 0;
+    }
+};
+
+}   // namespace zmq
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_ZMQ_MESSAGE_H */

fairmq/zeromq/Poller.h

@@ -0,0 +1,204 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_ZMQ_POLLER_H
+#define FAIR_MQ_ZMQ_POLLER_H
+
+#include <fairmq/zeromq/Socket.h>
+#include <fairmq/Tools.h>
+#include <FairMQChannel.h>
+#include <FairMQLogger.h>
+#include <FairMQPoller.h>
+
+#include <zmq.h>
+
+#include <unordered_map>
+#include <vector>
+
+namespace fair
+{
+namespace mq
+{
+namespace zmq
+{
+
+class Poller final : public fair::mq::Poller
+{
+  public:
+    Poller(const std::vector<FairMQChannel>& channels)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        fNumItems = channels.size();
+        fItems = new zmq_pollitem_t[fNumItems];   // TODO: fix me
+
+        for (int i = 0; i < fNumItems; ++i) {
+            fItems[i].socket = static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
+            fItems[i].fd = 0;
+            fItems[i].revents = 0;
+
+            int type = 0;
+            size_t size = sizeof(type);
+            zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+            SetItemEvents(fItems[i], type);
+        }
+    }
+
+    Poller(const std::vector<FairMQChannel*>& channels)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        fNumItems = channels.size();
+        fItems = new zmq_pollitem_t[fNumItems];
+
+        for (int i = 0; i < fNumItems; ++i) {
+            fItems[i].socket = static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
+            fItems[i].fd = 0;
+            fItems[i].revents = 0;
+
+            int type = 0;
+            size_t size = sizeof(type);
+            zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+            SetItemEvents(fItems[i], type);
+        }
+    }
+
+    Poller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList)
+        : fItems()
+        , fNumItems(0)
+        , fOffsetMap()
+    {
+        try {
+            int offset = 0;
+            // calculate offsets and the total size of the poll item set
+            for (std::string channel : channelList) {
+                fOffsetMap[channel] = offset;
+                offset += channelsMap.at(channel).size();
+                fNumItems += channelsMap.at(channel).size();
+            }
+
+            fItems = new zmq_pollitem_t[fNumItems];
+
+            int index = 0;
+            for (std::string channel : channelList) {
+                for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i) {
+                    index = fOffsetMap[channel] + i;
+
+                    fItems[index].socket = static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
+                    fItems[index].fd = 0;
+                    fItems[index].revents = 0;
+
+                    int type = 0;
+                    size_t size = sizeof(type);
+                    zmq_getsockopt(static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+
+                    SetItemEvents(fItems[index], type);
+                }
+            }
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "at least one of the provided channel keys for poller initialization is invalid";
+            LOG(error) << "out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("At least one of the provided channel keys for poller initialization is invalid. ", "Out of range error: ", oor.what()));
+        }
+    }
+
+    Poller(const Poller&) = delete;
+    Poller operator=(const Poller&) = delete;
+
+    void SetItemEvents(zmq_pollitem_t& item, const int type)
+    {
+        if (type == ZMQ_REQ || type == ZMQ_REP || type == ZMQ_PAIR || type == ZMQ_DEALER || type == ZMQ_ROUTER) {
+            item.events = ZMQ_POLLIN | ZMQ_POLLOUT;
+        } else if (type == ZMQ_PUSH || type == ZMQ_PUB || type == ZMQ_XPUB) {
+            item.events = ZMQ_POLLOUT;
+        } else if (type == ZMQ_PULL || type == ZMQ_SUB || type == ZMQ_XSUB) {
+            item.events = ZMQ_POLLIN;
+        } else {
+            LOG(error) << "invalid poller configuration, exiting.";
+            throw fair::mq::PollerError("Invalid poller configuration, exiting.");
+        }
+    }
+
+    void Poll(const int timeout) override
+    {
+        if (zmq_poll(fItems, fNumItems, timeout) < 0) {
+            if (errno == ETERM) {
+                LOG(debug) << "polling exited, reason: " << zmq_strerror(errno);
+            } else {
+                LOG(error) << "polling failed, reason: " << zmq_strerror(errno);
+                throw fair::mq::PollerError(fair::mq::tools::ToString("Polling failed, reason: ", zmq_strerror(errno)));
+            }
+        }
+    }
+
+    bool CheckInput(const int index) override
+    {
+        if (fItems[index].revents & ZMQ_POLLIN) {
+            return true;
+        }
+
+        return false;
+    }
+
+    bool CheckOutput(const int index) override
+    {
+        if (fItems[index].revents & ZMQ_POLLOUT) {
+            return true;
+        }
+
+        return false;
+    }
+
+    bool CheckInput(const std::string& channelKey, const int index) override
+    {
+        try {
+            if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN) {
+                return true;
+            }
+
+            return false;
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "invalid channel key: '" << channelKey << "'";
+            LOG(error) << "out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("Invalid channel key '", channelKey, "'. Out of range error: ", oor.what()));
+        }
+    }
+
+    bool CheckOutput(const std::string& channelKey, const int index) override
+    {
+        try {
+            if (fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT) {
+                return true;
+            }
+
+            return false;
+        } catch (const std::out_of_range& oor) {
+            LOG(error) << "invalid channel key: '" << channelKey << "'";
+            LOG(error) << "out of range error: " << oor.what();
+            throw fair::mq::PollerError(fair::mq::tools::ToString("Invalid channel key '", channelKey, "'. Out of range error: ", oor.what()));
+        }
+    }
+
+    ~Poller() override { delete[] fItems; }
+
+  private:
+    zmq_pollitem_t* fItems;
+    int fNumItems;
+
+    std::unordered_map<std::string, int> fOffsetMap;
+};
+
+}   // namespace zmq
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_ZMQ_POLLER_H */

fairmq/zeromq/Socket.h

@@ -0,0 +1,484 @@
+/********************************************************************************
+ * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_ZMQ_SOCKET_H
+#define FAIR_MQ_ZMQ_SOCKET_H
+
+#include <FairMQLogger.h>
+#include <FairMQMessage.h>
+#include <FairMQSocket.h>
+#include <atomic>
+#include <fairmq/Tools.h>
+#include <fairmq/zeromq/Context.h>
+#include <fairmq/zeromq/Message.h>
+#include <memory>   // unique_ptr
+#include <zmq.h>
+
+namespace fair {
+namespace mq {
+namespace zmq {
+
+class Socket final : public fair::mq::Socket
+{
+  public:
+    Socket(Context& ctx, const std::string& type, const std::string& name, const std::string& id, FairMQTransportFactory* factory = nullptr)
+        : fair::mq::Socket(factory)
+        , fCtx(ctx)
+        , fSocket(zmq_socket(fCtx.GetZmqCtx(), GetConstant(type)))
+        , fId(id + "." + name + "." + type)
+        , fBytesTx(0)
+        , fBytesRx(0)
+        , fMessagesTx(0)
+        , fMessagesRx(0)
+        , fSndTimeout(100)
+        , fRcvTimeout(100)
+    {
+        if (fSocket == nullptr) {
+            LOG(error) << "Failed creating socket " << fId << ", reason: " << zmq_strerror(errno);
+            throw SocketError(tools::ToString("Unavailable transport requested: ", type));
+        }
+
+        if (zmq_setsockopt(fSocket, ZMQ_IDENTITY, fId.c_str(), fId.length()) != 0) {
+            LOG(error) << "Failed setting ZMQ_IDENTITY socket option, reason: " << zmq_strerror(errno);
+        }
+
+        // Tell socket to try and send/receive outstanding messages for <linger> milliseconds before
+        // terminating. Default value for ZeroMQ is -1, which is to wait forever.
+        int linger = 1000;
+        if (zmq_setsockopt(fSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0) {
+            LOG(error) << "Failed setting ZMQ_LINGER socket option, reason: " << zmq_strerror(errno);
+        }
+
+        if (zmq_setsockopt(fSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0) {
+            LOG(error) << "Failed setting ZMQ_SNDTIMEO socket option, reason: " << zmq_strerror(errno);
+        }
+
+        if (zmq_setsockopt(fSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0) {
+            LOG(error) << "Failed setting ZMQ_RCVTIMEO socket option, reason: " << zmq_strerror(errno);
+        }
+
+        if (type == "sub") {
+            if (zmq_setsockopt(fSocket, ZMQ_SUBSCRIBE, nullptr, 0) != 0) {
+                LOG(error) << "Failed setting ZMQ_SUBSCRIBE socket option, reason: " << zmq_strerror(errno);
+            }
+        }
+
+        LOG(debug) << "Created socket " << GetId();
+    }
+
+    Socket(const Socket&) = delete;
+    Socket operator=(const Socket&) = delete;
+
+    std::string GetId() const override { return fId; }
+
+    bool Bind(const std::string& address) override
+    {
+        // LOG(info) << "bind socket " << fId << " on " << address;
+
+        if (zmq_bind(fSocket, address.c_str()) != 0) {
+            if (errno == EADDRINUSE) {
+                // do not print error in this case, this is handled by FairMQDevice in case no
+                // connection could be established after trying a number of random ports from a
+                // range.
+                return false;
+            }
+            LOG(error) << "Failed binding socket " << fId << ", address: " << address << ", reason: " << zmq_strerror(errno);
+            return false;
+        }
+
+        return true;
+    }
+
+    bool Connect(const std::string& address) override
+    {
+        // LOG(info) << "connect socket " << fId << " on " << address;
+
+        if (zmq_connect(fSocket, address.c_str()) != 0) {
+            LOG(error) << "Failed connecting socket " << fId << ", address: " << address << ", reason: " << zmq_strerror(errno);
+            return false;
+        }
+
+        return true;
+    }
+
+    int Send(MessagePtr& msg, const int timeout = -1) override
+    {
+        int flags = 0;
+        if (timeout == 0) {
+            flags = ZMQ_DONTWAIT;
+        }
+        int elapsed = 0;
+
+        static_cast<Message*>(msg.get())->ApplyUsedSize();
+
+        while (true) {
+            int nbytes = zmq_msg_send(static_cast<Message*>(msg.get())->GetMessage(), fSocket, flags);
+            if (nbytes >= 0) {
+                fBytesTx += nbytes;
+                ++fMessagesTx;
+
+                return nbytes;
+            } else if (zmq_errno() == EAGAIN) {
+                if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
+                    if (timeout > 0) {
+                        elapsed += fSndTimeout;
+                        if (elapsed >= timeout) {
+                            return -2;
+                        }
+                    }
+                    continue;
+                } else {
+                    return -2;
+                }
+            } else if (zmq_errno() == ETERM) {
+                LOG(info) << "terminating socket " << fId;
+                return -1;
+            } else if (zmq_errno() == EINTR) {
+                LOG(debug) << "Send interrupted by system call";
+                return nbytes;
+            } else {
+                LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
+                return nbytes;
+            }
+        }
+    }
+
+    int Receive(MessagePtr& msg, const int timeout = -1) override
+    {
+        int flags = 0;
+        if (timeout == 0) {
+            flags = ZMQ_DONTWAIT;
+        }
+        int elapsed = 0;
+
+        while (true) {
+            int nbytes = zmq_msg_recv(static_cast<Message*>(msg.get())->GetMessage(), fSocket, flags);
+            if (nbytes >= 0) {
+                fBytesRx += nbytes;
+                ++fMessagesRx;
+                return nbytes;
+            } else if (zmq_errno() == EAGAIN) {
+                if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
+                    if (timeout > 0) {
+                        elapsed += fRcvTimeout;
+                        if (elapsed >= timeout) {
+                            return -2;
+                        }
+                    }
+                    continue;
+                } else {
+                    return -2;
+                }
+            } else if (zmq_errno() == ETERM) {
+                LOG(info) << "terminating socket " << fId;
+                return -1;
+            } else if (zmq_errno() == EINTR) {
+                LOG(debug) << "Receive interrupted by system call";
+                return nbytes;
+            } else {
+                LOG(error) << "Failed receiving on socket " << fId << ", reason: " << zmq_strerror(errno);
+                return nbytes;
+            }
+        }
+    }
+
+    int64_t Send(std::vector<std::unique_ptr<fair::mq::Message>>& msgVec, const int timeout = -1) override
+    {
+        int flags = 0;
+        if (timeout == 0) {
+            flags = ZMQ_DONTWAIT;
+        }
+
+        const unsigned int vecSize = msgVec.size();
+
+        // Sending vector typicaly handles more then one part
+        if (vecSize > 1) {
+            int elapsed = 0;
+
+            while (true) {
+                int64_t totalSize = 0;
+                bool repeat = false;
+
+                for (unsigned int i = 0; i < vecSize; ++i) {
+                    static_cast<Message*>(msgVec[i].get())->ApplyUsedSize();
+
+                    int nbytes = zmq_msg_send(static_cast<Message*>(msgVec[i].get())->GetMessage(), fSocket, (i < vecSize - 1) ? ZMQ_SNDMORE | flags : flags);
+                    if (nbytes >= 0) {
+                        totalSize += nbytes;
+                    } else {
+                        // according to ZMQ docs, this can only occur for the first part
+                        if (zmq_errno() == EAGAIN) {
+                            if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
+                                if (timeout > 0) {
+                                    elapsed += fSndTimeout;
+                                    if (elapsed >= timeout) {
+                                        return -2;
+                                    }
+                                }
+                                repeat = true;
+                                break;
+                            } else {
+                                return -2;
+                            }
+                        }
+                        if (zmq_errno() == ETERM) {
+                            LOG(info) << "terminating socket " << fId;
+                            return -1;
+                        } else if (zmq_errno() == EINTR) {
+                            LOG(debug) << "Receive interrupted by system call";
+                            return nbytes;
+                        } else {
+                            LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
+                            return nbytes;
+                        }
+                    }
+                }
+
+                if (repeat) {
+                    continue;
+                }
+
+                // store statistics on how many messages have been sent (handle all parts as a
+                // single message)
+                ++fMessagesTx;
+                fBytesTx += totalSize;
+                return totalSize;
+            }
+        }   // If there's only one part, send it as a regular message
+        else if (vecSize == 1) {
+            return Send(msgVec.back(), timeout);
+        } else {   // if the vector is empty, something might be wrong
+            LOG(warn) << "Will not send empty vector";
+            return -1;
+        }
+    }
+
+    int64_t Receive(std::vector<std::unique_ptr<fair::mq::Message>>& msgVec, const int timeout = -1) override
+    {
+        int flags = 0;
+        if (timeout == 0) {
+            flags = ZMQ_DONTWAIT;
+        }
+        int elapsed = 0;
+
+        while (true) {
+            int64_t totalSize = 0;
+            int64_t more = 0;
+            bool repeat = false;
+
+            do {
+                FairMQMessagePtr part = tools::make_unique<Message>(GetTransport());
+
+                int nbytes = zmq_msg_recv(static_cast<Message*>(part.get())->GetMessage(), fSocket, flags);
+                if (nbytes >= 0) {
+                    msgVec.push_back(move(part));
+                    totalSize += nbytes;
+                } else if (zmq_errno() == EAGAIN) {
+                    if (!fCtx.Interrupted() && ((flags & ZMQ_DONTWAIT) == 0)) {
+                        if (timeout > 0) {
+                            elapsed += fRcvTimeout;
+                            if (elapsed >= timeout) {
+                                return -2;
+                            }
+                        }
+                        repeat = true;
+                        break;
+                    } else {
+                        return -2;
+                    }
+                } else if (zmq_errno() == EINTR) {
+                    LOG(debug) << "Receive interrupted by system call";
+                    return nbytes;
+                } else {
+                    return nbytes;
+                }
+
+                size_t moreSize = sizeof(more);
+                zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &moreSize);
+            } while (more);
+
+            if (repeat) {
+                continue;
+            }
+
+            // store statistics on how many messages have been received (handle all parts as a
+            // single message)
+            ++fMessagesRx;
+            fBytesRx += totalSize;
+            return totalSize;
+        }
+    }
+
+    void* GetSocket() const { return fSocket; }
+
+    void Close() override
+    {
+        // LOG(debug) << "Closing socket " << fId;
+
+        if (fSocket == nullptr) {
+            return;
+        }
+
+        if (zmq_close(fSocket) != 0) {
+            LOG(error) << "Failed closing socket " << fId << ", reason: " << zmq_strerror(errno);
+        }
+
+        fSocket = nullptr;
+    }
+
+    void SetOption(const std::string& option, const void* value, size_t valueSize) override
+    {
+        if (zmq_setsockopt(fSocket, GetConstant(option), value, valueSize) < 0) {
+            LOG(error) << "Failed setting socket option, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    void GetOption(const std::string& option, void* value, size_t* valueSize) override
+    {
+        if (zmq_getsockopt(fSocket, GetConstant(option), value, valueSize) < 0) {
+            LOG(error) << "Failed getting socket option, reason: " << zmq_strerror(errno);
+        }
+    }
+
+    void SetLinger(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_LINGER, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed setting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetLinger() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_LINGER, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    void SetSndBufSize(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_SNDHWM, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed setting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetSndBufSize() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_SNDHWM, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    void SetRcvBufSize(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_RCVHWM, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed setting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetRcvBufSize() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_RCVHWM, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    void SetSndKernelSize(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_SNDBUF, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetSndKernelSize() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_SNDBUF, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    void SetRcvKernelSize(const int value) override
+    {
+        if (zmq_setsockopt(fSocket, ZMQ_RCVBUF, &value, sizeof(value)) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
+        }
+    }
+
+    int GetRcvKernelSize() const override
+    {
+        int value = 0;
+        size_t valueSize = sizeof(value);
+        if (zmq_getsockopt(fSocket, ZMQ_RCVBUF, &value, &valueSize) < 0) {
+            throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
+        }
+        return value;
+    }
+
+    unsigned long GetBytesTx() const override { return fBytesTx; }
+    unsigned long GetBytesRx() const override { return fBytesRx; }
+    unsigned long GetMessagesTx() const override { return fMessagesTx; }
+    unsigned long GetMessagesRx() const override { return fMessagesRx; }
+
+    static int GetConstant(const std::string& constant)
+    {
+        if (constant == "") return 0;
+        if (constant == "sub") return ZMQ_SUB;
+        if (constant == "pub") return ZMQ_PUB;
+        if (constant == "xsub") return ZMQ_XSUB;
+        if (constant == "xpub") return ZMQ_XPUB;
+        if (constant == "push") return ZMQ_PUSH;
+        if (constant == "pull") return ZMQ_PULL;
+        if (constant == "req") return ZMQ_REQ;
+        if (constant == "rep") return ZMQ_REP;
+        if (constant == "dealer") return ZMQ_DEALER;
+        if (constant == "router") return ZMQ_ROUTER;
+        if (constant == "pair") return ZMQ_PAIR;
+
+        if (constant == "snd-hwm") return ZMQ_SNDHWM;
+        if (constant == "rcv-hwm") return ZMQ_RCVHWM;
+        if (constant == "snd-size") return ZMQ_SNDBUF;
+        if (constant == "rcv-size") return ZMQ_RCVBUF;
+        if (constant == "snd-more") return ZMQ_SNDMORE;
+        if (constant == "rcv-more") return ZMQ_RCVMORE;
+
+        if (constant == "linger") return ZMQ_LINGER;
+
+        return -1;
+    }
+
+    ~Socket() override { Close(); }
+
+  private:
+    Context& fCtx;
+    void* fSocket;
+    std::string fId;
+    std::atomic<unsigned long> fBytesTx;
+    std::atomic<unsigned long> fBytesRx;
+    std::atomic<unsigned long> fMessagesTx;
+    std::atomic<unsigned long> fMessagesRx;
+
+    int fSndTimeout;
+    int fRcvTimeout;
+};
+
+}   // namespace zmq
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_ZMQ_SOCKET_H */

fairmq/zeromq/TransportFactory.h

@@ -0,0 +1,153 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_ZMQ_TRANSPORTFACTORY_H
+#define FAIR_MQ_ZMQ_TRANSPORTFACTORY_H
+
+#include <fairmq/zeromq/Context.h>
+#include <fairmq/zeromq/Message.h>
+#include <fairmq/zeromq/Socket.h>
+#include <fairmq/zeromq/Poller.h>
+#include <fairmq/zeromq/UnmanagedRegion.h>
+#include <fairmq/Tools.h>
+#include <FairMQTransportFactory.h>
+#include <fairmq/ProgOptions.h>
+
+#include <memory> // unique_ptr
+#include <string>
+#include <vector>
+
+namespace fair
+{
+namespace mq
+{
+namespace zmq
+{
+
+class TransportFactory final : public FairMQTransportFactory
+{
+  public:
+    TransportFactory(const std::string& id = "", const ProgOptions* config = nullptr)
+        : FairMQTransportFactory(id)
+        , fCtx(nullptr)
+    {
+        int major, minor, patch;
+        zmq_version(&major, &minor, &patch);
+        LOG(debug) << "Transport: Using ZeroMQ library, version: " << major << "." << minor << "." << patch;
+
+        if (config) {
+            fCtx = tools::make_unique<Context>(config->GetProperty<int>("io-threads", 1));
+        } else {
+            LOG(debug) << "fair::mq::ProgOptions not available! Using defaults.";
+            fCtx = tools::make_unique<Context>(1);
+        }
+    }
+
+    TransportFactory(const TransportFactory&) = delete;
+    TransportFactory operator=(const TransportFactory&) = delete;
+
+    MessagePtr CreateMessage() override
+    {
+        return tools::make_unique<Message>(this);
+    }
+    
+    MessagePtr CreateMessage(Alignment alignment) override
+    {
+        return tools::make_unique<Message>(alignment, this);
+    }
+
+    MessagePtr CreateMessage(const size_t size) override
+    {
+        return tools::make_unique<Message>(size, this);
+    }
+
+    MessagePtr CreateMessage(const size_t size, Alignment alignment) override
+    {
+        return tools::make_unique<Message>(size, alignment, this);
+    }
+
+    MessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override
+    {
+        return tools::make_unique<Message>(data, size, ffn, hint, this);
+    }
+
+    MessagePtr CreateMessage(UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0) override
+    {
+        return tools::make_unique<Message>(region, data, size, hint, this);
+    }
+
+    SocketPtr CreateSocket(const std::string& type, const std::string& name) override
+    {
+        return tools::make_unique<Socket>(*fCtx, type, name, GetId(), this);
+    }
+
+    PollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override
+    {
+        return tools::make_unique<Poller>(channels);
+    }
+
+    PollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const override
+    {
+        return tools::make_unique<Poller>(channels);
+    }
+
+    PollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override
+    {
+        return tools::make_unique<Poller>(channelsMap, channelList);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, RegionCallback callback, const std::string& path = "", int flags = 0) override
+    {
+        return CreateUnmanagedRegion(size, 0, callback, nullptr, path, flags);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, RegionBulkCallback bulkCallback, const std::string& path = "", int flags = 0) override
+    {
+        return CreateUnmanagedRegion(size, 0, nullptr, bulkCallback, path, flags);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const std::string& path = "", int flags = 0) override
+    {
+        return CreateUnmanagedRegion(size, userFlags, callback, nullptr, path, flags);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, RegionBulkCallback bulkCallback, const std::string& path = "", int flags = 0) override
+    {
+        return CreateUnmanagedRegion(size, userFlags, nullptr, bulkCallback, path, flags);
+    }
+
+    UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, const int64_t userFlags, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string&, int)
+    {
+        UnmanagedRegionPtr ptr = tools::make_unique<UnmanagedRegion>(*fCtx, size, userFlags, callback, bulkCallback, this);
+        auto zPtr = static_cast<UnmanagedRegion*>(ptr.get());
+        fCtx->AddRegion(zPtr->GetId(), zPtr->GetData(), zPtr->GetSize(), zPtr->GetUserFlags(), RegionEvent::created);
+        return ptr;
+    }
+
+    void SubscribeToRegionEvents(RegionEventCallback callback) override { fCtx->SubscribeToRegionEvents(callback); }
+    bool SubscribedToRegionEvents() override { return fCtx->SubscribedToRegionEvents(); }
+    void UnsubscribeFromRegionEvents() override { fCtx->UnsubscribeFromRegionEvents(); }
+    std::vector<RegionInfo> GetRegionInfo() override { return fCtx->GetRegionInfo(); }
+
+    Transport GetType() const override { return Transport::ZMQ; }
+
+    void Interrupt() override { fCtx->Interrupt(); }
+    void Resume() override { fCtx->Resume(); }
+    void Reset() override { fCtx->Reset(); }
+
+    ~TransportFactory() override { LOG(debug) << "Destroying ZeroMQ transport..."; }
+
+  private:
+    std::unique_ptr<Context> fCtx;
+};
+
+}   // namespace zmq
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_ZMQ_TRANSPORTFACTORY_H */

fairmq/zeromq/UnmanagedRegion.h

@@ -0,0 +1,77 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_ZMQ_UNMANAGEDREGION_H
+#define FAIR_MQ_ZMQ_UNMANAGEDREGION_H
+
+#include <fairmq/zeromq/Context.h>
+#include <FairMQUnmanagedRegion.h>
+#include <FairMQLogger.h>
+
+#include <cstddef> // size_t
+#include <string>
+
+namespace fair
+{
+namespace mq
+{
+namespace zmq
+{
+
+class UnmanagedRegion final : public fair::mq::UnmanagedRegion
+{
+    friend class Socket;
+    friend class Message;
+
+  public:
+    UnmanagedRegion(Context& ctx,
+                    size_t size,
+                    int64_t userFlags,
+                    RegionCallback callback,
+                    RegionBulkCallback bulkCallback,
+                    FairMQTransportFactory* factory = nullptr)
+        : fair::mq::UnmanagedRegion(factory)
+        , fCtx(ctx)
+        , fId(fCtx.RegionCount())
+        , fBuffer(malloc(size))
+        , fSize(size)
+        , fUserFlags(userFlags)
+        , fCallback(callback)
+        , fBulkCallback(bulkCallback)
+    {}
+
+    UnmanagedRegion(const UnmanagedRegion&) = delete;
+    UnmanagedRegion operator=(const UnmanagedRegion&) = delete;
+
+    virtual void* GetData() const override { return fBuffer; }
+    virtual size_t GetSize() const override { return fSize; }
+    uint64_t GetId() const override { return fId; }
+    int64_t GetUserFlags() const { return fUserFlags; }
+
+    virtual ~UnmanagedRegion()
+    {
+        LOG(debug) << "destroying region " << fId;
+        fCtx.RemoveRegion(fId);
+        free(fBuffer);
+    }
+
+  private:
+    Context& fCtx;
+    uint64_t fId;
+    void* fBuffer;
+    size_t fSize;
+    int64_t fUserFlags;
+    RegionCallback fCallback;
+    RegionBulkCallback fBulkCallback;
+};
+
+}   // namespace zmq
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_ZMQ_UNMANAGEDREGION_H */

test/CMakeLists.txt

@@ -32,10 +32,6 @@ add_testhelper(runTestDevice
     LINKS FairMQ
 )
 
-if(BUILD_NANOMSG_TRANSPORT)
-  list(APPEND definitions BUILD_NANOMSG_TRANSPORT)
-endif()
-
 if(BUILD_OFI_TRANSPORT)
   LIST(APPEND definitions BUILD_OFI_TRANSPORT)
 endif()
@@ -96,6 +92,19 @@ add_testsuite(Message
     ${definitions}
 )
 
+add_testsuite(Region
+    SOURCES
+    ${CMAKE_CURRENT_BINARY_DIR}/runner.cxx
+    region/_region.cxx
+
+    LINKS FairMQ
+    INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}
+             ${CMAKE_CURRENT_SOURCE_DIR}/region
+             ${CMAKE_CURRENT_BINARY_DIR}
+    TIMEOUT 5
+    ${definitions}
+)
+
 add_testsuite(Device
     SOURCES
     ${CMAKE_CURRENT_BINARY_DIR}/runner.cxx

test/message/_message.cxx

@@ -15,6 +15,7 @@
 #include <gtest/gtest.h>
 
 #include <string>
+#include <cstdint>
 
 namespace
 {
@@ -77,6 +78,19 @@ void RunMsgRebuild(const string& transport)
     EXPECT_EQ(string(static_cast<char*>(msg->GetData()), msg->GetSize()), string("asdf"));
 }
 
+void Alignment(const string& transport)
+{
+    size_t session{fair::mq::tools::UuidHash()};
+
+    fair::mq::ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config);
+
+    FairMQMessagePtr msg(factory->CreateMessage(100, fair::mq::Alignment{64}));
+    ASSERT_EQ(reinterpret_cast<uintptr_t>(msg->GetData()) % 64, 0);
+}
+
 TEST(Resize, zeromq)
 {
     RunPushPullWithMsgResize("zeromq", "ipc://test_message_resize");
@@ -87,13 +101,6 @@ TEST(Resize, shmem)
     RunPushPullWithMsgResize("shmem", "ipc://test_message_resize");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(Resize, nanomsg)
-{
-    RunPushPullWithMsgResize("nanomsg", "ipc://test_message_resize");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 TEST(Rebuild, zeromq)
 {
     RunMsgRebuild("zeromq");
@@ -104,11 +111,9 @@ TEST(Rebuild, shmem)
     RunMsgRebuild("shmem");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(Rebuild, nanomsg)
+TEST(Alignment, shmem) // TODO: add test for ZeroMQ once it is implemented
 {
-    RunMsgRebuild("nanomsg");
+    Alignment("shmem");
 }
-#endif /* BUILD_NANOMSG_TRANSPORT */
 
 } // namespace

test/poller/_poller.cxx

@@ -55,13 +55,6 @@ TEST(Subchannel, zeromq)
     EXPECT_EXIT(RunPoller("zeromq", 0), ::testing::ExitedWithCode(0), "POLL test successfull");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(Subchannel, nanomsg)
-{
-    EXPECT_EXIT(RunPoller("nanomsg", 0), ::testing::ExitedWithCode(0), "POLL test successfull");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 TEST(Subchannel, shmem)
 {
     EXPECT_EXIT(RunPoller("shmem", 0), ::testing::ExitedWithCode(0), "POLL test successfull");
@@ -72,13 +65,6 @@ TEST(Channel, zeromq)
     EXPECT_EXIT(RunPoller("zeromq", 1), ::testing::ExitedWithCode(0), "POLL test successfull");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(Channel, nanomsg)
-{
-    EXPECT_EXIT(RunPoller("nanomsg", 1), ::testing::ExitedWithCode(0), "POLL test successfull");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 TEST(Channel, shmem)
 {
     EXPECT_EXIT(RunPoller("shmem", 1), ::testing::ExitedWithCode(0), "POLL test successfull");

test/protocols/_pair.cxx

@@ -56,13 +56,6 @@ TEST(Pair, SingleMsg_MP_tcp_shmem)
     EXPECT_EXIT(RunPair("shmem"), ::testing::ExitedWithCode(0), "PAIR test successfull");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(Pair, SingleMsg_MP_tcp_nanomsg)
-{
-    EXPECT_EXIT(RunPair("nanomsg"), ::testing::ExitedWithCode(0), "PAIR test successfull");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 #ifdef BUILD_OFI_TRANSPORT
 TEST(Pair, SingleMsg_MP_tcp_ofi)
 {

test/protocols/_pub_sub.cxx

@@ -60,11 +60,4 @@ TEST(PubSub, zeromq)
     EXPECT_EXIT(RunPubSub("zeromq"), ::testing::ExitedWithCode(0), "PUB-SUB test successfull");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(PubSub, nanomsg)
-{
-    EXPECT_EXIT(RunPubSub("nanomsg"), ::testing::ExitedWithCode(0), "PUB-SUB test successfull");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 } // namespace

test/protocols/_push_pull.cxx

@@ -56,11 +56,4 @@ TEST(PushPull, SingleMsg_MP_tcp_shmem)
     EXPECT_EXIT(RunPushPull("shmem"), ::testing::ExitedWithCode(0), "PUSH-PULL test successfull");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(PushPull, SingleMsg_MP_tcp_nanomsg)
-{
-    EXPECT_EXIT(RunPushPull("nanomsg"), ::testing::ExitedWithCode(0), "PUSH-PULL test successfull");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 } // namespace

test/protocols/_push_pull_multipart.cxx

@@ -158,13 +158,6 @@ TEST(PushPull, Multipart_ST_inproc_shmem)
     RunSingleThreadedMultipart("shmem", "inproc://test1", "inproc://test2");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(PushPull, Multipart_ST_inproc_nanomsg)
-{
-    RunSingleThreadedMultipart("nanomsg", "inproc://test1", "inproc://test2");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 TEST(PushPull, Multipart_ST_ipc_zeromq)
 {
     RunSingleThreadedMultipart("zeromq", "ipc://test_Multipart_ST_ipc_zeromq_1", "ipc://test_Multipart_ST_ipc_zeromq_2");
@@ -175,13 +168,6 @@ TEST(PushPull, Multipart_ST_ipc_shmen)
     RunSingleThreadedMultipart("shmem", "ipc://test_Multipart_ST_ipc_shmen_1", "ipc://test_Multipart_ST_ipc_shmen_2");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(PushPull, Multipart_ST_ipc_nanomsg)
-{
-    RunSingleThreadedMultipart("nanomsg", "ipc://test_Multipart_ST_ipc_nanomsg_1", "ipc://test_Multipart_ST_ipc_nanomsg_2");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 TEST(PushPull, Multipart_MT_inproc_zeromq)
 {
     RunMultiThreadedMultipart("zeromq", "inproc://test_1");
@@ -192,13 +178,6 @@ TEST(PushPull, Multipart_MT_inproc_shmem)
     RunMultiThreadedMultipart("shmem", "inproc://test_1");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(PushPull, Multipart_MT_inproc_nanomsg)
-{
-    RunMultiThreadedMultipart("nanomsg", "inproc://test_1");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 TEST(PushPull, Multipart_MT_ipc_zeromq)
 {
     RunMultiThreadedMultipart("zeromq", "ipc://test_Multipart_MT_ipc_zeromq_1");
@@ -209,11 +188,4 @@ TEST(PushPull, Multipart_MT_ipc_shmem)
     RunMultiThreadedMultipart("shmem", "ipc://test_Multipart_MT_ipc_shmem_1");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(PushPull, Multipart_MT_ipc_nanomsg)
-{
-    RunMultiThreadedMultipart("nanomsg", "ipc://test_Multipart_MT_ipc_nanomsg_1");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 } // namespace

test/protocols/_req_rep.cxx

@@ -65,11 +65,4 @@ TEST(ReqRep, shmem)
     EXPECT_EXIT(RunReqRep("shmem"), ::testing::ExitedWithCode(0), "REQ-REP test successfull");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(ReqRep, nanomsg)
-{
-    EXPECT_EXIT(RunReqRep("nanomsg"), ::testing::ExitedWithCode(0), "REQ-REP test successfull");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 } // namespace

test/protocols/config.json.in

@@ -27,32 +27,6 @@
                     }
                 ]
             },
-            {
-                "id": "pairleft_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5757",
-                        "method": "bind",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "pair"
-                    }
-                ]
-            },
-            {
-                "id": "pairright_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5757",
-                        "method": "connect",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "pair"
-                    }
-                ]
-            },
             {
                 "id": "pairleft_shmem",
                 "channels": [
@@ -131,32 +105,6 @@
                     }
                 ]
             },
-            {
-                "id": "push_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5757",
-                        "method": "bind",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "push"
-                    }
-                ]
-            },
-            {
-                "id": "pull_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5757",
-                        "method": "connect",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "pull"
-                    }
-                ]
-            },
             {
                 "id": "push_shmem",
                 "channels": [
@@ -204,27 +152,6 @@
                     }
                 ]
             },
-            {
-                "id": "pub_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5756",
-                        "method": "bind",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "pub"
-                    },
-                    {
-                        "address": "tcp://127.0.0.1:5755",
-                        "method": "bind",
-                        "name": "control",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "pull"
-                    }
-                ]
-            },
             {
                 "id": "sub_1zeromq",
                 "channels": [
@@ -267,48 +194,6 @@
                     }
                 ]
             },
-            {
-                "id": "sub_1nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5756",
-                        "method": "connect",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "sub"
-                    },
-                    {
-                        "address": "tcp://127.0.0.1:5755",
-                        "method": "connect",
-                        "name": "control",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "push"
-                    }
-                ]
-            },
-            {
-                "id": "sub_2nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5756",
-                        "method": "connect",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "sub"
-                    },
-                    {
-                        "address": "tcp://127.0.0.1:5755",
-                        "method": "connect",
-                        "name": "control",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "push"
-                    }
-                ]
-            },
             {
                 "id": "req_1zeromq",
                 "channels": [
@@ -322,19 +207,6 @@
                     }
                 ]
             },
-            {
-                "id": "req_1nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5758",
-                        "method": "connect",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "req"
-                    }
-                ]
-            },
             {
                 "id": "req_2zeromq",
                 "channels": [
@@ -348,19 +220,6 @@
                     }
                 ]
             },
-            {
-                "id": "req_2nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5758",
-                        "method": "connect",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "req"
-                    }
-                ]
-            },
             {
                 "id": "req_1shmem",
                 "channels": [
@@ -400,19 +259,6 @@
                     }
                 ]
             },
-            {
-                "id": "rep_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5758",
-                        "method": "bind",
-                        "name": "data",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "rep"
-                    }
-                ]
-            },
             {
                 "id": "rep_shmem",
                 "channels": [
@@ -468,27 +314,6 @@
                     }
                 ]
             },
-            {
-                "id": "transfer_timeout_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:5759",
-                        "method": "bind",
-                        "name": "data-in",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "pull"
-                    },
-                    {
-                        "address": "tcp://127.0.0.1:5560",
-                        "method": "bind",
-                        "name": "data-out",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "push"
-                    }
-                ]
-            },
             {
                 "id": "pollout_zeromq",
                 "channels": [
@@ -531,48 +356,6 @@
                     }
                 ]
             },
-            {
-                "id": "pollout_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:6002",
-                        "method": "bind",
-                        "name": "data1",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "push"
-                    },
-                    {
-                        "address": "tcp://127.0.0.1:6003",
-                        "method": "bind",
-                        "name": "data2",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "push"
-                    }
-                ]
-            },
-            {
-                "id": "pollin_nanomsg",
-                "channels": [
-                    {
-                        "address": "tcp://127.0.0.1:6002",
-                        "method": "connect",
-                        "name": "data1",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "pull"
-                    },
-                    {
-                        "address": "tcp://127.0.0.1:6003",
-                        "method": "connect",
-                        "name": "data2",
-                        "rateLogging": 0,
-                        "transport": "nanomsg",
-                        "type": "pull"
-                    }
-                ]
-            },
             {
                 "id": "pollout_shmem",
                 "channels": [

test/region/_region.cxx

@@ -0,0 +1,179 @@
+/********************************************************************************
+ *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include <FairMQLogger.h>
+#include <FairMQTransportFactory.h>
+#include <fairmq/ProgOptions.h>
+#include <fairmq/Tools.h>
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+namespace
+{
+
+using namespace std;
+
+void RegionEventSubscriptions(const string& transport)
+{
+    size_t session{fair::mq::tools::UuidHash()};
+
+    fair::mq::ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config);
+
+    constexpr int size1 = 1000000;
+    constexpr int size2 = 5000000;
+    constexpr int64_t userFlags = 12345;
+    fair::mq::tools::Semaphore blocker;
+
+    {
+        auto region1 = factory->CreateUnmanagedRegion(size1, [](void*, size_t, void*) {});
+        void* ptr1 = region1->GetData();
+        uint64_t id1 = region1->GetId();
+        ASSERT_EQ(region1->GetSize(), size1);
+
+        auto region2 = factory->CreateUnmanagedRegion(size2, userFlags, [](void*, size_t, void*) {});
+        void* ptr2 = region2->GetData();
+        uint64_t id2 = region2->GetId();
+        ASSERT_EQ(region2->GetSize(), size2);
+
+        ASSERT_EQ(factory->SubscribedToRegionEvents(), false);
+        factory->SubscribeToRegionEvents([&](FairMQRegionInfo info) {
+            LOG(warn) << ">>>" << info.event;
+            LOG(warn) << "id: " << info.id;
+            LOG(warn) << "ptr: " << info.ptr;
+            LOG(warn) << "size: " << info.size;
+            LOG(warn) << "flags: " << info.flags;
+            if (info.event == FairMQRegionEvent::created) {
+                if (info.id == id1) {
+                    ASSERT_EQ(info.size, size1);
+                    ASSERT_EQ(info.ptr, ptr1);
+                    blocker.Signal();
+                } else if (info.id == id2) {
+                    ASSERT_EQ(info.size, size2);
+                    ASSERT_EQ(info.ptr, ptr2);
+                    ASSERT_EQ(info.flags, userFlags);
+                    blocker.Signal();
+                }
+            } else if (info.event == FairMQRegionEvent::destroyed) {
+                if (info.id == id1) {
+                    blocker.Signal();
+                } else if (info.id == id2) {
+                    blocker.Signal();
+                }
+            }
+        });
+        ASSERT_EQ(factory->SubscribedToRegionEvents(), true);
+
+        LOG(info) << "waiting for blockers...";
+        blocker.Wait();
+        LOG(info) << "1 done.";
+        blocker.Wait();
+        LOG(info) << "2 done.";
+    }
+
+    blocker.Wait();
+    LOG(info) << "3 done.";
+    blocker.Wait();
+    LOG(info) << "4 done.";
+    LOG(info) << "All done.";
+
+    factory->UnsubscribeFromRegionEvents();
+    ASSERT_EQ(factory->SubscribedToRegionEvents(), false);
+}
+
+void RegionCallbacks(const string& transport, const string& _address)
+{
+    size_t session(fair::mq::tools::UuidHash());
+    std::string address(fair::mq::tools::ToString(_address, "_", transport));
+
+    fair::mq::ProgOptions config;
+    config.SetProperty<string>("session", to_string(session));
+
+    auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config);
+
+    unique_ptr<int> intPtr1 = fair::mq::tools::make_unique<int>(42);
+    unique_ptr<int> intPtr2 = fair::mq::tools::make_unique<int>(43);
+    fair::mq::tools::Semaphore blocker;
+
+    FairMQChannel push("Push", "push", factory);
+    push.Bind(address);
+
+    FairMQChannel pull("Pull", "pull", factory);
+    pull.Connect(address);
+
+    void* ptr1 = nullptr;
+    size_t size1 = 100;
+    void* ptr2 = nullptr;
+    size_t size2 = 200;
+
+    auto region1 = factory->CreateUnmanagedRegion(2000000, [&](void* ptr, size_t size, void* hint) {
+        ASSERT_EQ(ptr, ptr1);
+        ASSERT_EQ(size, size1);
+        ASSERT_EQ(hint, intPtr1.get());
+        ASSERT_EQ(*static_cast<int*>(hint), 42);
+        blocker.Signal();
+    });
+    ptr1 = region1->GetData();
+
+    auto region2 = factory->CreateUnmanagedRegion(3000000, [&](const std::vector<fair::mq::RegionBlock>& blocks) {
+        ASSERT_EQ(blocks.size(), 1);
+        ASSERT_EQ(blocks.at(0).ptr, ptr2);
+        ASSERT_EQ(blocks.at(0).size, size2);
+        ASSERT_EQ(blocks.at(0).hint, intPtr2.get());
+        ASSERT_EQ(*static_cast<int*>(blocks.at(0).hint), 43);
+        blocker.Signal();
+    });
+    ptr2 = region2->GetData();
+
+
+    {
+        FairMQMessagePtr msg1out(push.NewMessage(region1, ptr1, size1, intPtr1.get()));
+        FairMQMessagePtr msg2out(push.NewMessage(region2, ptr2, size2, intPtr2.get()));
+        ASSERT_EQ(push.Send(msg1out), size1);
+        ASSERT_EQ(push.Send(msg2out), size2);
+    }
+
+    {
+        FairMQMessagePtr msg1in(pull.NewMessage());
+        FairMQMessagePtr msg2in(pull.NewMessage());
+        ASSERT_EQ(pull.Receive(msg1in), size1);
+        ASSERT_EQ(pull.Receive(msg2in), size2);
+    }
+
+    LOG(info) << "waiting for blockers...";
+    blocker.Wait();
+    LOG(info) << "1 done.";
+    blocker.Wait();
+    LOG(info) << "2 done.";
+}
+
+TEST(EventSubscriptions, zeromq)
+{
+    RegionEventSubscriptions("zeromq");
+}
+
+TEST(EventSubscriptions, shmem)
+{
+    RegionEventSubscriptions("shmem");
+}
+
+TEST(Callbacks, zeromq)
+{
+    RegionCallbacks("zeromq", "ipc://test_region_callbacks");
+}
+
+TEST(Callbacks, shmem)
+{
+    RegionCallbacks("shmem", "ipc://test_region_callbacks");
+}
+
+} // namespace

test/transport/_options.cxx

@@ -25,18 +25,14 @@ namespace
 
 using namespace std;
 
-void CheckOldOptionInterface(FairMQChannel& channel, const string& option, const string& transport)
+void CheckOldOptionInterface(FairMQChannel& channel, const string& option)
 {
     int value = 500;
     channel.GetSocket().SetOption(option, &value, sizeof(value));
     value = 0;
     size_t valueSize = sizeof(value);
     channel.GetSocket().GetOption(option, &value, &valueSize);
-    if (transport == "nanomsg" && (option == "snd-hwm" || option == "rcv-hwm")) {
-        ASSERT_EQ(value, -1);
-    } else {
     ASSERT_EQ(value, 500);
-    }
 }
 
 void RunOptionsTest(const string& transport)
@@ -48,28 +44,20 @@ void RunOptionsTest(const string& transport)
     auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config);
     FairMQChannel channel("Push", "push", factory);
 
-    CheckOldOptionInterface(channel, "linger", transport);
-    CheckOldOptionInterface(channel, "snd-hwm", transport);
-    CheckOldOptionInterface(channel, "rcv-hwm", transport);
-    CheckOldOptionInterface(channel, "snd-size", transport);
-    CheckOldOptionInterface(channel, "rcv-size", transport);
+    CheckOldOptionInterface(channel, "linger");
+    CheckOldOptionInterface(channel, "snd-hwm");
+    CheckOldOptionInterface(channel, "rcv-hwm");
+    CheckOldOptionInterface(channel, "snd-size");
+    CheckOldOptionInterface(channel, "rcv-size");
 
     channel.GetSocket().SetLinger(300);
     ASSERT_EQ(channel.GetSocket().GetLinger(), 300);
 
     channel.GetSocket().SetSndBufSize(500);
-    if (transport == "nanomsg") { // nanomsg doesn't use this option and the getter always returns -1
-        ASSERT_EQ(channel.GetSocket().GetSndBufSize(), -1);
-    } else {
     ASSERT_EQ(channel.GetSocket().GetSndBufSize(), 500);
-    }
 
     channel.GetSocket().SetRcvBufSize(500);
-    if (transport == "nanomsg") { // nanomsg doesn't use this option and the getter always returns -1
-        ASSERT_EQ(channel.GetSocket().GetRcvBufSize(), -1);
-    } else {
     ASSERT_EQ(channel.GetSocket().GetRcvBufSize(), 500);
-    }
 
     channel.GetSocket().SetSndKernelSize(8000);
     ASSERT_EQ(channel.GetSocket().GetSndKernelSize(), 8000);
@@ -88,11 +76,4 @@ TEST(Options, shmem)
     RunOptionsTest("shmem");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(Options, nanomsg)
-{
-    RunOptionsTest("nanomsg");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 } // namespace

test/transport/_transfer_timeout.cxx

@@ -41,11 +41,4 @@ TEST(TransferTimeout, shmem)
     EXPECT_EXIT(RunTransferTimeout("shmem"), ::testing::ExitedWithCode(0), "Transfer timeout test successfull");
 }
 
-#ifdef BUILD_NANOMSG_TRANSPORT
-TEST(TransferTimeout, nanomsg)
-{
-    EXPECT_EXIT(RunTransferTimeout("nanomsg"), ::testing::ExitedWithCode(0), "Transfer timeout test successfull");
-}
-#endif /* BUILD_NANOMSG_TRANSPORT */
-
 } // namespace